0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-02 20:48:06 +00:00

Add Widnows CI jobs. ()

* Initial Windows CI support.

This also adds a proper set of production-focused packaging scripts
for working with building Netdata on Windows.

* Pivot to using existing tooling and fixing that up as required.

* General script cleanup.

* Auto-detect location of repository based on script location.

This makes the scripts properly independent of the current working
directory, allowing them to be more resilient to possible changes in the
build environment.

* Make dependency handling entirely separate from build.

* Remove development-only packages from dependency list.

They only waste time in CI, and it’s reasonable to assume that anybody
who needs them can install them themselves, just like we do with all
other platforms.

* Use absolute path for makensis.

This way it actually works without needing manual modification of the
environment.

* Add PowerShell scripts to invoke the build and package scripts.

* Clean up build directory handling.

* Fix fetching of msys2 installer.

* Further cleanup.

* Tidy up build output

* Provide proper output grouping in GitHub Actions.

* Use correct command name for makensis.

* Fix installer script path.

* Do not try to install MSYS2 using Chocolatey.

* Fix CPU architecture check.

* Add more detailed messages for MSYS2 install.

* Fix release fetching logic.

* Further release check logic fixes.

* Complete Windows CI support.

* Move second update pass into dependency handling script.

* Fix handling of icon for Windows installer.

* Default to using a working configuration for release builds.

And provide a way to pass extra options to CMake without needing to
modify the script.

* Fix expansion of optionally defined variable.
This commit is contained in:
Austin S. Hemmelgarn 2024-07-18 13:28:45 -04:00 committed by GitHub
parent 124757a2fa
commit 9e575dbec8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 445 additions and 110 deletions

View file

@ -70,6 +70,7 @@ jobs:
packaging/cmake/
packaging/makeself/
packaging/installer/
packaging/windows/
packaging/*.sh
packaging/*.version
packaging/*.checksums
@ -975,6 +976,7 @@ jobs:
}}
macos-build: # Test building on macOS
name: Test building on macOS
runs-on: ${{ matrix.runner }}
if: github.event_name != 'workflow_dispatch'
needs:
@ -1045,6 +1047,44 @@ jobs:
&& github.repository == 'netdata/netdata'
}}
windows-build: # Test building on Windows
name: Test building on Windows
runs-on: windows-latest
if: github.event_name != 'workflow_dispatch'
needs:
- file-check
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: Write-Output "SKIPPED"
- name: Checkout
uses: actions/checkout@v4
id: checkout
if: needs.file-check.outputs.run == 'true'
with:
submodules: recursive
lfs: true
- name: Set Up Dependencies
id: deps
if: needs.file-check.outputs.run == 'true'
run: ./packaging/windows/install-dependencies.ps1
- name: Build Netdata
id: build
if: needs.file-check.outputs.run == 'true'
run: ./packaging/windows/build.ps1
- name: Package Netdata
id: package
if: needs.file-check.outputs.run == 'true'
run: ./packaging/windows/package.ps1
- name: Upload Installer
id: upload
uses: actions/upload-artifact@v4
with:
name: windows-x86_64-installer
path: packaging\windows\netdata-installer.exe
retention-days: 30
updater-check: # Test the generated dist archive using the updater code.
name: Test Generated Distfile and Updater Code
runs-on: ubuntu-latest

View file

@ -1,79 +0,0 @@
#!/bin/sh
# On MSYS2, install these dependencies to build netdata:
install_dependencies() {
pacman -S \
git cmake ninja base-devel msys2-devel \
libyaml-devel libzstd-devel libutil-linux libutil-linux-devel \
mingw-w64-x86_64-toolchain mingw-w64-ucrt-x86_64-toolchain \
mingw64/mingw-w64-x86_64-mold ucrt64/mingw-w64-ucrt-x86_64-mold \
msys/gdb ucrt64/mingw-w64-ucrt-x86_64-gdb mingw64/mingw-w64-x86_64-gdb \
msys/zlib-devel mingw64/mingw-w64-x86_64-zlib ucrt64/mingw-w64-ucrt-x86_64-zlib \
msys/libuv-devel ucrt64/mingw-w64-ucrt-x86_64-libuv mingw64/mingw-w64-x86_64-libuv \
liblz4-devel mingw64/mingw-w64-x86_64-lz4 ucrt64/mingw-w64-ucrt-x86_64-lz4 \
openssl-devel mingw64/mingw-w64-x86_64-openssl ucrt64/mingw-w64-ucrt-x86_64-openssl \
protobuf-devel mingw64/mingw-w64-x86_64-protobuf ucrt64/mingw-w64-ucrt-x86_64-protobuf \
msys/pcre2-devel mingw64/mingw-w64-x86_64-pcre2 ucrt64/mingw-w64-ucrt-x86_64-pcre2 \
msys/brotli-devel mingw64/mingw-w64-x86_64-brotli ucrt64/mingw-w64-ucrt-x86_64-brotli \
msys/ccache ucrt64/mingw-w64-ucrt-x86_64-ccache mingw64/mingw-w64-x86_64-ccache \
mingw64/mingw-w64-x86_64-go ucrt64/mingw-w64-ucrt-x86_64-go \
mingw64/mingw-w64-x86_64-nsis
}
if [ "${1}" = "install" ]
then
install_dependencies || exit 1
exit 0
fi
BUILD_FOR_PACKAGING="Off"
if [ "${1}" = "package" ]
then
BUILD_FOR_PACKAGING="On"
fi
export PATH="/usr/local/bin:${PATH}"
WT_ROOT="$(pwd)"
BUILD_TYPE="Debug"
NULL=""
if [ -z "${MSYSTEM}" ]; then
build="${WT_ROOT}/build-${OSTYPE}"
else
build="${WT_ROOT}/build-${OSTYPE}-${MSYSTEM}"
fi
if [ "$USER" = "vk" ]; then
build="${WT_ROOT}/build"
fi
set -exu -o pipefail
if [ -d "${build}" ]
then
rm -rf "${build}"
fi
/usr/bin/cmake -S "${WT_ROOT}" -B "${build}" \
-G Ninja \
-DCMAKE_INSTALL_PREFIX="/opt/netdata" \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DCMAKE_C_FLAGS="-fstack-protector-all -O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \
-DBUILD_FOR_PACKAGING=${BUILD_FOR_PACKAGING} \
-DUSE_MOLD=Off \
-DNETDATA_USER="${USER}" \
-DDEFAULT_FEATURE_STATE=Off \
-DENABLE_H2O=Off \
-DENABLE_ACLK=On \
-DENABLE_CLOUD=On \
-DENABLE_ML=On \
-DENABLE_BUNDLED_JSONC=On \
-DENABLE_BUNDLED_PROTOBUF=Off \
${NULL}
ninja -v -C "${build}" || ninja -v -C "${build}" -j 1
echo
echo "Compile with:"
echo "ninja -v -C \"${build}\" || ninja -v -C \"${build}\" -j 1"

View file

@ -1,31 +0,0 @@
#!/bin/bash
export PATH="/usr/local/bin:${PATH}"
WT_ROOT="$(pwd)"
if [ -z "${MSYSTEM}" ]; then
build="${WT_ROOT}/build-${OSTYPE}"
else
build="${WT_ROOT}/build-${OSTYPE}-${MSYSTEM}"
fi
if [ "$USER" = "vk" ]; then
build="${WT_ROOT}/build"
fi
set -exu -o pipefail
ninja -v -C "${build}" install
if [ ! -f "/msys2-installer.exe" ]; then
wget -O /msys2-installer.exe \
"https://github.com/msys2/msys2-installer/releases/download/2024-05-07/msys2-x86_64-20240507.exe"
fi
NDVERSION=$"$(grep 'CMAKE_PROJECT_VERSION:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
NDMAJORVERSION=$"$(grep 'CMAKE_PROJECT_VERSION_MAJOR:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
NDMINORVERSION=$"$(grep 'CMAKE_PROJECT_VERSION_MINOR:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
makensis -DCURRVERSION="${NDVERSION}" -DMAJORVERSION="${NDMAJORVERSION}" -DMINORVERSION="${NDMINORVERSION}" "${WT_ROOT}/packaging/utils/installer.nsi"

View file

Before

(image error) Size: 15 KiB

After

(image error) Size: 15 KiB

View file

View file

@ -0,0 +1,16 @@
# Run the build
#Requires -Version 4.0
$ErrorActionPreference = "Stop"
. "$PSScriptRoot\functions.ps1"
$msysbash = Get-MSYS2Bash "$msysprefix"
$env:CHERE_INVOKING = 'yes'
& $msysbash -l "$PSScriptRoot\compile-on-windows.sh"
if ($LastExitcode -ne 0) {
exit 1
}

View file

@ -0,0 +1,53 @@
#!/bin/bash
repo_root="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd -P)")")"
BUILD_TYPE="Debug"
if [ -n "${BUILD_DIR}" ]; then
build="${BUILD_DIR}"
elif [ -n "${OSTYPE}" ]; then
if [ -n "${MSYSTEM}" ]; then
build="${repo_root}/build-${OSTYPE}-${MSYSTEM}"
else
build="${repo_root}/build-${OSTYPE}"
fi
elif [ "$USER" = "vk" ]; then
build="${repo_root}/build"
else
build="${repo_root}/build"
fi
set -exu -o pipefail
if [ -d "${build}" ]; then
rm -rf "${build}"
fi
${GITHUB_ACTIONS+echo "::group::Configuring"}
# shellcheck disable=SC2086
/usr/bin/cmake -S "${repo_root}" -B "${build}" \
-G Ninja \
-DCMAKE_INSTALL_PREFIX="/opt/netdata" \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DCMAKE_C_FLAGS="-fstack-protector-all -O0 -ggdb -Wall -Wextra -Wno-char-subscripts -Wa,-mbig-obj -pipe -DNETDATA_INTERNAL_CHECKS=1 -D_FILE_OFFSET_BITS=64 -D__USE_MINGW_ANSI_STDIO=1" \
-DBUILD_FOR_PACKAGING=On \
-DNETDATA_USER="${USER}" \
-DDEFAULT_FEATURE_STATE=Off \
-DENABLE_H2O=Off \
-DENABLE_ACLK=On \
-DENABLE_CLOUD=On \
-DENABLE_ML=On \
-DENABLE_BUNDLED_JSONC=On \
-DENABLE_BUNDLED_PROTOBUF=Off \
${EXTRA_CMAKE_OPTIONS:-''}
${GITHUB_ACTIONS+echo "::endgroup::"}
${GITHUB_ACTIONS+echo "::group::Building"}
ninja -C "${build}" -k 1
${GITHUB_ACTIONS+echo "::endgroup::"}
if [ -t 1 ]; then
echo
echo "Compile with:"
echo "ninja -v -C \"${build}\" || ninja -v -C \"${build}\" -j 1"
fi

View file

@ -0,0 +1,101 @@
#!/usr/bin/env python
'''Fetch the MSYS2 installer.'''
from __future__ import annotations
import hashlib
import json
import shutil
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Final
from urllib.request import Request, urlopen
REPO: Final = 'msys2/msys2-installer'
def get_latest_release() -> tuple[str, str]:
'''Get the latest release for the repo.'''
REQUEST: Final = Request(
url=f'https://api.github.com/repos/{REPO}/releases',
headers={
'Accept': 'application/vnd.github+json',
'X-GitHub-API-Version': '2022-11-28',
},
method='GET',
)
print('>>> Fetching release list')
with urlopen(REQUEST, timeout=15) as response:
if response.status != 200:
print(f'!!! Failed to fetch release list, status={response.status}')
sys.exit(1)
data = json.load(response)
data = list(filter(lambda x: x['name'] != 'Nightly Installer Build', data))
name = data[0]['name']
version = data[0]['tag_name'].replace('-', '')
return name, version
def fetch_release_asset(tmpdir: Path, name: str, file: str) -> Path:
'''Fetch a specific release asset.'''
REQUEST: Final = Request(
url=f'https://github.com/{REPO}/releases/download/{name}/{file}',
method='GET',
)
TARGET: Final = tmpdir / file
print(f'>>> Downloading {file}')
with urlopen(REQUEST, timeout=15) as response:
if response.status != 200:
print(f'!!! Failed to fetch {file}, status={response.status}')
sys.exit(1)
TARGET.write_bytes(response.read())
return TARGET
def main() -> None:
'''Core program logic.'''
if len(sys.argv) != 2:
print(f'{__file__} must be run with exactly one argument.')
target = Path(sys.argv[1])
tmp_target = target.with_name(f'.{target.name}.tmp')
name, version = get_latest_release()
with TemporaryDirectory() as tmpdir:
tmppath = Path(tmpdir)
installer = fetch_release_asset(tmppath, name, f'msys2-x86_64-{version}.exe')
checksums = fetch_release_asset(tmppath, name, f'msys2-x86_64-{version}.exe.sha256')
print('>>> Verifying SHA256 checksum')
expected_checksum = checksums.read_text().partition(' ')[0].casefold()
actual_checksum = hashlib.sha256(installer.read_bytes()).hexdigest().casefold()
if expected_checksum != actual_checksum:
print('!!! Checksum mismatch')
print(f'!!! Expected: {expected_checksum}')
print(f'!!! Actual: {actual_checksum}')
sys.exit(1)
print(f'>>> Copying to {target}')
shutil.copy(installer, tmp_target)
tmp_target.replace(target)
if __name__ == '__main__':
main()

View file

@ -0,0 +1,31 @@
# Functions used by the PowerShell scripts in this directory.
#Requires -Version 4.0
function Get-MSYS2Prefix {
if (-Not ($msysprefix)) {
if (Test-Path -Path C:\msys64\usr\bin\bash.exe) {
return "C:\msys64"
} elseif ($env:ChocolateyToolsLocation) {
if (Test-Path -Path "$env:ChocolateyToolsLocation\msys64\usr\bin\bash.exe") {
Write-Host "Found MSYS2 installed via Chocolatey"
Write-Host "This will work for building Netdata, but not for packaging it"
return "$env:ChocolateyToolsLocation\msys64"
}
}
}
return ""
}
function Get-MSYS2Bash {
$msysprefix = $args[0]
if (-Not ($msysprefix)) {
$msysprefix = Get-MSYS2Prefix
}
Write-Host "Using MSYS2 from $msysprefix"
return "$msysprefix\usr\bin\bash.exe"
}

View file

@ -0,0 +1,84 @@
# Set up Windows build dependencies.
#
# This script first sees if msys is installed. If so, it just uses it. If not, it tries to bootstrap it with chocolatey or winget.
#Requires -Version 4.0
$ErrorActionPreference = "Stop"
. "$PSScriptRoot\functions.ps1"
$msysprefix = Get-MSYS2Prefix
function Check-FileHash {
$file_path = $args[0]
Write-Host "Checking SHA256 hash of $file_path"
$actual_hash = (Get-FileHash -Algorithm SHA256 -Path $file_path).Hash.toLower()
$expected_hash = (Get-Content "$file_path.sha256").split()[0]
if ($actual_hash -ne $expected_hash) {
Write-Host "SHA256 hash mismatch!"
Write-Host "Expected: $expected_hash"
Write-Host "Actual: $actual_hash"
exit 1
}
}
function Install-MSYS2 {
$repo = 'msys2/msys2-installer'
$uri = "https://api.github.com/repos/$repo/releases"
$headers = @{
'Accept' = 'application/vnd.github+json'
'X-GitHub-API-Version' = '2022-11-28'
}
$installer_path = "$env:TEMP\msys2-base.exe"
if ($env:PROCESSOR_ARCHITECTURE -ne "AMD64") {
Write-Host "We can only install MSYS2 for 64-bit x86 systems, but you appear to have a different processor architecture ($env:PROCESSOR_ARCHITECTURE)."
Write-Host "You will need to install MSYS2 yourself instead."
exit 1
}
Write-Host "Determining latest release"
$release_list = Invoke-RestMethod -Uri $uri -Headers $headers -TimeoutSec 30
$release = $release_list[0]
$release_name = $release.name
$version = $release.tag_name.Replace('-', '')
$installer_url = "https://github.com/$repo/releases/download/$release_name/msys2-x86_64-$version.exe"
Write-Host "Fetching $installer_url"
Invoke-WebRequest $installer_url -OutFile $installer_path
Write-Host "Fetching $installer_url.sha256"
Invoke-WebRequest "$installer_url.sha256" -OutFile "$installer_path.sha256"
Write-Host "Checking file hash"
Check-FileHash $installer_path
Write-Host "Installing"
& $installer_path in --confirm-command --accept-messages --root C:/msys64
return "C:\msys64"
}
if (-Not ($msysprefix)) {
Write-Host "Could not find MSYS2, attempting to install it"
$msysprefix = Install-MSYS2
}
$msysbash = Get-MSYS2Bash "$msysprefix"
$env:CHERE_INVOKING = 'yes'
& $msysbash -l "$PSScriptRoot\msys2-dependencies.sh"
if ($LastExitcode -ne 0) {
Write-Host "First update attempt failed. This is expected if the msys-runtime package needed updated, trying again."
& $msysbash -l "$PSScriptRoot\msys2-dependencies.sh"
if ($LastExitcode -ne 0) {
exit 1
}
}

View file

@ -0,0 +1,16 @@
# Invoke the specified script using MSYS2
#Requires -Version 4.0
$ErrorActionPreference = "Stop"
. "$PSScriptRoot\functions.ps1"
$msysbash = Get-MSYS2Bash "$msysprefix"
$env:CHERE_INVOKING = 'yes'
& $msysbash -l $args[0]
if ($LastExitcode -ne 0) {
exit 1
}

View file

@ -0,0 +1,51 @@
#!/bin/bash
#
# Install the dependencies we need to build Netdata on MSYS2
. /etc/profile
set -euo pipefail
${GITHUB_ACTIONS+echo "::group::Updating MSYS2"}
pacman -Syuu --noconfirm
${GITHUB_ACTIONS+echo "::endgroup::"}
${GITHUB_ACTIONS+echo "::group::Installing dependencies"}
pacman -S --noconfirm \
base-devel \
cmake \
git \
liblz4-devel \
libutil-linux \
libutil-linux-devel \
libyaml-devel \
libzstd-devel \
mingw64/mingw-w64-x86_64-brotli \
mingw64/mingw-w64-x86_64-go \
mingw64/mingw-w64-x86_64-libuv \
mingw64/mingw-w64-x86_64-lz4 \
mingw64/mingw-w64-x86_64-nsis \
mingw64/mingw-w64-x86_64-openssl \
mingw64/mingw-w64-x86_64-pcre2 \
mingw64/mingw-w64-x86_64-protobuf \
mingw64/mingw-w64-x86_64-zlib \
mingw-w64-ucrt-x86_64-toolchain \
mingw-w64-x86_64-toolchain \
msys2-devel \
msys/brotli-devel \
msys/libuv-devel \
msys/pcre2-devel \
msys/zlib-devel \
ninja \
openssl-devel \
protobuf-devel \
python \
ucrt64/mingw-w64-ucrt-x86_64-brotli \
ucrt64/mingw-w64-ucrt-x86_64-go \
ucrt64/mingw-w64-ucrt-x86_64-libuv \
ucrt64/mingw-w64-ucrt-x86_64-lz4 \
ucrt64/mingw-w64-ucrt-x86_64-openssl \
ucrt64/mingw-w64-ucrt-x86_64-pcre2 \
ucrt64/mingw-w64-ucrt-x86_64-protobuf \
ucrt64/mingw-w64-ucrt-x86_64-zlib
${GITHUB_ACTIONS+echo "::endgroup::"}

View file

@ -0,0 +1,37 @@
#!/bin/bash
repo_root="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd -P)")")"
if [ -n "${BUILD_DIR}" ]; then
build="${BUILD_DIR}"
elif [ -n "${OSTYPE}" ]; then
if [ -n "${MSYSTEM}" ]; then
build="${repo_root}/build-${OSTYPE}-${MSYSTEM}"
else
build="${repo_root}/build-${OSTYPE}"
fi
elif [ "$USER" = "vk" ]; then
build="${repo_root}/build"
else
build="${repo_root}/build"
fi
set -exu -o pipefail
${GITHUB_ACTIONS+echo "::group::Installing"}
ninja -C "${build}" -k 1 install
${GITHUB_ACTIONS+echo "::endgroup::"}
if [ ! -f "/msys2-installer.exe" ]; then
${GITHUB_ACTIONS+echo "::group::Fetching MSYS2 installer"}
"${repo_root}/packaging/windows/fetch-msys2-installer.py" /msys2-installer.exe
${GITHUB_ACTIONS+echo "::endgroup::"}
fi
${GITHUB_ACTIONS+echo "::group::Packaging"}
NDVERSION=$"$(grep 'CMAKE_PROJECT_VERSION:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
NDMAJORVERSION=$"$(grep 'CMAKE_PROJECT_VERSION_MAJOR:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
NDMINORVERSION=$"$(grep 'CMAKE_PROJECT_VERSION_MINOR:STATIC' "${build}/CMakeCache.txt"| cut -d= -f2)"
/mingw64/bin/makensis.exe -DCURRVERSION="${NDVERSION}" -DMAJORVERSION="${NDMAJORVERSION}" -DMINORVERSION="${NDMINORVERSION}" "${repo_root}/packaging/windows/installer.nsi"
${GITHUB_ACTIONS+echo "::endgroup::"}

View file

@ -0,0 +1,16 @@
# Package the build
#Requires -Version 4.0
$ErrorActionPreference = "Stop"
. "$PSScriptRoot\functions.ps1"
$msysbash = Get-MSYS2Bash "$msysprefix"
$env:CHERE_INVOKING = 'yes'
& $msysbash -l "$PSScriptRoot\package-windows.sh"
if ($LastExitcode -ne 0) {
exit 1
}