netdata_netdata/.github/workflows/build.yml

1069 lines
41 KiB
YAML
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

---
# Ci code for building release artifacts.
name: Build
on:
push: # Master branch checks only validate the build and generate artifacts for testing.
branches:
- master
pull_request: null # PR checks only validate the build and generate artifacts for testing.
workflow_dispatch: # Dispatch runs build and validate, then push to the appropriate storage location.
inputs:
type:
description: Build Type
default: nightly
required: true
version:
description: Version Tag
default: nightly
required: true
concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type.
group: build-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
file-check: # Check what files changed if were being run in a PR or on a push.
name: Check Modified Files
runs-on: ubuntu-latest
outputs:
run: ${{ steps.check-run.outputs.run }}
skip-go: ${{ steps.check-go.outputs.skip-go }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Check source files
id: check-source-files
uses: tj-actions/changed-files@v44
with:
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
files: |
**/*.c
**/*.cc
**/*.h
**/*.hh
**/*.in
**/*.patch
src/aclk/aclk-schemas/
src/ml/dlib/
src/fluent-bit/
src/web/server/h2o/libh2o/
files_ignore: |
netdata.spec.in
**/*.md
- name: Check build files
id: check-build-files
uses: tj-actions/changed-files@v44
with:
since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
files: |
**/*.cmake
CMakeLists.txt
netdata-installer.sh
.github/data/distros.yml
.github/workflows/build.yml
.github/scripts/build-static.sh
.github/scripts/get-static-cache-key.sh
.github/scripts/gen-matrix-build.py
.github/scripts/run-updater-check.sh
packaging/cmake/
packaging/makeself/
packaging/installer/
packaging/*.sh
packaging/*.version
packaging/*.checksums
files_ignore: |
**/*.md
- name: List all changed files in pattern
continue-on-error: true
env:
CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
run: |
for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
echo "$file was changed"
done
- name: Check Run
id: check-run
run: |
if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo 'run=true' >> "${GITHUB_OUTPUT}"
else
echo 'run=false' >> "${GITHUB_OUTPUT}"
fi
- name: Check Go
id: check-go
env:
OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
run: |
if [ '${{ github.event_name }}' == 'pull_request' ]; then
if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
echo 'skip-go=' >> "${GITHUB_OUTPUT}"
else
echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
fi
else
echo 'skip-go=' >> "${GITHUB_OUTPUT}"
fi
build-dist: # Build the distribution tarball and store it as an artifact.
name: Build Distribution Tarball
runs-on: ubuntu-latest
needs:
- file-check
outputs:
distfile: ${{ steps.build.outputs.distfile }}
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Fix tags
id: fix-tags
if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
run: |
git fetch --tags --force
- name: Mark Stable
id: channel
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
run: |
sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh
- name: Build
id: build
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p artifacts/
tar --create --file "artifacts/netdata-$(git describe).tar.gz" \
--sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \
--exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \
--transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose .
cd artifacts/
echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}"
- name: Store
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: dist-tarball
path: artifacts/*.tar.gz
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Distribution tarball creation failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create source tarball for distribution.
Checkout: ${{ steps.checkout.outcome }}
Fix Tags: ${{ steps.fix-tags.outcome }}
Mark stable: ${{ steps.channel.outcome }}
Build: ${{ steps.build.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
build-static: # Build the static binary archives, and store them as artifacts.
name: Build Static
runs-on: ubuntu-latest
needs:
- file-check
strategy:
matrix:
arch:
- x86_64
- armv6l
- armv7l
- aarch64
- ppc64le
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: Fix tags
id: fix-tags
if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
run: |
git fetch --tags --force
- name: Mark Stable
id: channel
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
run: |
sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh packaging/makeself/install-or-update.sh
- name: Get Cache Key
if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
id: cache-key
run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }} "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache') }}"
- name: Cache
if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
id: cache
uses: actions/cache@v4
with:
path: artifacts/cache
key: ${{ steps.cache-key.outputs.key }}
- name: Build
if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Dont use retries on PRs.
run: |
export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
.github/scripts/build-static.sh ${{ matrix.arch }}
- name: Build
if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
id: build
uses: nick-fields/retry@v3
with:
timeout_minutes: 180
max_attempts: 3
command: |
export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
.github/scripts/build-static.sh ${{ matrix.arch }}
- name: Store
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: dist-static-${{ matrix.arch }}
path: artifacts/*.gz.run
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Static build failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}.
Checkout: ${{ steps.checkout.outcome }}
Fix Tags: ${{ steps.fix-tags.outcome }}
Mark stable: ${{ steps.channel.outcome }}
Build: ${{ steps.build.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
matrix: # Generate the shared build matrix for our build tests.
name: Prepare Build Matrix
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Prepare tools
id: prepare
run: |
sudo apt-get update || true
sudo apt-get install -y python3-ruamel.yaml
- name: Read build matrix
id: set-matrix
run: |
matrix="$(.github/scripts/gen-matrix-build.py)"
echo "Generated matrix: ${matrix}"
echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Build matrix preparation failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to prepare build matrix for build checks.
Checkout: ${{ steps.checkout.outcome }}
Prepare tools: ${{ steps.prepare.outcome }}
Read build matrix: ${{ steps.set-matrix.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
}}
prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
name: Prepare Test Environments
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- matrix
env:
RETRY_DELAY: 300
strategy:
# Unlike the actual build tests, this completes _very_ fast (average of about 3 minutes for each job), so we
# just run everything in parallel instead lof limiting job concurrency.
fail-fast: false
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Setup Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build test environment
id: build1
uses: docker/build-push-action@v5
continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Retry delay
if: ${{ steps.build1.outcome == 'failure' }}
run: sleep "${RETRY_DELAY}"
- name: Build test environment (attempt 2)
if: ${{ steps.build1.outcome == 'failure' }}
id: build2
uses: docker/build-push-action@v5
continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Retry delay
if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
run: sleep "${RETRY_DELAY}"
- name: Build test environment (attempt 3)
if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
id: build3
uses: docker/build-push-action@v5
with:
push: false
load: false
file: .github/dockerfiles/Dockerfile.build_test
build-args: |
BASE=${{ matrix.distro }}
PRE=${{ matrix.env_prep }}
RMJSONC=${{ matrix.jsonc_removal }}
outputs: type=docker,dest=/tmp/image.tar
tags: test:${{ matrix.artifact_key }}
- name: Upload image artifact
id: upload
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.artifact_key }}-test-env
path: /tmp/image.tar
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Test environment preparation for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Set up Buildx: ${{ steps.buildx.outcome }}
Build test environment: ${{ steps.build1.outcome }}
Build test environment (attempt 2): ${{ steps.build2.outcome }}
Build test environment (attempt 3): ${{ steps.build3.outcome }}
Upload: ${{ steps.upload.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
}}
source-build: # Test various source build arrangements.
name: Test Source Build
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- matrix
- prepare-test-images
- file-check
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
with:
submodules: recursive
- name: Fetch test environment
id: fetch
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: ${{ matrix.artifact_key }}-test-env
path: .
attempt_limit: 3
attempt_delay: 2000
- name: Load test environment
id: load
if: needs.file-check.outputs.run == 'true'
run: docker load --input image.tar
- name: netdata-installer on ${{ matrix.distro }}, disable cloud
id: build-no-cloud
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: netdata-installer on ${{ matrix.distro }}, require cloud
id: build-cloud
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
id: build-no-jsonc
if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
/bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Build tests for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch test environment: ${{ steps.fetch.outcome }}
Load test environment: ${{ steps.load.outcome }}
netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }}
netdata-installer, require cloud: ${{ steps.build-cloud.outcome }}
netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
updater-check: # Test the generated dist archive using the updater code.
name: Test Generated Distfile and Updater Code
runs-on: ubuntu-latest
if: github.event_name != 'workflow_dispatch'
needs:
- build-dist
- matrix
- prepare-test-images
- file-check
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
services:
apache: # This gets used to serve the dist tarball for the updater script.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch dist tarball artifacts
id: fetch-tarball
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: dist-tarball
path: dist-tarball
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifact directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p artifacts/download/v9999.0.0 || exit 1
mkdir -p artifacts/latest || exit 1
echo "v9999.0.0" > artifacts/latest/latest-version.txt || exit 1
cp dist-tarball/* artifacts/download/v9999.0.0 || exit 1
cd artifacts/download/v9999.0.0 || exit 1
ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
ls -lFh
sha256sum -b ./* > "sha256sums.txt" || exit 1
cat sha256sums.txt
cd ../.. || exit 1
ls -lR
- name: Fetch test environment
id: fetch-test-environment
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: ${{ matrix.artifact_key }}-test-env
path: .
attempt_limit: 3
attempt_delay: 2000
- name: Load test environment
id: load
if: needs.file-check.outputs.run == 'true'
run: docker load --input image.tar
- name: Install netdata and run the updater on ${{ matrix.distro }}
id: updater-check
if: needs.file-check.outputs.run == 'true'
run: |
docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
-e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch dist tarball: ${{ steps.fetch-tarball.outcome }}
Prepare artifact directory: ${{ steps.prepare.outcome }}
Fetch test environment: ${{ steps.fetch-test-environment.outcome }}
Load test environment: ${{ steps.load.outcome }}
Updater check: ${{ steps.updater-check.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
prepare-upload: # Consolidate the artifacts for uploading or releasing.
name: Prepare Artifacts
runs-on: ubuntu-latest
needs:
- build-dist
- build-static
- file-check
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Prepare Environment
id: prepare
if: needs.file-check.outputs.run == 'true'
run: mkdir -p artifacts
- name: Retrieve Build Artifacts
id: fetch-dist
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
pattern: dist-*
path: dist-artifacts
merge-multiple: true
attempt_limit: 3
attempt_delay: 2000
- name: Prepare Artifacts
id: consolidate
if: needs.file-check.outputs.run == 'true'
working-directory: ./artifacts/
run: |
mv ../dist-artifacts/* . || exit 1
ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
cp ../packaging/version ./latest-version.txt || exit 1
cp ../integrations/integrations.js ./integrations.js || exit 1
sha256sum -b ./* > sha256sums.txt || exit 1
cat sha256sums.txt
- name: Store Artifacts
id: store
if: needs.file-check.outputs.run == 'true'
uses: actions/upload-artifact@v4
with:
name: final-artifacts
path: artifacts/*
retention-days: 30
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to prepare release artifacts for upload.
Checkout: ${{ steps.checkout.outcome }}
Prepare environment: ${{ steps.prepare.outcome }}
Fetch dist tarball: ${{ steps.fetch-dist.outcome }}
Consolidate artifacts: ${{ steps.consolidate.outcome }}
Store: ${{ steps.store.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts.
name: Test Consolidated Artifacts (Source)
runs-on: ubuntu-latest
needs:
- prepare-upload
- file-check
services:
apache: # This gets used to serve the dist tarball for the updater script.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch artifacts
id: fetch
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifacts directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p download/latest
mv artifacts/* download/latest
ls -al download/latest
- name: Verify that artifacts work with installer
id: verify
if: needs.file-check.outputs.run == 'true'
env:
NETDATA_TARBALL_BASEURL: http://localhost:8080/
run: packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Artifact verification for source tarball failed.'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Artifact verification for source tarball failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Verify artifacts: ${{ steps.verify.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
artifact-verification-static: # Verify the static installer works with the consolidated artifacts.
name: Test Consolidated Artifacts (Static)
runs-on: ubuntu-latest
needs:
- prepare-upload
- file-check
services:
apache: # This gets used to serve the static archives.
image: httpd:2.4
ports:
- 8080:80
volumes:
- ${{ github.workspace }}:/usr/local/apache2/htdocs/
steps:
- name: Skip Check
id: skip
if: needs.file-check.outputs.run != 'true'
run: echo "SKIPPED"
- name: Checkout
id: checkout
if: needs.file-check.outputs.run == 'true'
uses: actions/checkout@v4
- name: Fetch artifacts
id: fetch-artifacts
if: needs.file-check.outputs.run == 'true'
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare artifacts directory
id: prepare
if: needs.file-check.outputs.run == 'true'
run: |
mkdir -p download/latest
mv artifacts/* download/latest
ls -al download/latest
- name: Verify that artifacts work with installer
id: verify
if: needs.file-check.outputs.run == 'true'
env:
NETDATA_TARBALL_BASEURL: http://localhost:8080/
run: packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Artifact verification for static build failed.'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Artifact verification for static build failed.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
Verify artifacts: ${{ steps.verify.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
&& github.repository == 'netdata/netdata'
&& needs.file-check.outputs.run == 'true'
}}
upload-nightly: # Upload the nightly build artifacts to GCS.
name: Upload Nightly Artifacts
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
steps:
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Authenticate to GCS
id: gcs-auth
uses: google-github-actions/auth@v2
with:
project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }}
credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }}
- name: Setup GCS
id: gcs-setup
uses: google-github-actions/setup-gcloud@v2.1.0
- name: Upload Artifacts
id: upload
uses: google-github-actions/upload-cloud-storage@v2.1.0
with:
destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }}
gzip: false
path: ./final-artifacts/latest-version.txt
parent: false
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to upload nightly release artifacts:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to upload nightly release artifacts.
Fetch artifacts: ${{ steps.fetch.outcome }}
Authenticatie GCS: ${{ steps.gcs-auth.outcome }}
Setup GCS: ${{ steps.gcs-setup.outcome }}
Upload artifacts: ${{ steps.upload.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& startsWith(github.ref, 'refs/heads/master')
&& github.event_name != 'pull_request'
}}
create-nightly: # Create a nightly build release in netdata/netdata-nightlies
name: Create Nightly Release
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
steps:
- name: Checkout Main Repo
id: checkout-main
uses: actions/checkout@v4
with:
path: main
- name: Checkout Nightly Repo
id: checkout-nightly
uses: actions/checkout@v4
with:
repository: netdata/netdata-nightlies
path: nightlies
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Prepare version info
id: version
run: |
# shellcheck disable=SC2129
echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}"
echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}"
- name: Create Release
id: create-release
uses: ncipollo/release-action@v1
with:
allowUpdates: false
artifactErrorsFailBuild: true
artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
owner: netdata
repo: netdata-nightlies
body: Netdata nightly build for ${{ steps.version.outputs.date }}.
commit: ${{ steps.version.outputs.commit }}
makeLatest: true
tag: ${{ steps.version.outputs.version }}
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions
id: checkout-netdata
uses: actions/checkout@v4
with:
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Init python environment for publish release metadata
uses: actions/setup-python@v5
id: init-python
with:
python-version: "3.12"
- name: Setup python environment
id: setup-python
run: |
pip install -r .github/scripts/modules/requirements.txt
- name: Check if the version is latest and published
id: check-latest-version
run: |
python .github/scripts/check_latest_versions.py ${{ steps.version.outputs.version }}
- name: SSH setup
id: ssh-setup
if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
uses: shimataro/ssh-key-action@v2
with:
key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
name: id_ecdsa
known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
- name: Sync newer releases
id: sync-releases
if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
run: |
.github/scripts/upload-new-version-tags.sh
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create nightly release or attach artifacts.
Checkout netdata/netdata: ${{ steps.checkout-main.outcome }}
Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Prepare version info: ${{ steps.version.outcome }}
Create release: ${{ steps.create-release.outcome }}
Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }}
Init python environment: ${{ steps.init-python.outcome }}
Setup python environment: ${{ steps.setup-python.outcome }}
Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }}
Setup ssh: ${{ steps.ssh-setup.outcome }}
Sync with the releases: ${{ steps.sync-releases.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& github.event_name == 'workflow_dispatch'
}}
normalize-tag: # Fix the release tag if needed
name: Normalize Release Tag
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
outputs:
tag: ${{ steps.tag.outputs.tag }}
steps:
- name: Normalize Tag
id: tag
run: |
if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
else
echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
fi
upload-release: # Create the draft release and upload the build artifacts.
name: Create Release Draft
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' && github.repository == 'netdata/netdata'
needs:
- artifact-verification-dist
- artifact-verification-static
- normalize-tag
steps:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Retrieve Artifacts
id: fetch
uses: Wandalen/wretry.action@v3
with:
action: actions/download-artifact@v4
with: |
name: final-artifacts
path: final-artifacts
attempt_limit: 3
attempt_delay: 2000
- name: Create Release
id: create-release
uses: ncipollo/release-action@v1
with:
allowUpdates: false
artifactErrorsFailBuild: true
artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
draft: true
tag: ${{ needs.normalize-tag.outputs.tag }}
token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
- name: Failure Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'danger'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Failed to draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: |-
${{ github.repository }}: Failed to create draft release or attach artifacts.
Checkout: ${{ steps.checkout.outcome }}
Fetch artifacts: ${{ steps.fetch.outcome }}
Create draft release: ${{ steps.create-release.outcome }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
failure()
&& github.event_name == 'workflow_dispatch'
}}
- name: Success Notification
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: 'good'
SLACK_FOOTER: ''
SLACK_ICON_EMOJI: ':github-actions:'
SLACK_TITLE: 'Created agent draft release:'
SLACK_USERNAME: 'GitHub Actions'
SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
if: >-
${{
success()
&& github.event_name == 'workflow_dispatch'
}}