Skip Go code in CI if it hasn’t changed. (#17077)

* Skip building Go components for Docker CI if they have not changed.

* Properly handle Go code in general checks PR.

* Skip Go code in build checks if it hasn’t changed.

* Fix linting issues.

* Fix propagation of installer flags.

* Fix propagation of environment variables through static build process.

* Fix handling of extra install options in static builds.

* Skip starting the agent in updater checks.

* Fix actionlint warning.

.github/scripts/build-static.sh

@@ -22,7 +22,7 @@ prepare_build() {
 build_static() {
   progress "Building static ${BUILDARCH}"
   (
-    USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
+    EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
   ) >&2
 }
 

.github/scripts/run-updater-check.sh

@@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..."
 /netdata/.github/scripts/ci-support-pkgs.sh
 mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms.
 echo ">>> Installing Netdata..."
-/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
+/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1
 echo "::group::>>> Pre-Update Environment File Contents"
 cat /etc/netdata/.environment
 echo "::endgroup::"

.github/workflows/build.yml

@@ -25,6 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
     steps:
       - name: Checkout
         id: checkout
@@ -32,8 +33,8 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Check files
+      - name: Check source files
-        id: check-files
+        id: check-source-files
         uses: tj-actions/changed-files@v44
         with:
           since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@@ -44,6 +45,19 @@ jobs:
             **/*.hh
             **/*.in
             **/*.patch
+            src/aclk/aclk-schemas/
+            src/ml/dlib/
+            src/fluent-bit/
+            src/web/server/h2o/libh2o/
+          files_ignore: |
+            netdata.spec.in
+            **/*.md
+      - name: Check build files
+        id: check-build-files
+        uses: tj-actions/changed-files@v43
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
             **/*.cmake
             CMakeLists.txt
             netdata-installer.sh
@@ -59,29 +73,39 @@ jobs:
             packaging/*.sh
             packaging/*.version
             packaging/*.checksums
-            src/aclk/aclk-schemas/
-            src/ml/dlib/
-            src/fluent-bit/
-            src/web/server/h2o/libh2o/
           files_ignore: |
-            netdata.spec.in
             **/*.md
       - name: List all changed files in pattern
         continue-on-error: true
         env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
         run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
             echo "$file was changed"
           done
       - name: Check Run
         id: check-run
         run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo 'run=true' >> "${GITHUB_OUTPUT}"
           else
             echo 'run=false' >> "${GITHUB_OUTPUT}"
           fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi
 
   build-dist: # Build the distribution tarball and store it as an artifact.
     name: Build Distribution Tarball
@@ -204,7 +228,9 @@ jobs:
           key: ${{ steps.cache-key.outputs.key }}
       - name: Build
         if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
-        run: .github/scripts/build-static.sh ${{ matrix.arch }}
+        run: |
+          export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
+          .github/scripts/build-static.sh ${{ matrix.arch }}
       - name: Build
         if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
         id: build
@@ -212,7 +238,9 @@ jobs:
         with:
           timeout_minutes: 180
           max_attempts: 3
-          command: .github/scripts/build-static.sh ${{ matrix.arch }}
+          command: |
+            export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
+            .github/scripts/build-static.sh ${{ matrix.arch }}
       - name: Store
         id: store
         if: needs.file-check.outputs.run == 'true'
@@ -432,19 +460,19 @@ jobs:
         if: needs.file-check.outputs.run == 'true'
         run: |
           docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build'
+              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
       - name: netdata-installer on ${{ matrix.distro }}, require cloud
         id: build-cloud
         if: needs.file-check.outputs.run == 'true'
         run: |
           docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
+              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
       - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
         id: build-no-jsonc
         if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
         run: |
           docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
+              /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -545,8 +573,9 @@ jobs:
         id: updater-check
         if: needs.file-check.outputs.run == 'true'
         run: |
-          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata test:${{ matrix.artifact_key }} \
+          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
-                /netdata/.github/scripts/run-updater-check.sh
+                 -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
+                 test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:

.github/workflows/checks.yml

@@ -16,6 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
     steps:
       - name: Checkout
         id: checkout
@@ -23,8 +24,8 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Check files
+      - name: Check source files
-        id: check-files
+        id: check-source-files
         uses: tj-actions/changed-files@v44
         with:
           since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@@ -35,18 +36,6 @@ jobs:
             **/*.hh
             **/*.in
             **/*.patch
-            **/*.cmake
-            CMakeLists.txt
-            .gitignore
-            .github/data/distros.yml
-            .github/workflows/build.yml
-            .github/scripts/build-static.sh
-            .github/scripts/get-static-cache-key.sh
-            .github/scripts/gen-matrix-build.py
-            .github/scripts/run-updater-check.sh
-            packaging/cmake/
-            packaging/*.version
-            packaging/*.checksums
             src/aclk/aclk-schemas/
             src/ml/dlib/
             src/fluent-bit/
@@ -54,22 +43,53 @@ jobs:
           files_ignore: |
             netdata.spec.in
             **/*.md
+      - name: Check build files
+        id: check-build-files
+        uses: tj-actions/changed-files@v43
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
+            **/*.cmake
+            CMakeLists.txt
+            .gitignore
+            .github/data/distros.yml
+            .github/workflows/build.yml
+            packaging/cmake/
+            packaging/*.version
+            packaging/*.checksums
+          files_ignore: |
+            **/*.md
       - name: List all changed files in pattern
         continue-on-error: true
         env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
         run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
             echo "$file was changed"
           done
       - name: Check Run
         id: check-run
         run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo 'run=true' >> "${GITHUB_OUTPUT}"
           else
             echo 'run=false' >> "${GITHUB_OUTPUT}"
           fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi
 
   libressl-checks:
     name: LibreSSL
@@ -94,7 +114,8 @@ jobs:
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata;
           apk del openssl openssl-dev;
           apk add libressl libressl-dev protobuf-dev;
-          ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;'
+          ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build --disable-go;'
+
   clang-checks:
     name: Clang
     needs:
@@ -134,7 +155,7 @@ jobs:
         run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
       - name: Build netdata
         if: needs.file-check.outputs.run == 'true'
-        run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build
+        run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }}
       - name: Check that repo is clean
         if: needs.file-check.outputs.run == 'true'
         run: |

.github/workflows/docker.yml

@@ -31,6 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
     steps:
       - name: Checkout
         id: checkout
@@ -39,8 +40,8 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Check files
+      - name: Check source files
-        id: check-files
+        id: check-source-files
         if: github.event_name != 'workflow_dispatch'
         uses: tj-actions/changed-files@v44
         with:
@@ -52,7 +53,20 @@ jobs:
             **/*.hh
             **/*.in
             **/*.patch
-            **/*.cmake
+            src/aclk/aclk-schemas/
+            src/ml/dlib/
+            src/fluent-bit/
+            src/web/server/h2o/libh2o/
+          files_ignore: |
+            netdata.spec.in
+            **/*.md
+      - name: Check build system files
+        id: check-build-files
+        if: github.event_name != 'workflow_dispatch'
+        uses: tj-actions/changed-files@v42
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
             .dockerignore
             CMakeLists.txt
             netdata-installer.sh
@@ -66,30 +80,40 @@ jobs:
             packaging/runtime-check.sh
             packaging/*.version
             packaging/*.checksums
-            src/aclk/aclk-schemas/
-            src/ml/dlib/
-            src/fluent-bit/
-            src/web/server/h2o/libh2o/
           files_ignore: |
-            netdata.spec.in
             **/*.md
       - name: List all changed files in pattern
         continue-on-error: true
         if: github.event_name != 'workflow_dispatch'
         env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
         run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
             echo "$file was changed"
           done
       - name: Check Run
         id: check-run
         run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo 'run=true' >> "${GITHUB_OUTPUT}"
           else
             echo 'run=false' >> "${GITHUB_OUTPUT}"
           fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi
 
   build-images:
     name: Build Docker Images
@@ -143,7 +167,9 @@ jobs:
           tags: netdata/netdata:test
           load: true
           cache-to: type=local,dest=/tmp/build-cache,mode=max
-          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
+          build-args: |
+            OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
+            EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }}
       - name: Test Image
         id: test
         if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64'
@@ -257,24 +283,8 @@ jobs:
         with:
           platforms: ${{ matrix.platform }}
           cache-from: type=local,src=/tmp/build-cache
-          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
           outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true
-      - name: Export Digest
+          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
-        id: export-digest
-        if: github.repository == 'netdata/netdata'
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        id: upload-digest
-        if: github.repository == 'netdata/netdata'
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-digests-${{ steps.artifact-name.outputs.platform }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:

netdata-installer.sh

@@ -211,7 +211,7 @@ USAGE: ${PROGRAM} [options]
   --disable-dbengine         Explicitly disable DB engine support.
   --enable-plugin-go         Enable the Go plugin. Default: Enabled when possible.
   --disable-plugin-go        Disable the Go plugin.
-  --disable-go               Equivalent to --disable-go-plugin
+  --disable-go               Disable all Go components.
   --enable-plugin-nfacct     Enable nfacct plugin. Default: enable it when libmnl and libnetfilter_acct are available.
   --disable-plugin-nfacct    Explicitly disable the nfacct plugin.
   --enable-plugin-xenstat    Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available.

packaging/makeself/build-static.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -54,10 +54,11 @@ fi
 # Run the build script inside the container
 if [ -t 1 ]; then
   run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
-    --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
+    --platform "${platform}" ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
-    /bin/sh /netdata/packaging/makeself/build.sh "${@}"
+    "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 else
   run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
-    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
+    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" \
-    /bin/sh /netdata/packaging/makeself/build.sh "${@}"
+    ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
+    "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 fi

packaging/makeself/build.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/bin/bash
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # -----------------------------------------------------------------------------

packaging/makeself/jobs/70-netdata-git.install.sh

@@ -37,7 +37,8 @@ run ./netdata-installer.sh \
   --dont-scrub-cflags-even-though-it-may-break-things \
   --one-time-build \
   --disable-logsmanagement \
-  --enable-lto
+  --enable-lto \
+  ${EXTRA_INSTALL_FLAGS:+${EXTRA_INSTALL_FLAGS}} \
 
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Finishing netdata install" || true

packaging/makeself/run-all-jobs.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 set -e