From 7ffe03690b02ec9961837f09f3c225172fb31fad Mon Sep 17 00:00:00 2001
From: "Austin S. Hemmelgarn" <austin@netdata.cloud>
Date: Fri, 15 Jan 2021 09:45:05 -0500
Subject: [PATCH] Switch to using GitHub Actions for publishing Docker images.
 (#10365)

* Switch to using GitHub Actions for publishing Docker images.

This simplifies handling of multiarch images and publishing to multiple
registries, and unifies testing with the code actually being used to
build and publish the images.

This also removes a handful of scripts that are no longer needed due to
this change, and switches our Dockerfile to not needing an architecture
to be specified in a build argument, instead relying on proper multiarch
docker image support.

* Fix YAML syntax.

* Add separate tagging for nightly and stable builds.

* Correct YAML syntax errors.
---
 .github/workflows/docker.yml    |  50 ++++++++------
 .travis.yml                     |  86 +++--------------------
 packaging/docker/Dockerfile     |  10 ++-
 packaging/docker/build.sh       |  80 ----------------------
 packaging/docker/check_login.sh |  41 -----------
 packaging/docker/publish.sh     | 116 --------------------------------
 6 files changed, 45 insertions(+), 338 deletions(-)
 delete mode 100755 packaging/docker/build.sh
 delete mode 100755 packaging/docker/check_login.sh
 delete mode 100755 packaging/docker/publish.sh

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 447213554a..8f0795d1c0 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -13,37 +13,47 @@ on:
       - '.github/workflows/docker.yml'
       - 'netdata-installer.sh'
       - 'packaging/**'
+  workflow_dispatch:
+    inputs:
+      version:
+        name: Version Tag
+        default: latest
+        required: true
 jobs:
   docker-build:
     name: Docker Build
-    strategy:
-      matrix:
-        arch:
-          - linux/amd64
-          - linux/i386
-          - linux/arm/v7
-          - linux/arm64
-        include:
-          - arch: linux/amd64
-            base: amd64
-          - arch: linux/i386
-            base: i386
-          - arch: linux/arm/v7
-            base: armhf
-          - arch: linux/arm64
-            base: aarch64
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Determine if we should push changes and which tags to use
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly'
+        run: |
+          echo "publish=true" >> $GITHUB_ENV
+          echo "tags=netdata/netdata:latest,netdata/netdata:stable,netdata/netdata:${{ github.event.inputs.version }}" >> $GITHUB_ENV
+      - name: Determine if we should push changes and which tags to use
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly'
+        run: |
+          echo "publish=true" >> $GITHUB_ENV
+          echo "tags=netdata/netdata:latest,netdata/netdata:edge" >> $GITHUB_ENV
+      - name: Determine if we should push changes and which tags to use
+        if: github.event_name != 'workflow_dispatch'
+        run: |
+          echo "publish=false" >> $GITHUB_ENV
+          echo "tags=netdata/netdata:test" >> $GITHUB_ENV
       - name: Setup QEMU
         uses: docker/setup-qemu-action@v1
       - name: Setup Buildx
         uses: docker/setup-buildx-action@v1
+      - name: Docker Hub Login
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: Docker Build
         uses: docker/build-push-action@v2
         with:
-          platforms: ${{ matrix.arch }}
-          push: false
-          build-args: |
-            ARCH=${{ matrix.base }}
+          platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64
+          push: ${{ env.publish }}
+          tags: ${{ env.tags }}
diff --git a/.travis.yml b/.travis.yml
index 677150239c..cb0aa85628 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -103,14 +103,6 @@ jobs:
       env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common   -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1'
       after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)"
 
-    - name: Docker container build process (alpine installation)
-      script:
-        - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker"
-        - packaging/docker/build.sh
-      env:
-        - ARCH=amd64
-      after_failure: post_message "TRAVIS_MESSAGE" "Docker build process failed"
-
     - name: Build/Install for ubuntu 20.04 (not containerized)
       script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME
       after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04"
@@ -353,48 +345,7 @@ jobs:
 
     # We only publish if a TAG has been set during packaging
     - stage: Publish for release
-      _template: &RELEASE_TEMPLATE
-        env:
-          - RELEASE_CHANNEL: stable
-        git:
-          depth: false
-        script:
-          - echo "GIT Branch:" && git branch
-          - echo "Last commit:" && git log -1
-          - echo "GIT Describe:" && git describe
-          - echo "packaging/version:" && cat packaging/version
-          - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker"
-          - packaging/docker/check_login.sh
-            && echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull
-            && tick packaging/docker/build.sh
-            && packaging/docker/publish.sh
-        after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed"
-
-      name: Build & Publish docker image for i386
-      <<: *RELEASE_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=i386
-
-    - name: Build & Publish docker image for amd64
-      <<: *RELEASE_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=amd64
-
-    - name: Build & Publish docker image for armhf
-      <<: *RELEASE_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=armhf
-
-    - name: Build & Publish docker image for aarch64
-      <<: *RELEASE_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=aarch64
-
-    - name: Create release draft
+      name: Create release draft
       git:
         depth: false
       env:
@@ -409,6 +360,10 @@ jobs:
         - .travis/draft_release.sh
       after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed"
 
+    - name: Trigger Docker image build and publish
+      script: >-
+        curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "${build_version}"}}'
+      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}"
 
     - stage: Trigger deb and rpm package build (release)
       name: Trigger deb and rpm package build
@@ -452,31 +407,7 @@ jobs:
             && packaging/docker/publish.sh
         after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed"
 
-      name: Build & Publish docker image for i386
-      <<: *NIGHTLY_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=i386
-
-    - name: Build & Publish docker image for amd64
-      <<: *NIGHTLY_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=amd64
-
-    - name: Build & Publish docker image for armhf
-      <<: *NIGHTLY_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=armhf
-
-    - name: Build & Publish docker image for aarch64
-      <<: *NIGHTLY_TEMPLATE
-      env:
-        - ALLOW_SOFT_FAILURE_HERE=true
-        - ARCH=aarch64
-
-    - name: Create nightly release artifacts, publish to GCS
+      name: Create nightly release artifacts, publish to GCS
       script:
         - echo "GIT Branch:" && git branch
         - echo "Last commit:" && git log -1
@@ -526,6 +457,11 @@ jobs:
             condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata"
       after_deploy: rm -f .travis/gcs-credentials.json
 
+    - name: Trigger Docker image build and publish
+      script: >-
+        curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "nightly"}}'
+      after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}"
+
     - stage: Trigger deb and rpm package build (nightly release)
       name: Trigger deb and rpm package build
       script: .travis/trigger_package_generation.sh "[Build latest]"
diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile
index 65171d55eb..41e8870012 100644
--- a/packaging/docker/Dockerfile
+++ b/packaging/docker/Dockerfile
@@ -1,11 +1,9 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # author  : paulfantom
 
-# Cross-arch building is achieved by specifying ARCH as a build parameter with `--build-arg` option.
-# It is automated in `build.sh` script
-ARG ARCH=amd64
 # This image contains preinstalled dependecies
-FROM netdata/builder:${ARCH} as builder
+# hadolint ignore=DL3007
+FROM netdata/builder:latest as builder
 
 # One of 'nightly' or 'stable'
 ARG RELEASE_CHANNEL=nightly
@@ -54,9 +52,9 @@ RUN mkdir -p /app/usr/sbin/ \
     chmod +x /app/usr/sbin/run.sh
 
 #####################################################################
-ARG ARCH
 # This image contains preinstalled dependecies
-FROM netdata/base:${ARCH}
+# hadolint ignore=DL3007
+FROM netdata/base:latest as base
 
 # Configure system
 ARG NETDATA_UID=201
diff --git a/packaging/docker/build.sh b/packaging/docker/build.sh
deleted file mode 100755
index 4522936e8a..0000000000
--- a/packaging/docker/build.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
-#
-# Author  : Pawel Krupa (paulfantom)
-# Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
-
-set -e
-
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
-	echo "This mechanism currently can only run on BASH version 4 and above"
-	exit 1
-fi
-
-VERSION="$1"
-
-if [ -z "${ARCH}" ]; then
-  echo "ARCH not set, build cannot proceed"
-  exit 1
-fi
-
-if [ "${RELEASE_CHANNEL}" != "nightly" ] && [ "${RELEASE_CHANNEL}" != "stable" ]; then
-  echo "RELEASE_CHANNEL must be set to either 'nightly' or 'stable' - build cannot proceed"
-  exit 1
-fi
-
-if [ -z ${REPOSITORY} ]; then
-	REPOSITORY="${TRAVIS_REPO_SLUG}"
-	if [ -z ${REPOSITORY} ]; then
-		echo "REPOSITORY not set, build cannot proceed"
-		exit 1
-	else
-		echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}"
-	fi
-fi
-
-# Ensure there is a version, the most appropriate one
-if [ "${VERSION}" == "" ]; then
-    VERSION=$(git tag --points-at)
-    if [ "${VERSION}" == "" ]; then
-        VERSION="latest"
-    fi
-fi
-
-# If we are not in netdata git repo, at the top level directory, fail
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
-CWD=$(git rev-parse --show-cdup)
-if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
-    echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
-    echo "Docker build process aborted"
-    exit 1
-fi
-
-case "${ARCH}" in
-  amd64) DOCKER_PLATFORM="linux/amd64" ;;
-  i386) DOCKER_PLATFORM="linux/i386" ;;
-  armhf) DOCKER_PLATFORM="linux/arm/v7" ;;
-  aarch64) DOCKER_PLATFORM="linux/arm64" ;;
-esac
-
-echo "Docker image build in progress.."
-echo "Version     : ${VERSION}"
-echo "Repository  : ${REPOSITORY}"
-echo "Architecture: ${ARCH}"
-
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-# Build images using multi-arch Dockerfile.
-TAG="${REPOSITORY,,}:${VERSION}-${ARCH}"
-echo "Building tag ${TAG}.."
-docker build --no-cache                                  \
-	--build-arg ARCH="${ARCH}"                       \
-	--build-arg RELEASE_CHANNEL="${RELEASE_CHANNEL}" \
-	--platform "${DOCKER_PLATFORM}"                  \
-	--tag "${TAG}"                                   \
-	--file packaging/docker/Dockerfile .
-echo "..Done!"
-
-echo "Docker build process completed!"
diff --git a/packaging/docker/check_login.sh b/packaging/docker/check_login.sh
deleted file mode 100755
index 7cc8d4e50d..0000000000
--- a/packaging/docker/check_login.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-#
-# This is a credential checker script, to help get early input on docker credentials status
-# If these are wrong, then build/publish has no point running
-#
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
-#
-# Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
-
-set -e
-
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
-	echo "This mechanism currently can only run on BASH version 4 and above"
-	exit 1
-fi
-
-DOCKER_CMD="docker "
-
-# There is no reason to continue if we cannot log in to docker hub
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then
-    echo "No docker hub username or password found, aborting without publishing"
-    exit 1
-fi
-
-# If we are not in netdata git repo, at the top level directory, fail
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
-CWD=$(git rev-parse --show-cdup)
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
-    echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
-    echo "Docker build process aborted"
-    exit 1
-fi
-
-# Login to docker hub to allow futher operations
-echo "Attempting to login to docker"
-echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin
-
-echo "Docker login successful!"
-$DOCKER_CMD logout
-
-echo "Docker login validation completed"
diff --git a/packaging/docker/publish.sh b/packaging/docker/publish.sh
deleted file mode 100755
index 172f8df45f..0000000000
--- a/packaging/docker/publish.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-#
-# Cross-arch docker publish helper script
-# Needs docker in version >18.02 due to usage of manifests
-#
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
-#
-# Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
-
-set -e
-
-if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then
-	echo "This mechanism currently can only run on BASH version 4 and above"
-	exit 1
-fi
-
-WORKDIR="$(mktemp -d)" # Temporary folder, removed after script is done
-VERSION="$1"
-
-if [ -z "${ARCH}" ]; then
-  echo "ARCH not set, build cannot proceed"
-  exit 1
-fi
-
-DOCKER_CMD="docker --config ${WORKDIR}"
-GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
-GIT_USER=${GIT_USER:-"netdatabot"}
-
-if [ -z ${REPOSITORY} ]; then
-	REPOSITORY="${TRAVIS_REPO_SLUG}"
-	if [ -z ${REPOSITORY} ]; then
-		echo "REPOSITORY not set, publish cannot proceed"
-		exit 1
-	else
-		echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}"
-	fi
-fi
-
-# Ensure there is a version, the most appropriate one
-if [ "${VERSION}" == "" ]; then
-    VERSION=$(git tag --points-at)
-    if [ "${VERSION}" == "" ]; then
-        VERSION="latest"
-    fi
-fi
-MANIFEST_LIST="${REPOSITORY}:${VERSION}"
-
-# There is no reason to continue if we cannot log in to docker hub
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then
-    echo "No docker hub username or password found, aborting without publishing"
-    exit 1
-fi
-
-# If we are not in netdata git repo, at the top level directory, fail
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
-CWD=$(git rev-parse --show-cdup)
-if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
-    echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository"
-    echo "Docker build process aborted"
-    exit 1
-fi
-
-echo "Docker image publishing in progress.."
-echo "Version      : ${VERSION}"
-echo "Repository   : ${REPOSITORY}"
-echo "Architecture : ${ARCH}"
-echo "Manifest list: ${MANIFEST_LIST}"
-
-# Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
-echo '{"experimental":"enabled"}' > "${WORKDIR}"/config.json
-
-# Login to docker hub to allow futher operations
-echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin
-
-# Push images to registry
-TAG="${MANIFEST_LIST}-${ARCH}"
-echo "Publishing image ${TAG}.."
-$DOCKER_CMD push "${TAG}"
-
-published() {
-	curl -s "https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags" | jq -e -r '.results[] | select(.name == "'"${VERSION}-${ARCH}"'")' > /dev/null
-}
-retry 5 published
-
-echo "Image ${TAG} published succesfully!"
-
-# Recreate docker manifest list
-echo "Getting tag list for version '${VERSION}'.."
-TAGS=($(curl -s https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags/ | jq -r '.results[]["name"]' | grep "^${VERSION}-"))
-
-echo "Creating manifest list.."
-$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" "${TAGS[@]/#/${REPOSITORY}:}"
-
-# Annotate manifest with CPU architecture information
-declare -A ARCH_MAP
-ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
-
-echo "Executing manifest annotate.."
-for TAG in "${TAGS[@]}"; do
-     ARCH="${TAG#${VERSION}-}"
-     echo "Annotating manifest for $ARCH, with TAG: ${REPOSITORY}:${TAG} (Manifest list: ${MANIFEST_LIST})"
-     $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${REPOSITORY}:${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}"
-done
-
-# Push manifest to docker hub
-echo "Pushing manifest list to docker.."
-$DOCKER_CMD manifest push -p "${MANIFEST_LIST}"
-
-# Show current manifest (debugging purpose only)
-echo "Evaluating manifest list entry"
-$DOCKER_CMD manifest inspect "${MANIFEST_LIST}"
-
-# Cleanup
-rm -r "${WORKDIR}"
-
-echo "Docker publishing process completed!"