diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 447213554a..8f0795d1c0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,37 +13,47 @@ on: - '.github/workflows/docker.yml' - 'netdata-installer.sh' - 'packaging/**' + workflow_dispatch: + inputs: + version: + name: Version Tag + default: latest + required: true jobs: docker-build: name: Docker Build - strategy: - matrix: - arch: - - linux/amd64 - - linux/i386 - - linux/arm/v7 - - linux/arm64 - include: - - arch: linux/amd64 - base: amd64 - - arch: linux/i386 - base: i386 - - arch: linux/arm/v7 - base: armhf - - arch: linux/arm64 - base: aarch64 runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 + - name: Determine if we should push changes and which tags to use + if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' + run: | + echo "publish=true" >> $GITHUB_ENV + echo "tags=netdata/netdata:latest,netdata/netdata:stable,netdata/netdata:${{ github.event.inputs.version }}" >> $GITHUB_ENV + - name: Determine if we should push changes and which tags to use + if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly' + run: | + echo "publish=true" >> $GITHUB_ENV + echo "tags=netdata/netdata:latest,netdata/netdata:edge" >> $GITHUB_ENV + - name: Determine if we should push changes and which tags to use + if: github.event_name != 'workflow_dispatch' + run: | + echo "publish=false" >> $GITHUB_ENV + echo "tags=netdata/netdata:test" >> $GITHUB_ENV - name: Setup QEMU uses: docker/setup-qemu-action@v1 - name: Setup Buildx uses: docker/setup-buildx-action@v1 + - name: Docker Hub Login + if: github.event_name == 'workflow_dispatch' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Docker Build uses: docker/build-push-action@v2 with: - platforms: ${{ matrix.arch }} - push: false - build-args: | - ARCH=${{ matrix.base }} + platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64 + push: ${{ env.publish }} + tags: ${{ env.tags }} diff --git a/.travis.yml b/.travis.yml index 677150239c..cb0aa85628 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,14 +103,6 @@ jobs: env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1' after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" - - name: Docker container build process (alpine installation) - script: - - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker" - - packaging/docker/build.sh - env: - - ARCH=amd64 - after_failure: post_message "TRAVIS_MESSAGE" "Docker build process failed" - - name: Build/Install for ubuntu 20.04 (not containerized) script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04" @@ -353,48 +345,7 @@ jobs: # We only publish if a TAG has been set during packaging - stage: Publish for release - _template: &RELEASE_TEMPLATE - env: - - RELEASE_CHANNEL: stable - git: - depth: false - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker" - - packaging/docker/check_login.sh - && echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull - && tick packaging/docker/build.sh - && packaging/docker/publish.sh - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed" - - name: Build & Publish docker image for i386 - <<: *RELEASE_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=i386 - - - name: Build & Publish docker image for amd64 - <<: *RELEASE_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=amd64 - - - name: Build & Publish docker image for armhf - <<: *RELEASE_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=armhf - - - name: Build & Publish docker image for aarch64 - <<: *RELEASE_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=aarch64 - - - name: Create release draft + name: Create release draft git: depth: false env: @@ -409,6 +360,10 @@ jobs: - .travis/draft_release.sh after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed" + - name: Trigger Docker image build and publish + script: >- + curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "${build_version}"}}' + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}" - stage: Trigger deb and rpm package build (release) name: Trigger deb and rpm package build @@ -452,31 +407,7 @@ jobs: && packaging/docker/publish.sh after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed" - name: Build & Publish docker image for i386 - <<: *NIGHTLY_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=i386 - - - name: Build & Publish docker image for amd64 - <<: *NIGHTLY_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=amd64 - - - name: Build & Publish docker image for armhf - <<: *NIGHTLY_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=armhf - - - name: Build & Publish docker image for aarch64 - <<: *NIGHTLY_TEMPLATE - env: - - ALLOW_SOFT_FAILURE_HERE=true - - ARCH=aarch64 - - - name: Create nightly release artifacts, publish to GCS + name: Create nightly release artifacts, publish to GCS script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 @@ -526,6 +457,11 @@ jobs: condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata" after_deploy: rm -f .travis/gcs-credentials.json + - name: Trigger Docker image build and publish + script: >- + curl -X POST -H 'Accept: application/vnd.github.v3+json' 'https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml' -d '{"ref": "master", "inputs": {"version": "nightly"}}' + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}" + - stage: Trigger deb and rpm package build (nightly release) name: Trigger deb and rpm package build script: .travis/trigger_package_generation.sh "[Build latest]" diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 65171d55eb..41e8870012 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -1,11 +1,9 @@ # SPDX-License-Identifier: GPL-3.0-or-later # author : paulfantom -# Cross-arch building is achieved by specifying ARCH as a build parameter with `--build-arg` option. -# It is automated in `build.sh` script -ARG ARCH=amd64 # This image contains preinstalled dependecies -FROM netdata/builder:${ARCH} as builder +# hadolint ignore=DL3007 +FROM netdata/builder:latest as builder # One of 'nightly' or 'stable' ARG RELEASE_CHANNEL=nightly @@ -54,9 +52,9 @@ RUN mkdir -p /app/usr/sbin/ \ chmod +x /app/usr/sbin/run.sh ##################################################################### -ARG ARCH # This image contains preinstalled dependecies -FROM netdata/base:${ARCH} +# hadolint ignore=DL3007 +FROM netdata/base:latest as base # Configure system ARG NETDATA_UID=201 diff --git a/packaging/docker/build.sh b/packaging/docker/build.sh deleted file mode 100755 index 4522936e8a..0000000000 --- a/packaging/docker/build.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pawel Krupa (paulfantom) -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) - -set -e - -if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then - echo "This mechanism currently can only run on BASH version 4 and above" - exit 1 -fi - -VERSION="$1" - -if [ -z "${ARCH}" ]; then - echo "ARCH not set, build cannot proceed" - exit 1 -fi - -if [ "${RELEASE_CHANNEL}" != "nightly" ] && [ "${RELEASE_CHANNEL}" != "stable" ]; then - echo "RELEASE_CHANNEL must be set to either 'nightly' or 'stable' - build cannot proceed" - exit 1 -fi - -if [ -z ${REPOSITORY} ]; then - REPOSITORY="${TRAVIS_REPO_SLUG}" - if [ -z ${REPOSITORY} ]; then - echo "REPOSITORY not set, build cannot proceed" - exit 1 - else - echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}" - fi -fi - -# Ensure there is a version, the most appropriate one -if [ "${VERSION}" == "" ]; then - VERSION=$(git tag --points-at) - if [ "${VERSION}" == "" ]; then - VERSION="latest" - fi -fi - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository" - echo "Docker build process aborted" - exit 1 -fi - -case "${ARCH}" in - amd64) DOCKER_PLATFORM="linux/amd64" ;; - i386) DOCKER_PLATFORM="linux/i386" ;; - armhf) DOCKER_PLATFORM="linux/arm/v7" ;; - aarch64) DOCKER_PLATFORM="linux/arm64" ;; -esac - -echo "Docker image build in progress.." -echo "Version : ${VERSION}" -echo "Repository : ${REPOSITORY}" -echo "Architecture: ${ARCH}" - -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - -# Build images using multi-arch Dockerfile. -TAG="${REPOSITORY,,}:${VERSION}-${ARCH}" -echo "Building tag ${TAG}.." -docker build --no-cache \ - --build-arg ARCH="${ARCH}" \ - --build-arg RELEASE_CHANNEL="${RELEASE_CHANNEL}" \ - --platform "${DOCKER_PLATFORM}" \ - --tag "${TAG}" \ - --file packaging/docker/Dockerfile . -echo "..Done!" - -echo "Docker build process completed!" diff --git a/packaging/docker/check_login.sh b/packaging/docker/check_login.sh deleted file mode 100755 index 7cc8d4e50d..0000000000 --- a/packaging/docker/check_login.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# This is a credential checker script, to help get early input on docker credentials status -# If these are wrong, then build/publish has no point running -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) - -set -e - -if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then - echo "This mechanism currently can only run on BASH version 4 and above" - exit 1 -fi - -DOCKER_CMD="docker " - -# There is no reason to continue if we cannot log in to docker hub -if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then - echo "No docker hub username or password found, aborting without publishing" - exit 1 -fi - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository" - echo "Docker build process aborted" - exit 1 -fi - -# Login to docker hub to allow futher operations -echo "Attempting to login to docker" -echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin - -echo "Docker login successful!" -$DOCKER_CMD logout - -echo "Docker login validation completed" diff --git a/packaging/docker/publish.sh b/packaging/docker/publish.sh deleted file mode 100755 index 172f8df45f..0000000000 --- a/packaging/docker/publish.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# -# Cross-arch docker publish helper script -# Needs docker in version >18.02 due to usage of manifests -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) - -set -e - -if [ "${BASH_VERSINFO[0]}" -lt "4" ]; then - echo "This mechanism currently can only run on BASH version 4 and above" - exit 1 -fi - -WORKDIR="$(mktemp -d)" # Temporary folder, removed after script is done -VERSION="$1" - -if [ -z "${ARCH}" ]; then - echo "ARCH not set, build cannot proceed" - exit 1 -fi - -DOCKER_CMD="docker --config ${WORKDIR}" -GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"} -GIT_USER=${GIT_USER:-"netdatabot"} - -if [ -z ${REPOSITORY} ]; then - REPOSITORY="${TRAVIS_REPO_SLUG}" - if [ -z ${REPOSITORY} ]; then - echo "REPOSITORY not set, publish cannot proceed" - exit 1 - else - echo "REPOSITORY was not detected, attempted to use TRAVIS_REPO_SLUG setting: ${TRAVIS_REPO_SLUG}" - fi -fi - -# Ensure there is a version, the most appropriate one -if [ "${VERSION}" == "" ]; then - VERSION=$(git tag --points-at) - if [ "${VERSION}" == "" ]; then - VERSION="latest" - fi -fi -MANIFEST_LIST="${REPOSITORY}:${VERSION}" - -# There is no reason to continue if we cannot log in to docker hub -if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PWD+x} ]; then - echo "No docker hub username or password found, aborting without publishing" - exit 1 -fi - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ ! -z $CWD ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as ./packaging/docker/$(basename "$0") from top level directory of netdata git repository" - echo "Docker build process aborted" - exit 1 -fi - -echo "Docker image publishing in progress.." -echo "Version : ${VERSION}" -echo "Repository : ${REPOSITORY}" -echo "Architecture : ${ARCH}" -echo "Manifest list: ${MANIFEST_LIST}" - -# Create temporary docker CLI config with experimental features enabled (manifests v2 need it) -echo '{"experimental":"enabled"}' > "${WORKDIR}"/config.json - -# Login to docker hub to allow futher operations -echo "$DOCKER_PWD" | $DOCKER_CMD login -u "$DOCKER_USERNAME" --password-stdin - -# Push images to registry -TAG="${MANIFEST_LIST}-${ARCH}" -echo "Publishing image ${TAG}.." -$DOCKER_CMD push "${TAG}" - -published() { - curl -s "https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags" | jq -e -r '.results[] | select(.name == "'"${VERSION}-${ARCH}"'")' > /dev/null -} -retry 5 published - -echo "Image ${TAG} published succesfully!" - -# Recreate docker manifest list -echo "Getting tag list for version '${VERSION}'.." -TAGS=($(curl -s https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags/ | jq -r '.results[]["name"]' | grep "^${VERSION}-")) - -echo "Creating manifest list.." -$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" "${TAGS[@]/#/${REPOSITORY}:}" - -# Annotate manifest with CPU architecture information -declare -A ARCH_MAP -ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64") - -echo "Executing manifest annotate.." -for TAG in "${TAGS[@]}"; do - ARCH="${TAG#${VERSION}-}" - echo "Annotating manifest for $ARCH, with TAG: ${REPOSITORY}:${TAG} (Manifest list: ${MANIFEST_LIST})" - $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${REPOSITORY}:${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}" -done - -# Push manifest to docker hub -echo "Pushing manifest list to docker.." -$DOCKER_CMD manifest push -p "${MANIFEST_LIST}" - -# Show current manifest (debugging purpose only) -echo "Evaluating manifest list entry" -$DOCKER_CMD manifest inspect "${MANIFEST_LIST}" - -# Cleanup -rm -r "${WORKDIR}" - -echo "Docker publishing process completed!"