releaser (#4328)

.travis.yml

@@ -12,11 +12,9 @@ stages:
   - test
   - build
   - name: release
-    if: branch = master AND type = cron
-# TODO allow creating release on commit message
-#        type = master AND \    
-#        type != pull_request AND \
-#        ( type = cron OR commit_message =~ /[(major|minor|patch)]/ )
+    if: branch = master AND type != pull_request
+  - name: package
+    if: branch = master AND type != pull_request
 
 jobs:
   include:
@@ -46,6 +44,7 @@ jobs:
       script: ./coverity-scan.sh || echo "Coverity failed :("
       env: INSTALL_COVERITY="true"
       if: type = cron
+
     - stage: "build"
       name: "OSX"
       install: brew install fakeroot ossp-uuid
@@ -62,12 +61,13 @@ jobs:
       script: ./.travis/containerized_build.sh centos6
     - name: "alpine"
       script: ./.travis/containerized_build.sh alpine
+
     - stage: "release"
-      name: "Docker"
-      install: sudo apt update -y && sudo apt install -y --only-upgrade docker-ce && docker info
-      script: docker/build.sh
-      env: REPOSITORY="netdata/netdata"
-    - name: "GitHub"
+      name: "Create release"
+      script: .travis/releaser.sh
+
+    - stage: "package"
+      name: "Archive and self-extractor"
       install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot
       script: .travis/create_artifacts.sh
       deploy:
@@ -78,11 +78,16 @@ jobs:
           skip_cleanup: true
           file_glob: true
           file:
-            - "netdata*.tar.*"
+            - "netdata*.tar.gz*"
             - "netdata*.gz.run*"
+            - "sha256sums.txt"
           on:
             repo: netdata/netdata
             tags: true
+    - name: "Docker images"
+      script: docker/build.sh
+      env: REPOSITORY="netdata/netdata"
+      if: type = cron OR tag =~ ^v
 
 notifications:
   webhooks: https://app.fossa.io/hooks/travisci

.travis/README.md

@@ -1,9 +1,60 @@
-# Variables needed by travis
+# Description of CI build configuration
+
+## Variables needed by travis
 
 - GH_TOKEN - GitHub token with push access to repository
 - DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository
 - DOCKER_PASSWORD - Password to docker hub
-- encrypted_decb6f6387c4_key - Something to do with package releasing (need more info from @philwhineray)
-- encrypted_decb6f6387c4_iv - Something to do with package releasing (need more info from @philwhineray)
+- encrypted_decb6f6387c4_key - Something to do with package releasing (soon to be deprecated)
+- encrypted_decb6f6387c4_iv - Something to do with package releasing (soon to be deprecated)
 - OLD_DOCKER_USERNAME - Username used to push images to firehol/netdata # TODO: remove after deprecating that repo
 - OLD_DOCKER_PASSWORD - Password used to push images to firehol/netdata # TODO: remove after deprecating that repo
+
+## Stages
+
+### Lint
+
+Stage is executed every time and consists of 4 parallel jobs which check for any deviation from coding conventions.
+Checks are performed using:
+  - csslint (CSS)
+  - eslint (JS)
+  - shellcheck (shell scripts)
+  - flake8 (python)
+
+Ideally any error should cause whole CI pipeline to fail. Unfortunatelly right now only shellcheck and eslint are 
+causing pipeline to fail.
+
+### Test
+
+Unit tests and coverage tests are executed here. Stage consists of 2 parallel jobs:
+  - C tests - executed every time
+  - coverity test - executed only when pipeline was triggered from cron
+
+### Build
+
+Stage is executed every time and consists of 5 parallel jobs which execute containerized and non-containerized
+installations of netdata. Jobs are run on following operating systems:
+  - OSX
+  - ubuntu 14.04
+  - ubuntu 16.04 (containerized)
+  - CentOS 6 (containerized)
+  - CentOS 7 (containerized)
+
+### Release
+
+This stage is executed only on "master" brach and allows us to create a new tag just looking at git commit message.
+It also has an option to automatically generate changelog based on GitHub labels and sync it with GitHub release.
+For the sake of simplicity and to use travis features this stage cannot be integrated with next stage.
+
+### Packages
+
+As a name might suggest, this stage is responsible for creating netdata packages such as:
+  - docker images
+  - tar repository (soon to be removed)
+  - self-extracting package
+
+This stage is separated into 2 parallel jobs. One creating docker images and second one creating github artifacts.
+Whole stage is executed only on master branch, but there are also additional parameters assigned to jobs:
+  - docker images - execution only on cron job or when there is a tag assigned starting with `v`
+  - artifacts - script responsible for creating artifacts doesn't have special conditions, but deployment to GitHub
+                releases is done only when there is tag assigned

.travis/create_artifacts.sh

@@ -22,4 +22,5 @@ python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL);
 ./makeself/build-x86_64-static.sh
 for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done
 for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done
+sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
 ./.travis/deploy-if-have-key

.travis/releaser.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved
+# Permission to copy and modify is granted under the MIT license
+#
+# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
+#
+# Script to automatically do a couple of things:
+#   - generate a new tag according to semver (https://semver.org/)
+#   - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator
+#   - sync CHANGELOG with GitHub releases by using https://github.com/mattbrictson/chandler
+#
+# Tags are generated by searching for a keyword in last commit message. Keywords are:
+#  - [patch] or [fix] to bump patch number
+#  - [minor], [feature] or [feat] to bump minor number
+#  - [major] or [breaking change] to bump major number
+# All keywords MUST be surrounded with square braces.
+#
+# Script uses git mechanisms for locking, so it can be used in parallel builds
+#
+# Requirements:
+#   - GH_TOKEN variable set with GitHub token. Access level: repo.public_repo
+#   - docker
+#   - git-semver python package (pip install git-semver)
+# 
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+# Exit when latest commit is tagged
+[[ $(git tag --points-at) ]] && exit 0
+
+# Some basic variables
+GIT_MAIL="pawel+bot@netdata.cloud"
+GIT_USER="netdatabot"
+# ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
+# PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
+
+# Git config
+git config --global user.email "${GIT_MAIL}"
+git config --global user.name "${GIT_USER}"
+GIT_URL=$(git config --get remote.origin.url)
+GIT_URL=${GIT_URL#*//}
+
+# Figure out next tag based on commit message
+# OLD_TAG="$(git semver)"
+GIT_TAG=none
+echo "Last commit message: $TRAVIS_COMMIT_MESSAGE"
+case "${TRAVIS_COMMIT_MESSAGE}" in
+  *"[patch]"*|*"[fix]"* )                GIT_TAG="v$(git semver --next-patch)" ;;
+  *"[minor]"*|*"[feat]"*|*"[feature]"* ) GIT_TAG="v$(git semver --next-minor)" ;;
+  *"[major]"*|*"[breaking change]"* )    GIT_TAG="v$(git semver --next-major)" ;;
+  *) echo "Keyword not detected. Doing nothing" ;;
+esac
+
+# Tag it!
+if [ "$GIT_TAG" != "none" ]; then
+    echo "Assigning new tag: $GIT_TAG"
+    git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
+    git push "https://${GH_TOKEN}:@${GIT_URL}" --tags || exit 0
+fi
+
+## Generate CHANGELOG.md
+#git checkout master
+#git pull
+#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
+#                                                        --user "${ORGANIZATION}" \
+#                                                        --token "${GH_TOKEN}" \
+#                                                        --project "${PROJECT}" \
+#                                                        --since-tag "${OLD_TAG}" \
+#                                                        --unreleased-label "**Next release**" --no-compare-link
+#
+#git add CHANGELOG.md
+#git commit -m '[ci skip] Automatic changelog update'
+#
+#git push "https://${GH_TOKEN}:@${GIT_URL}" || exit 0
+#
+## Sync changelog to github releases
+#if [ "$GIT_TAG" != "none" ]; then
+#    docker run -e CHANDLER_GITHUB_API_TOKEN="${GH_TOKEN}" -v "$(pwd)":/chandler -ti whizark/chandler push "${GIT_TAG}"
+#fi

docker/build.sh

@@ -8,9 +8,8 @@ set -e
 
 REPOSITORY="${REPOSITORY:-netdata}"
 
-if [ ${VERSION+x} ]; then
-    VERSION="${VERSION}"
-else
+VERSION=$(git tag --points-at)
+if [ "${VERSION}" == "" ]; then
     VERSION="latest"
 fi