Add code for repository configuration packages. (#11273)

* Add code for DEB repository config packages.

* Add code for RPM repository config packages.

And sync up Debian related code for consistency.

* Add workflow to build and upload repoconfig packages.

* Added repoconfig repository to repoconfig packages.

This ensures we have a singe source of truth for them, and will also
make switching between stable and nightly builds much easier.

* Use base images instead of dedicated package builders.

* Don0t prompt during DEB package builds.

* Fix typos.

.github/workflows/repoconfig-packages.yml

@@ -0,0 +1,83 @@
+---
+# Handles building of binary packages for the agent.
+name: Repository Packages
+on:
+  workflow_dispatch:
+    inputs:
+      distros:
+        name: Distros to Build
+        default: all
+        required: true
+env:
+  DO_NOT_TRACK: 1
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    env:
+      DO_NOT_TRACK: 1
+      DOCKER_CLI_EXPERIMENTAL: enabled
+    strategy:
+      # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines.
+      matrix:
+        include:
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
+          - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
+      # We intentiaonally disable the fail-fast behavior so that a
+      # build failure for one version doesn't prevent us from publishing
+      # successfully built and tested packages for another version.
+      fail-fast: false
+      max-parallel: 8
+    steps:
+      - name: Checkout
+        if: contains(github.event.input.distros, join([matrix.distro, matrix.version], '-')) || github.event.input.distros == 'all'
+        uses: actions/checkout@v2
+        # Unlike normally, we do not need a deep clone or submodules for this.
+      - name: Build Packages
+        if: contains(github.event.input.distros, join([matrix.distro, matrix.version], '-')) || github.event.input.distros == 'all'
+        shell: bash
+        run: |
+          docker run -e DO_NOT_TRACK=1 --platform ${{ matrix.platform }} -v $PWD:/netdata ${{ matrix.base_image }}:${{ matrix.version }} /netdata/packaging/repoconfig/build-${{ matrix.format }}.sh
+      - name: Upload Packages
+        if: contains(github.event.input.distros, join([matrix.distro, matrix.version], '-')) || github.event.input.distros == 'all'
+        shell: bash
+        env:
+          PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
+        run: |
+          echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})"
+          for pkgfile in artifacts/*.${{ matrix.format }} ; do
+            .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} ${pkgfile} || true
+            .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} ${pkgfile}
+            .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} ${pkgfile} || true
+            .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} ${pkgfile}
+            .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} ${pkgfile} || true
+            .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} ${pkgfile}
+          done
+      - name: Failure Notification
+        if: contains(github.event.input.distros, join([matrix.distro, matrix.version], '-')) || github.event.input.distros == 'all'
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER:
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Repository Package Build failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} repository package build failed."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && github.event_name != 'pull_request'
+            && startsWith(github.ref, 'refs/heads/master')
+          }}

.gitignore

@@ -217,3 +217,6 @@ externaldeps/
 # vim sessions
 Session.vim
 Session.*.vim
+
+# Special exceptions
+!packaging/repoconfig/Makefile

packaging/repoconfig/Makefile

@@ -0,0 +1,31 @@
+FILES = netdata.list netdata-edge.list netdata-archive-keyring.gpg netdata-edge-archive-keyring.gpg netdata-repoconfig-archive-keyring.gpg
+
+all: $(FILES)
+
+netdata.list: netdata.list.in
+	cp netdata.list.in netdata.list
+	set -a && . /etc/os-release && sed -i -e "s/__DISTRO__/$${ID}/" -e "s/__SUITE__/$${VERSION_CODENAME}/" -e "s/__VARIANT__//" netdata.list
+
+netdata-edge.list: netdata.list.in
+	cp netdata.list.in netdata-edge.list
+	set -a && . /etc/os-release && sed -i -e "s/__DISTRO__/$${ID}/" -e "s/__SUITE__/$${VERSION_CODENAME}/" -e "s/__VARIANT__/-edge/" netdata-edge.list
+
+netdata-archive-keyring.gpg:
+	curl -L https://packagecloud.io/netdata/netdata/gpgkey | gpg --dearmor > netdata-archive-keyring.gpg
+
+netdata-edge-archive-keyring.gpg:
+	curl -L https://packagecloud.io/netdata/netdata-edge/gpgkey | gpg --dearmor > netdata-edge-archive-keyring.gpg
+
+netdata-repoconfig-archive-keyring.gpg:
+	curl -L https://packagecloud.io/netdata/netdata-repoconfig/gpgkey | gpg --dearmor > netdata-repoconfig-archive-keyring.gpg
+
+debian/tmp:
+	mkdir -p debian/tmp
+
+install: $(FILES) debian/tmp
+	cp $(FILES) debian/tmp/
+
+clean:
+	rm -f $(FILES)
+
+.PHONY: clean

packaging/repoconfig/build-deb.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+# Needed because dpkg is stupid and tries to configure things interactively if it sees a terminal.
+export DEBIAN_FRONTEND=noninteractive
+
+# Pull in our dependencies
+apt update || exit 1
+apt upgrade -y || exit 1
+apt install -y build-essential debhelper curl gnupg || exit 1
+
+# Run the builds in an isolated source directory.
+# This removes the need for cleanup, and ensures anything the build does
+# doesn't muck with the user's sources.
+cp -a /netdata/packaging/repoconfig /usr/src || exit 1
+cd /usr/src/repoconfig || exit 1
+
+# pre/post options are after 1.18.8, is simpler to just check help for their existence than parsing version
+if dpkg-buildpackage --help | grep "\-\-post\-clean" 2> /dev/null > /dev/null; then
+  dpkg-buildpackage --post-clean --pre-clean -b -us -uc || exit 1
+else
+  dpkg-buildpackage -b -us -uc || exit 1
+fi
+
+# Copy the built packages to /netdata/artifacts (which may be bind-mounted)
+# Also ensure /netdata/artifacts exists and create it if it doesn't
+[ -d /netdata/artifacts ] || mkdir -p /netdata/artifacts
+cp -a /usr/src/*.deb /netdata/artifacts/ || exit 1
+
+# Correct ownership of the artifacts.
+# Without this, the artifacts directory and it's contents end up owned
+# by root instead of the local user on Linux boxes
+chown -R --reference=/netdata /netdata/artifacts

packaging/repoconfig/build-rpm.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+prefix='/root/rpmbuild'
+
+if command -v dnf > /dev/null ; then
+    dnf distro-sync -y --nodocs || exit 1
+    dnf install -y --nodocs --setopt=install_weak_deps=False rpm-build || exit 1
+elif command -v yum > /dev/null ; then
+    yum distro-sync -y || exit 1
+    yum install -y rpm-build || exit 1
+elif command -v zypper > /dev/null ; then
+    zypper update -y || exit 1
+    zypper install -y rpm-build || exit 1
+    prefix="/usr/src/packages"
+fi
+
+mkdir -p "${prefix}/BUILD" "${prefix}/RPMS" "${prefix}/SRPMS" "${prefix}/SPECS" "${prefix}/SOURCES" || exit 1
+cp -a /netdata/packaging/repoconfig/netdata-repo.spec "${prefix}/SPECS" || exit 1
+cp -a /netdata/packaging/repoconfig/* "${prefix}/SOURCES/" || exit 1
+
+rpmbuild -bb --rebuild "${prefix}/SPECS/netdata-repo.spec" || exit 1
+
+[ -d /netdata/artifacts ] || mkdir -p /netdata/artifacts
+find "${prefix}/RPMS/" -type f -name '*.rpm' -exec cp '{}' /netdata/artifacts \; || exit 1
+
+chown -R --reference=/netdata /netdata/artifacts

packaging/repoconfig/debian/changelog

@@ -0,0 +1,6 @@
+netdata-repo (1-1) unstable; urgency=medium
+
+  * Initial Release
+
+ -- Netdata Builder <bot@netdata.cloud>  Mon, 14 Jun 2021 08:00:00 -0500
+

packaging/repoconfig/debian/compat

@@ -0,0 +1 @@
+10

packaging/repoconfig/debian/control

@@ -0,0 +1,19 @@
+Source: netdata-repo
+Section: net
+Priority: optional
+Maintainer: Netdata Builder <bot@netdata.cloud>
+Standards-Version: 3.9.6
+Build-Depends: debhelper (>= 10), curl, gnupg
+Homepage: https://netdata.cloud
+
+Package: netdata-repo
+Architecture: all
+Depends: apt-transport-https, debian-archive-keyring, gnupg
+Conflicts: netdata-repo-edge
+Description: Configuration for the official Netdata Stable package repository.
+
+Package: netdata-repo-edge
+Architecture:all
+Depends: apt-transport-https, debian-archive-keyring, gnupg
+Conflicts: netdata-repo
+Description: Configuration for the official Netdata Edge package repository.

packaging/repoconfig/debian/copyright

@@ -0,0 +1,10 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: Netdata
+Upstream-Contact: Costa Tsaousis <costa@netdata.cloud>
+Source: https://github.com/netdata/netdata
+
+Files: *
+Copyright: 2021 Netdata Inc.
+License: GPL-3+
+ On Debian systems, the complete text of the GNU General Public
+ License version 3 can be found in /usr/share/common-licenses/GPL-3.

packaging/repoconfig/debian/rules

@@ -0,0 +1,21 @@
+#!/usr/bin/make -f
+
+TOP = $(CURDIR)/debian/netdata-repo
+TOP_EDGE = $(CURDIR)/debian/netdata-edge-repo
+TEMPTOP = $(CURDIR)/debian/tmp
+
+%:
+	dh $@
+
+override_dh_configure:
+	true
+
+override_dh_install:
+	mkdir -p $(TOP)/etc/apt/sources.list.d $(TOP)/etc/apt/trusted.gpg.d/
+	mv -f $(TEMPTOP)/netdata.list $(TOP)/etc/apt/sources.list.d
+	mv -f $(TEMPTOP)/netdata-archive-keyring.gpg $(TOP)/etc/apt/trusted.gpg.d
+	cp $(TEMPTOP)/netdata-repoconfig-archive-keyring.gpg $(TOP)/etc/apt/trusted.gpg.d
+	mkdir -p $(TOP_EDGE)/etc/apt/sources.list.d $(TOP_EDGE)/etc/apt/trusted.gpg.d/
+	mv -f $(TEMPTOP)/netdata-edge.list $(TOP_EDGE)/etc/apt/sources.list.d
+	mv -f $(TEMPTOP)/netdata-edge-archive-keyring.gpg $(TOP_EDGE)/etc/apt/trusted.gpg.d
+	cp $(TEMPTOP)/netdata-repoconfig-archive-keyring.gpg $(TOP_EDGE)/etc/apt/trusted.gpg.d

packaging/repoconfig/debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

packaging/repoconfig/netdata-edge.repo.centos

@@ -0,0 +1,19 @@
+[netdata-edge]
+name=Netdata Edge
+baseurl=https://packagecloud.io/netdata/netdata-edge/el/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-edge/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+
+[netdata-repoconfig]
+name=Netdata Repository Config
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/el/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt

packaging/repoconfig/netdata-edge.repo.fedora

@@ -0,0 +1,19 @@
+[netdata-edge]
+name=Netdata Edge
+baseurl=https://packagecloud.io/netdata/netdata-edge/fedora/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-edge/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+
+[netdata-repoconfig]
+name=Netdata Repository Config
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/fedora/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt

packaging/repoconfig/netdata-edge.repo.suse

@@ -0,0 +1,19 @@
+[netdata-edge]
+name=Netdata Edge
+baseurl=https://packagecloud.io/netdata/netdata-edge/opensuse/$releasever/$basearch
+repo_gpgcheck=1
+pkg_gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-edge/gpgkey
+enabled=1
+type=rpm-md
+autorefresh=1
+
+[netdata-repoconfig]
+name=Netdata Repoconfig
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/opensuse/$releasever/$basearch
+repo_gpgcheck=1
+pkg_gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+type=rpm-md
+autorefresh=1

packaging/repoconfig/netdata-repo.spec

@@ -0,0 +1,89 @@
+%{?rhel:%global centos_ver %rhel}
+
+Name:           netdata-repo
+Version:        1
+Release:        1
+Summary:        Netdata stable repositories configuration.
+
+Group:          System Environment/Base
+License:        GPLv2
+
+Source0:        netdata.repo.fedora
+Source1:        netdata-edge.repo.fedora
+Source2:        netdata.repo.suse
+Source3:        netdata-edge.repo.suse
+Source4:        netdata.repo.centos
+Source5:        netdata-edge.repo.centos
+
+BuildArch:      noarch
+
+# Overlapping file installs
+Conflicts:      netdata-repo-edge
+
+%description
+This package contains the official Netdata package repository configuration for stable versions of Netdata.
+
+%prep
+%setup -q  -c -T
+
+%if 0%{?fedora}
+install -pm 644 %{SOURCE0} ./netdata.repo
+install -pm 644 %{SOURCE1} ./netdata-edge.repo
+%endif
+
+%if 0%{?suse_version}
+install -pm 644 %{SOURCE2} ./netdata.repo
+install -pm 644 %{SOURCE3} ./netdata-edge.repo
+%endif
+
+%if 0%{?centos_ver}
+install -pm 644 %{SOURCE4} ./netdata.repo
+install -pm 644 %{SOURCE5} ./netdata-edge.repo
+%endif
+
+%build
+true
+
+%install
+rm -rf $RPM_BUILD_ROOT
+
+%if 0%{?suse_version}
+install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d
+install -pm 644 netdata.repo $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d
+install -pm 644 netdata-edge.repo $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d
+%else
+install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d
+install -pm 644 netdata.repo $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d
+install -pm 644 netdata-edge.repo $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d
+%endif
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%if 0%{?suse_version}
+%attr(644,root,root) /etc/zypp/repos.d/netdata.repo
+%else
+%attr(644,root,root) /etc/yum.repos.d/netdata.repo
+%endif
+
+%package edge
+Summary:   Netdata nightly repositories configuration.
+Group:     System Environment/Base
+
+# Overlapping file installs
+Conflicts: netdata-repo
+
+%description edge
+This package contains the official Netdata package repository configuration for nightly versions of Netdata.
+
+%files edge
+%if 0%{?suse_version}
+%attr(644,root,root) /etc/zypp/repos.d/netdata-edge.repo
+%else
+%attr(644,root,root) /etc/yum.repos.d/netdata-edge.repo
+%endif
+
+%changelog
+* Mon Jun 14 2021 Austin Hemmelgarn <austin@netdata.cloud> 1-1
+- Initial revision

packaging/repoconfig/netdata.list.in

@@ -0,0 +1,2 @@
+deb https://packagecloud.io/netdata/netdata__VARIANT__/__DISTRO__/ __SUITE__ main
+deb https://packagecloud.io/netdata/netdata-repoconfig/__DISTRO__/ __SUITE__ main

packaging/repoconfig/netdata.repo.centos

@@ -0,0 +1,19 @@
+[netdata]
+name=Netdata
+baseurl=https://packagecloud.io/netdata/netdata/el/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+
+[netdata-repoconfig]
+name=Netdata Repository Config
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/el/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt

packaging/repoconfig/netdata.repo.fedora

@@ -0,0 +1,19 @@
+[netdata]
+name=Netdata
+baseurl=https://packagecloud.io/netdata/netdata/fedora/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+
+[netdata-repoconfig]
+name=Netdata Repository Config
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/fedora/$releasever/$basearch
+repo_gpgcheck=1
+gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt

packaging/repoconfig/netdata.repo.suse

@@ -0,0 +1,19 @@
+[netdata]
+name=Netdata
+baseurl=https://packagecloud.io/netdata/netdata/opensuse/$releasever/$basearch
+repo_gpgcheck=1
+pkg_gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata/gpgkey
+enabled=1
+type=rpm-md
+autorefresh=1
+
+[netdata-repoconfig]
+name=Netdata Repoconfig
+baseurl=https://packagecloud.io/netdata/netdata-repoconfig/opensuse/$releasever/$basearch
+repo_gpgcheck=1
+pkg_gpgcheck=0
+gpgkey=https://packagecloud.io/netdata/netdata-repoconfig/gpgkey
+enabled=1
+type=rpm-md
+autorefresh=1