netdata/packaging: Enable built-in support for prometheus remote write in packaging (#6480)

* netdata/packaging: [ci skip] Add prometheus remote write plugin deps

* netdata/packaging: [ci skip] adjust for the tests

* netdata/packaging: [ci skip] Apparently we need proto-c too

* netdata/packaging: [ci skip] Add dependencies also

* netdata/packaging: [ci skip] fix suse dependency

* netdata/packaging: [ci skip] epel-release should always go first dumdum

* netdata/packaging: [ci skip] g++ needed for prometheus

* netdata/packaging: First prepare the repo, then start throwing in packages

* netdata/packaging: [ci skip] fix attempt for missing snappy on some debian-based packages

* netdata/packaging: [ci skip] protobuf compiler dep

* netdata/packaging: [ci skip] Add the generic maintainer info and the new website. Also stop pushing snappy to Debian/Stretch, doesnt have it

* netdata/packaging: [ci skip] Remove branch dev setup

* Empty commit to re-trigger Travis

.travis/package_management/common.py

@@ -61,43 +61,73 @@ def run_command_in_host(cmd):
     print('Error: '  + e.decode('ascii'))
     print('code: ' + str(proc.returncode))
 
-def install_common_dependendencies(container):
+def prepare_repo(container):
     if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
         run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"])
         run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
 
     elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
         run_command(container, [os.environ["REPO_TOOL"], "clean", "all"])
         run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
 
         if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1:
             print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"]))
         else:
             run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"])
+
     elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
         run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
     else:
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
         run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
 
-    if os.environ["BUILD_STRING"].count("el/6") <= 0:
-        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"])
-
     run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"])
     run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"])
     run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"])
 
+def install_common_dependendencies(container):
+    if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"])
+
+    elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+
+    elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+        if os.environ["BUILD_STRING"].count("debian/jessie") == 1:
+            run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"])
+    else:
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+
+    if os.environ["BUILD_STRING"].count("el/6") <= 0:
+        run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"])
+
 def prepare_version_source(dest_archive, pkg_friendly_version, tag=None):
     print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version)
     tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive

.travis/package_management/configure_deb_lxc_environment.py

@@ -47,8 +47,8 @@ print("1. Adding user %s" % os.environ['BUILDER_NAME'])
 common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
 
 # Fetch package dependencies for the build
-print("2. Installing package dependencies within LXC container")
+print("2. Preparing repo on LXC container")
-common.install_common_dependendencies(container)
+common.prepare_repo(container)
 
 print("2.1 Install .DEB build support packages")
 common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"])
@@ -62,10 +62,13 @@ print("2.2 Add more dependencies")
 common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"])
 common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
 
-print ("3. Run install-required-packages scriptlet")
+print ("3.1 Run install-required-packages scriptlet")
 common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"])
 common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"])
 
+print("3.2 Installing package dependencies within LXC container")
+common.install_common_dependendencies(container)
+
 friendly_version=""
 dest_archive=""
 download_url=""

.travis/package_management/configure_rpm_lxc_environment.py

@@ -46,11 +46,15 @@ print("1. Adding user %s" % os.environ['BUILDER_NAME'])
 common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
 
 # Fetch package dependencies for the build
-print("2. Installing package dependencies within LXC container")
+print("2.1 Preparing repo on LXC container")
-common.install_common_dependendencies(container)
+common.prepare_repo(container)
+
+common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"])
+common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"])
 
 # Exceptional cases, not available everywhere
 #
+print("2.2 Running uncommon dependencies and preparing LXC environment")
 # Not on Centos-7
 if os.environ["BUILD_STRING"].count("el/7") <= 0:
     common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"])
@@ -59,8 +63,8 @@ if os.environ["BUILD_STRING"].count("el/7") <= 0:
 if os.environ["BUILD_STRING"].count("el/6") <= 0:
     common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"])
 
-common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"])
+print("2.3 Installing common dependencies")
-common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"])
+common.install_common_dependendencies(container)
 
 print("3. Setting up macros")
 common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"])

contrib/debian/control

@@ -14,17 +14,21 @@ Build-Depends: debhelper (>= 9),
                libcups2-dev,
                libipmimonitoring-dev,
                libnetfilter-acct-dev,
+               libsnappy-dev,
+               libprotobuf-dev,
+               libprotoc-dev,
                autogen,
                autoconf,
                automake,
                pkg-config,
                curl,
-               gcc
+               gcc,
+               g++
 Section: net
 Priority: optional
-Maintainer: Costa Tsaousis <costa@tsaousis.gr>
+Maintainer: Netdata Builder <bot@netdata.cloud>
 Standards-Version: 3.9.6
-Homepage: https://github.com/netdata/netdata/wiki
+Homepage: https://netdata.cloud
 
 Package: netdata
 Architecture: any
@@ -42,6 +46,9 @@ Depends: adduser,
          cups,
          freeipmi,
          libnetfilter-acct1,
+         libprotobuf-c1,
+         libsnappy1v5,
+         libprotoc10,
          ${misc:Depends},
          ${shlibs:Depends}
 Description: real-time charts for system monitoring

contrib/debian/control.jessie

@@ -13,12 +13,16 @@ Build-Depends: debhelper (>= 9),
                libcups2-dev,
                libipmimonitoring-dev,
                libnetfilter-acct-dev,
+               libsnappy-dev,
+               libprotobuf-dev,
+               libprotoc-dev,
                autogen,
                autoconf,
                automake,
                pkg-config,
                curl,
-               gcc
+               gcc,
+               g++
 Section: net
 Priority: optional
 Maintainer: Costa Tsaousis <costa@tsaousis.gr>
@@ -40,6 +44,9 @@ Depends: adduser,
          cups,
          freeipmi,
          libnetfilter-acct1,
+         libprotobuf-c1,
+         libsnappy1v5,
+         libprotoc10,
          ${misc:Depends},
          ${shlibs:Depends}
 Description: real-time charts for system monitoring

netdata.spec.in

@@ -92,6 +92,7 @@ URL:		http://my-netdata.io
 # Build dependencies
 #
 BuildRequires: gcc
+BuildRequires: gcc-c++
 BuildRequires: make
 BuildRequires: git
 BuildRequires: autoconf
@@ -190,6 +191,26 @@ BuildRequires: cups-devel
 Requires: cups
 # end - cups plugin dependencies
 
+# Prometheus remote write dependencies
+BuildRequires: snappy-devel
+BuildRequires: protobuf-devel
+%if 0%{?suse_version}
+BuildRequires: libprotobuf-c-devel
+%else
+BuildRequires: protobuf-c-devel
+%endif
+
+%if 0%{?suse_version}
+Requires: libsnappy1
+Requires: protobuf-c
+Requires: libprotobuf15
+%else
+Requires: snappy
+Requires: protobuf-c
+Requires: protobuf
+%endif
+# end - prometheus remote write dependencies
+
 # #####################################################################
 # End of dependency management configuration
 # #####################################################################