diff --git a/.gitignore b/.gitignore index 789043e744..6cd8a33057 100644 --- a/.gitignore +++ b/.gitignore @@ -139,8 +139,6 @@ cmake_install.cmake .DS_Store webcopylocal* -contrib/debian/changelog - # converted diagrams diagrams/*.png diagrams/*.svg diff --git a/.travis.yml b/.travis.yml index 6716f5faef..74d8152d44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,8 @@ install: - export NOTIF_CHANNEL="automation-beta" - if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; - export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" +- export LATEST_RELEASE_VERSION="$(cat packaging/version | cut -d'-' -f1)" +- export LATEST_RELEASE_DATE="$(git log -1 --format=%aD "${LATEST_RELEASE_VERSION}" | cat)" - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; - export DEPLOY_REPO="netdata" # Default production packaging repository - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; @@ -69,8 +71,6 @@ stages: if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ - name: "Package ubuntu/bionic" if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ -- name: "Package ubuntu/artful" - if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ # Debian - name: "Package debian/buster" @@ -79,8 +79,6 @@ stages: if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ - name: "Package debian/jessie" if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ -- name: "Package debian/wheezy" - if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ # Build RPM packages under special conditions # Enterprise linux (Covers CentOS, Redhat, Amazon linux) @@ -129,7 +127,6 @@ stages: before_deploy: - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" deploy: - # Beta packages deployment - provider: packagecloud repository: "${DEPLOY_REPO}" username: "${PACKAGING_USER}" @@ -142,19 +139,6 @@ stages: repo: ${TRAVIS_REPO_SLUG} branch: "master" condition: -d "${PACKAGES_DIRECTORY}" - # Production release packages deployment - - provider: packagecloud - repository: "netdata" - username: "netdata" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when packages directory is created - repo: "netdata/netdata" - branch: "master" - condition: -d "${PACKAGES_DIRECTORY}" after_deploy: - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; @@ -165,7 +149,7 @@ stages: - stage: &_DEB_TEMPLATE name: "Build & Publish DEB package" before_install: - - sudo apt-get install -y wget lxc lxc-templates + - sudo apt-get install -y wget lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl - source tests/installer/slack.sh before_script: - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" @@ -177,7 +161,8 @@ stages: - echo "packaging/version:" && cat packaging/version - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chmod -R 755 "/var/lib/lxc" + - sudo chown -R root:travis "/var/lib/lxc" + - sudo chmod -R 750 "/var/lib/lxc" - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh git: depth: false @@ -185,7 +170,6 @@ stages: before_deploy: - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" deploy: - # Beta packages deployment - provider: packagecloud repository: "${DEPLOY_REPO}" username: "${PACKAGING_USER}" @@ -198,19 +182,6 @@ stages: repo: ${TRAVIS_REPO_SLUG} branch: "master" condition: -d "${PACKAGES_DIRECTORY}" - # Production release packages deployment - - provider: packagecloud - repository: "netdata" - username: "netdata" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when build-area directory is created - repo: "netdata/netdata" - branch: "master" - condition: -d "${PACKAGES_DIRECTORY}" after_deploy: - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; @@ -422,16 +393,6 @@ jobs: - - stage: - <<: *_DEB_TEMPLATE - stage: "Package ubuntu/artful" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="artful" BUILD_STRING="ubuntu/artful" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - # Debian distros build - stage: <<: *_DEB_TEMPLATE @@ -463,16 +424,6 @@ jobs: - - stage: - <<: *_DEB_TEMPLATE - stage: "Package debian/wheezy" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="wheezy" BUILD_STRING="debian/wheezy" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - # Enterprise linux builds (Centos, Redhat, Amazon linux (el/6)) # - stage: diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh new file mode 100644 index 0000000000..beb522a35f --- /dev/null +++ b/.travis/package_management/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +UNPACKAGED_NETDATA_PATH="$1" +LATEST_RELEASE_VERSION="$2" + +if [ -z "${LATEST_RELEASE_VERSION}" ]; then + echo "Parameter 'LATEST_RELEASE_VERSION' not defined" + exit 1 +fi + +if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then + echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined" + exit 1 +fi + +echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}" + +echo "Entering ${UNPACKAGED_NETDATA_PATH}" +cd "${UNPACKAGED_NETDATA_PATH}" + +echo "Linking debian -> contrib/debian" +ln -sf contrib/debian debian + +echo "Executing dpkg-buildpackage" +if dpkg-buildpackage --version 2> /dev/null | grep -q "1.18"; then + dpkg-buildpackage --post-clean --pre-clean --build=binary +else + dpkg-buildpackage -b +fi + +echo "DEB build script completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py index b96f994e95..e057362820 100755 --- a/.travis/package_management/common.py +++ b/.travis/package_management/common.py @@ -8,6 +8,21 @@ import lxc import subprocess import os +def fetch_version(orig_build_version): + tag = None + friendly_version = "" + + # TODO: Checksum validations + if str(orig_build_version).count(".latest") == 1: + version_list=str(orig_build_version).replace('v', '').split('.') + friendly_version='.'.join(version_list[0:2]) + "." + version_list[3] + else: + friendly_version = orig_build_version.replace('v', '') + tag = friendly_version # Go to stable tag + print("Version set to %s from %s" % (friendly_version, orig_build_version)) + + return friendly_version, tag + def replace_tag(tag_name, spec, new_tag_content): print("Fixing tag %s in %s" % (tag_name, spec)) @@ -51,24 +66,36 @@ def install_common_dependendencies(container): run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) elif str(os.environ["REPO_TOOL"]).count("yum") == 1: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1: print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"])) else: run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) + elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) else: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + if os.environ["BUILD_STRING"].count("el/6") < 0: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) def prepare_version_source(dest_archive, pkg_friendly_version, tag=None): print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py index c2f2e59a66..91aa4c8c49 100755 --- a/.travis/package_management/configure_deb_lxc_environment.py +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -39,6 +39,8 @@ print("Waiting for container connectivity to start configuration sequence") if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + # Run the required activities now # 1. Create the builder user print("1. Adding user %s" % os.environ['BUILDER_NAME']) @@ -48,28 +50,43 @@ common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) print("2. Installing package dependencies within LXC container") common.install_common_dependendencies(container) +print("2.1 Install .DEB build support packages") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"]) + +print("2.2 Add more dependencies") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) + print ("3. Run install-required-packages scriptlet") -common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) +common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"]) friendly_version="" dest_archive="" download_url="" tag = None - -# TODO: Checksum validations -if str(os.environ['BUILD_VERSION']).count(".latest") == 1: - version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') - friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] -else: - friendly_version = os.environ['BUILD_VERSION'].replace('v', '') - tag = friendly_version # Go to stable tag +friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version) print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) -dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], friendly_version) +dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version) + +if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1: + print("5.1 We are building for Jessie, adjusting control file") + common.run_command_in_host(['sudo', 'rm', 'contrib/debian/control']) + common.run_command_in_host(['sudo', 'cp', 'contrib/debian/control.jessie', 'contrib/debian/control']) common.prepare_version_source(dest_archive, friendly_version, tag=tag) +print("6. Installing build.sh script to build path") +common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian']) + print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py index 5f37808786..e507b6976f 100755 --- a/.travis/package_management/configure_rpm_lxc_environment.py +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -79,15 +79,7 @@ dest_archive="" download_url="" spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] tag = None - -# TODO: Checksum validations -if str(os.environ['BUILD_VERSION']).count(".latest") == 1: - version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') - rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] -else: - rpm_friendly_version = os.environ['BUILD_VERSION'].replace('v', '') - tag = rpm_friendly_version # Go to stable tag - +rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh index 1fb26a95ed..12ed07cc77 100755 --- a/.travis/package_management/prepare_packages.sh +++ b/.travis/package_management/prepare_packages.sh @@ -27,29 +27,36 @@ for d in ${CREATED_CONTAINERS[@]}; do # Pick up any RPMS from builder RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" - echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" + if [ -d "${RPM_BUILD_PATH}" ]; then + echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" - if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then - echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/RPMS" - [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then + echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/RPMS" + [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then + echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/SRPMS" + [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + else + DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}" + echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" + if [ -d "${DEB_BUILD_PATH}" ]; then + cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files" + cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files" + cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files" + cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files" + else + echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation" + fi fi - - if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then - echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/SRPMS" - [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" - fi - - # Pick up any DEBs from builder - DEB_BUILD_PATH="${d}/home/${BUILDER_NAME}/build-area" - echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" - #TODO: During debian clean up we 'll fill this up - done chmod -R 777 "${PACKAGES_DIRECTORY}" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py index 3040bdd621..a0235a73d1 100755 --- a/.travis/package_management/trigger_deb_lxc_build.py +++ b/.travis/package_management/trigger_deb_lxc_build.py @@ -37,15 +37,37 @@ if not container.running or not container.state == "RUNNING": if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") os.environ["EMAIL"] = "bot@netdata.cloud" os.environ["DEBFULLNAME"] = "Netdata builder" # Run the build process on the container -print("Starting DEB build process, running dh-make") -new_version = os.environ["BUILD_VERSION"].replace('v', '') +new_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) +print("Starting DEB build process for version %s" % new_version) -print("Building the package") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "dpkg-buildpackage", "--host-arch", "amd64", "--target-arch", "amd64", "--post-clean", "--pre-clean", "--build=binary", "--release-by=\"Netdata Builder\"", "--build-by=\"Netdata Builder\""]) +netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version) +unpacked_netdata = netdata_tarball.replace(".tar.gz", "") + +print("Extracting tarball %s" % netdata_tarball) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path]) + +print("Fixing changelog tags") +changelog_in_host = "contrib/debian/changelog" +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % os.environ["LATEST_RELEASE_VERSION"].replace("v", ""), changelog_in_host]) +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host]) + +print("Executing gbp dch command..") +common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % os.environ["LATEST_RELEASE_VERSION"], '--new-version=%s' % new_version]) + +print("Copying over changelog to the destination machine") +common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)]) + +print("Running debian build script since %s" % os.environ["LATEST_RELEASE_VERSION"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version]) + +print("Listing contents on build path") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path]) print('Done!') diff --git a/.travis/package_management/yank_stale_rpm.sh b/.travis/package_management/yank_stale_rpm.sh index 5cf938664f..3f76697128 100755 --- a/.travis/package_management/yank_stale_rpm.sh +++ b/.travis/package_management/yank_stale_rpm.sh @@ -21,7 +21,7 @@ fi PACKAGES_DIR="$1" DISTRO="$2" -PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep '\.rpm')" +PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )" if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" diff --git a/contrib/debian/changelog b/contrib/debian/changelog new file mode 100644 index 0000000000..b635606ab9 --- /dev/null +++ b/contrib/debian/changelog @@ -0,0 +1,5 @@ +netdata (PREVIOUS_PACKAGE_VERSION) unstable; urgency=medium + + * Initial Release + +-- Netdata Builder <bot@netdata.cloud> PREVIOUS_PACKAGE_DATE diff --git a/contrib/debian/control b/contrib/debian/control index 0f4f1bc059..36e5d6bc61 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -4,7 +4,22 @@ Build-Depends: debhelper (>= 9), dh-systemd (>= 1.5), dpkg-dev (>= 1.13.19), zlib1g-dev, - uuid-dev + uuid-dev, + libuv1-dev, + liblz4-dev, + libjudy-dev, + libssl-dev, + libmnl-dev, + libjson-c-dev, + libcups2-dev, + libipmimonitoring-dev, + libnetfilter-acct-dev, + autogen, + autoconf, + automake, + pkg-config, + curl, + gcc Section: net Priority: optional Maintainer: Costa Tsaousis <costa@tsaousis.gr> @@ -16,6 +31,17 @@ Architecture: any Depends: adduser, libcap2-bin (>= 1:2.0), lsb-base (>= 3.1-23.2), + zlib1g, + libuuid1, + libuv1, + liblz4-1, + libjudydebian1, + openssl, + libmnl0, + libjson-c3, + cups, + freeipmi, + libnetfilter-acct1, ${misc:Depends}, ${shlibs:Depends} Description: real-time charts for system monitoring diff --git a/contrib/debian/control.jessie b/contrib/debian/control.jessie new file mode 100644 index 0000000000..7478d2e3cc --- /dev/null +++ b/contrib/debian/control.jessie @@ -0,0 +1,49 @@ +Source: netdata +Build-Depends: debhelper (>= 9), + dh-autoreconf, + dh-systemd (>= 1.5), + dpkg-dev (>= 1.13.19), + zlib1g-dev, + uuid-dev, + liblz4-dev, + libjudy-dev, + libssl-dev, + libmnl-dev, + libjson-c-dev, + libcups2-dev, + libipmimonitoring-dev, + libnetfilter-acct-dev, + autogen, + autoconf, + automake, + pkg-config, + curl, + gcc +Section: net +Priority: optional +Maintainer: Costa Tsaousis <costa@tsaousis.gr> +Standards-Version: 3.9.6 +Homepage: https://github.com/netdata/netdata/wiki + +Package: netdata +Architecture: any +Depends: adduser, + libcap2-bin (>= 1:2.0), + lsb-base (>= 3.1-23.2), + zlib1g, + libuuid1, + liblz4-1, + libjudydebian1, + openssl, + libmnl0, + libjson-c3, + cups, + freeipmi, + libnetfilter-acct1, + ${misc:Depends}, + ${shlibs:Depends} +Description: real-time charts for system monitoring + Netdata is a daemon that collects data in realtime (per second) + and presents a web site to view and analyze them. The presentation + is also real-time and full of interactive charts that precisely + render all collected values. diff --git a/contrib/debian/install_go.sh b/contrib/debian/install_go.sh new file mode 100755 index 0000000000..17a3b409e9 --- /dev/null +++ b/contrib/debian/install_go.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +LIB_DIR="$1" +LIBEXEC_DIR="$2" + +# ############################################################ +# Package Go within netdata (TBD: Package it separately) +safe_sha256sum() { + # Within the contexct of the installer, we only use -c option that is common between the two commands + # We will have to reconsider if we start non-common options + if command -v sha256sum >/dev/null 2>&1; then + sha256sum $@ + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 $@ + else + fatal "I could not find a suitable checksum binary to use" + fi +} + +download_go() { + url="${1}" + dest="${2}" + + if command -v curl >/dev/null 2>&1; then + curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" + elif command -v wget >/dev/null 2>&1; then + wget -T 15 -O - "${url}" > "${dest}" + else + echo >&2 + echo >&2 "Downloading go.d plugin from '${url}' failed because of missing mandatory packages." + echo >&2 "Either add packages or disable it by issuing '--disable-go' in the installer" + echo >&2 + exit 1 + fi +} + +install_go() { + # When updating this value, ensure correct checksums in packaging/go.d.checksums + GO_PACKAGE_VERSION="v0.7.0" + ARCH_MAP=( + 'i386::386' + 'i686::386' + 'x86_64::amd64' + 'aarch64::arm64' + 'armv64::arm64' + 'armv6l::arm' + 'armv7l::arm' + 'armv5tel::arm' + ) + + if [ -z "${NETDATA_DISABLE_GO+x}" ]; then + echo >&2 "Install go.d.plugin" + ARCH=$(uname -m) + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + + for index in "${ARCH_MAP[@]}" ; do + KEY="${index%%::*}" + VALUE="${index##*::}" + if [ "$KEY" = "$ARCH" ]; then + ARCH="${VALUE}" + break + fi + done + tmp=$(mktemp -d /tmp/netdata-go-XXXXXX) + GO_PACKAGE_BASENAME="go.d.plugin-${GO_PACKAGE_VERSION}.${OS}-${ARCH}" + download_go "https://github.com/netdata/go.d.plugin/releases/download/${GO_PACKAGE_VERSION}/${GO_PACKAGE_BASENAME}" "${tmp}/${GO_PACKAGE_BASENAME}" + download_go "https://github.com/netdata/go.d.plugin/releases/download/${GO_PACKAGE_VERSION}/config.tar.gz" "${tmp}/config.tar.gz" + + if [ ! -f "${tmp}/${GO_PACKAGE_BASENAME}" ] || [ ! -f "${tmp}/config.tar.gz" ] || [ ! -s "${tmp}/config.tar.gz" ] || [ ! -s "${tmp}/${GO_PACKAGE_BASENAME}" ]; then + echo >&2 "Either check the error or consider disabling it by issuing '--disable-go' in the installer" + echo >&2 + return 1 + fi + + grep "${GO_PACKAGE_BASENAME}\$" "packaging/go.d.checksums" > "${tmp}/sha256sums.txt" 2>/dev/null + grep "config.tar.gz" "packaging/go.d.checksums" >> "${tmp}/sha256sums.txt" 2>/dev/null + + # Checksum validation + if ! (cd "${tmp}" && safe_sha256sum -c "sha256sums.txt"); then + + echo >&2 "go.d plugin checksum validation failure." + echo >&2 "Either check the error or consider disabling it by issuing '--disable-go' in the installer" + echo >&2 + + echo "go.d.plugin package files checksum validation failed." + exit 1 + fi + + # Install files + tar -xf "${tmp}/config.tar.gz" -C "${LIB_DIR}/conf.d/" + mv "${tmp}/$GO_PACKAGE_BASENAME" "${LIBEXEC_DIR}/plugins.d/go.d.plugin" + fi + return 0 +} + +install_go diff --git a/contrib/debian/netdata.postinst.in b/contrib/debian/netdata.postinst.in index 29615f5413..44b53ccfbd 100644 --- a/contrib/debian/netdata.postinst.in +++ b/contrib/debian/netdata.postinst.in @@ -28,9 +28,10 @@ case "$1" in fi dpkg-statoverride --update --add --force root netdata 0775 /var/lib/netdata/registry - chown -R root:netdata /usr/share/netdata/* - chown -R root:netdata /usr/lib/@DEB_HOST_MULTIARCH@/netdata/plugins.d - setcap cap_dac_read_search,cap_sys_ptrace+ep /usr/lib/@DEB_HOST_MULTIARCH@/netdata/plugins.d/apps.plugin + chown -R root:netdata /usr/share/netdata + chown -R root:netdata /usr/libexec/netdata/plugins.d + chown -R root:netdata /var/lib/netdata/www + setcap cap_dac_read_search,cap_sys_ptrace+ep /usr/libexec/netdata/plugins.d/apps.plugin #PERMS# ;; diff --git a/contrib/debian/rules b/contrib/debian/rules index c19323960a..88a8ab36a6 100755 --- a/contrib/debian/rules +++ b/contrib/debian/rules @@ -17,7 +17,9 @@ TOP = $(CURDIR)/debian/netdata #dh $@ --with autoreconf override_dh_auto_configure: - dh_auto_configure -- --with-math --with-webdir=/var/lib/netdata/www + autoreconf -ivf + dh_auto_configure -- --prefix=/usr --sysconfdir=/etc --localstatedir=/var \ + --libexecdir=/usr/libexec --with-user=netdata --with-math --with-webdir=/var/lib/netdata/www debian/%.postinst: debian/%.postinst.in sed 's/@DEB_HOST_MULTIARCH@/$(DEB_HOST_MULTIARCH)/g' $< > $@ @@ -34,7 +36,7 @@ override_dh_install: debian/netdata.postinst mkdir -p "$(TOP)/usr/share/netdata" for D in $$(find "$(TOP)/var/lib/netdata/www/" -maxdepth 1 -type d -printf '%f '); do \ echo Relocating $$D; \ - mv "$(TOP)/var/lib/netdata/www/$$D" "$(TOP)/usr/share/netdata/$$D"; \ + mv "$(TOP)/var/lib/netdata/www/$$D" "$(TOP)/usr/share/netdata/www/$$D"; \ ln -s "/usr/share/netdata/$$D" "$(TOP)/var/lib/netdata/www/$$D"; \ done @@ -51,6 +53,10 @@ override_dh_install: debian/netdata.postinst done sed -i "/^#PERMS#/d" $(CURDIR)/debian/netdata.postinst + # Install go + # + debian/install_go.sh $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/ $(TOP)/usr/libexec/netdata + override_dh_installdocs: dh_installdocs @@ -58,10 +64,11 @@ override_dh_installdocs: -name README.md \ -not -path './.travis/*' \ -not -path './debian/*' \ + -not -path './contrib/*' \ -exec cp \ - --parents \ - --target $(TOP)/usr/share/doc/netdata/ \ - {} \; + --parents \ + --target $(TOP)/usr/share/doc/netdata/ \ + {} \; override_dh_fixperms: dh_fixperms @@ -69,7 +76,10 @@ override_dh_fixperms: # apps.plugin should only be runnable by the netdata user. It will be # given extra capabilities in the postinst script. # - chmod 0754 $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/plugins.d/apps.plugin + chmod 0754 $(TOP)/usr/libexec/netdata/plugins.d/apps.plugin + chmod 0754 $(TOP)/usr/libexec/netdata/plugins.d/freeipmi.plugin + chmod 0754 $(TOP)/usr/libexec/netdata/plugins.d/perf.plugin + chmod 0750 $(TOP)/usr/libexec/netdata/plugins.d/go.d.plugin override_dh_installlogrotate: cp system/netdata.logrotate debian/netdata.logrotate diff --git a/packaging/installer/README.md b/packaging/installer/README.md index 41965ebcbd..15796387e7 100644 --- a/packaging/installer/README.md +++ b/packaging/installer/README.md @@ -298,7 +298,9 @@ To apply the changes you made, you have to restart Netdata. We provide our own flavour of binary packages for the most common operating systems that comply with .RPM and .DEB packaging formats. -We have currently released .RPM versions with version [1.16.0](https://github.com/netdata/netdata/releases/tag/v1.16.0). We are planning to release packages following the .DEB format on one of the following releases. Our current packaging infrastructure provider is [Package Cloud](https://packagecloud.io). +We have currently released packages following the .RPM format with version [1.16.0](https://github.com/netdata/netdata/releases/tag/v1.16.0). +We have planned to release packages following the .DEB format with version [1.17.0](https://github.com/netdata/netdata/releases/tag/v1.17.0). +Early adopters may experiment with our .DEB formatted packages using our nightly releases. Our current packaging infrastructure provider is [Package Cloud](https://packagecloud.io). Netdata is committed to support installation of our solution to all operating systems. This is a constant battle for Netdata, as we strive to automate and make things easier for our users. For the operating system support matrix, please visit our [distributions](../../packaging/DISTRIBUTIONS.md) support page.