Use dagger to build and test the agent. (#16868)

Dag

packaging/dag/README.md

@@ -0,0 +1,23 @@
+- Install Dagger CLI:
+  ```
+  cd /usr/local
+  curl -L https://dl.dagger.io/dagger/install.sh | sudo sh
+  ```
+- Install Python's Dagger SDK:
+  ```
+  pip install dagger-io
+  ```
+
+Now you can run something like this:
+
+```
+dagger run python packaging/dag/main.py build -p linux/x86_64 -d debian12
+```
+
+or
+
+```
+dagger run python packaging/dag/main.py test
+```.
+
+

packaging/dag/build_command.py

@@ -0,0 +1,65 @@
+import click
+import asyncio
+import sys
+import dagger
+import pathlib
+import uuid
+
+from nd import (
+    Distribution,
+    NetdataInstaller,
+    FeatureFlags,
+    Endpoint,
+    AgentContext,
+    SUPPORTED_PLATFORMS,
+    SUPPORTED_DISTRIBUTIONS,
+)
+
+
+def run_async(func):
+    def wrapper(*args, **kwargs):
+        return asyncio.run(func(*args, **kwargs))
+
+    return wrapper
+
+
+@run_async
+async def simple_build(platform, distro):
+    config = dagger.Config(log_output=sys.stdout)
+
+    async with dagger.Connection(config) as client:
+        repo_root = pathlib.Path("/netdata")
+        prefix_path = pathlib.Path("/opt/netdata")
+
+        installer = NetdataInstaller(
+            platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine
+        )
+
+        endpoint = Endpoint("node", 19999)
+        api_key = uuid.uuid4()
+        allow_children = False
+
+        agent_ctx = AgentContext(
+            client, platform, distro, installer, endpoint, api_key, allow_children
+        )
+
+        await agent_ctx.build_container()
+
+
+@click.command()
+@click.option(
+    "--platform",
+    "-p",
+    type=click.Choice(sorted([str(p) for p in SUPPORTED_PLATFORMS])),
+    help="Specify the platform.",
+)
+@click.option(
+    "--distribution",
+    "-d",
+    type=click.Choice(sorted([str(p) for p in SUPPORTED_DISTRIBUTIONS])),
+    help="Specify the distribution.",
+)
+def build(platform, distribution):
+    platform = dagger.Platform(platform)
+    distro = Distribution(distribution)
+    simple_build(platform, distro)

packaging/dag/files/child_stream.conf

@@ -0,0 +1,10 @@
+[stream]
+    enabled = {{ enabled }}
+    destination = {{ destination }}
+    api key = {{ api_key }}
+    timeout seconds = {{ timeout_seconds }}
+    default port = {{ default_port }}
+    send charts matching = {{ send_charts_matching }}
+    buffer size bytes = {{ buffer_size_bytes }}
+    reconnect delay seconds = {{ reconnect_delay_seconds }}
+    initial clock resync iterations = {{ initial_clock_resync_iterations }}

packaging/dag/files/cmake-aarch64.sha256

@@ -0,0 +1 @@
+a83e01ed1cdf44c2e33e0726513b9a35a8c09e3b5a126fd720b3c8a9d5552368  cmake-aarch64.sh

packaging/dag/files/cmake-x86_64.sha256

@@ -0,0 +1 @@
+8c449dabb2b2563ec4e6d5e0fb0ae09e729680efab71527b59015131cea4a042  cmake-x86_64.sh

packaging/dag/files/ol8-epel.repo

@@ -0,0 +1,6 @@
+[ol8_developer_EPEL]
+name=Oracle Linux $releasever EPEL Packages for Development ($basearch)
+baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL8/developer/EPEL/$basearch/
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
+gpgcheck=1
+enabled=1

packaging/dag/files/ol9-epel.repo

@@ -0,0 +1,6 @@
+[ol9_developer_EPEL]
+name=Oracle Linux $releasever EPEL Packages for Development ($basearch)
+baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL9/developer/EPEL/$basearch/
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
+gpgcheck=1
+enabled=1

packaging/dag/files/parent_stream.conf

@@ -0,0 +1,7 @@
+[{{ api_key }}]
+    enabled = {{ enabled }}
+    allow from = {{ allow_from }}
+    default history = {{ default_history }}
+    health enabled by default = {{ health_enabled_by_default }}
+    default postpone alarms on connect seconds = {{ default_postpone_alarms_on_connect_seconds }}
+    multiple connections = {{ multiple_connections }}

packaging/dag/main.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import click
+
+from test_command import test
+from build_command import build
+
+
+@click.group()
+def cli():
+    pass
+
+
+cli.add_command(test)
+cli.add_command(build)
+
+if __name__ == "__main__":
+    cli()

packaging/dag/nd.py

@@ -0,0 +1,409 @@
+from typing import List
+
+import enum
+import os
+import pathlib
+import uuid
+
+import dagger
+import jinja2
+
+import imageutils
+
+
+class Platform:
+    def __init__(self, platform: str):
+        self.platform = dagger.Platform(platform)
+
+    def escaped(self) -> str:
+        return str(self.platform).removeprefix("linux/").replace("/", "_")
+
+    def __eq__(self, other):
+        if isinstance(other, Platform):
+            return self.platform == other.platform
+        elif isinstance(other, dagger.Platform):
+            return self.platform == other
+        else:
+            return NotImplemented
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(self.platform)
+
+    def __str__(self) -> str:
+        return str(self.platform)
+
+
+SUPPORTED_PLATFORMS = set(
+    [
+        Platform("linux/x86_64"),
+        Platform("linux/arm64"),
+        Platform("linux/i386"),
+        Platform("linux/arm/v7"),
+        Platform("linux/arm/v6"),
+        Platform("linux/ppc64le"),
+        Platform("linux/s390x"),
+        Platform("linux/riscv64"),
+    ]
+)
+
+
+SUPPORTED_DISTRIBUTIONS = set(
+    [
+        "alpine_3_18",
+        "alpine_3_19",
+        "amazonlinux2",
+        "centos7",
+        "centos-stream8",
+        "centos-stream9",
+        "debian10",
+        "debian11",
+        "debian12",
+        "fedora37",
+        "fedora38",
+        "fedora39",
+        "opensuse15.4",
+        "opensuse15.5",
+        "opensusetumbleweed",
+        "oraclelinux8",
+        "oraclelinux9",
+        "rockylinux8",
+        "rockylinux9",
+        "ubuntu20.04",
+        "ubuntu22.04",
+        "ubuntu23.04",
+        "ubuntu23.10",
+    ]
+)
+
+
+class Distribution:
+    def __init__(self, display_name):
+        self.display_name = display_name
+
+        if self.display_name == "alpine_3_18":
+            self.docker_tag = "alpine:3.18"
+            self.builder = imageutils.build_alpine_3_18
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "alpine_3_19":
+            self.docker_tag = "alpine:3.19"
+            self.builder = imageutils.build_alpine_3_19
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "amazonlinux2":
+            self.docker_tag = "amazonlinux:2"
+            self.builder = imageutils.build_amazon_linux_2
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos7":
+            self.docker_tag = "centos:7"
+            self.builder = imageutils.build_centos_7
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos-stream8":
+            self.docker_tag = "quay.io/centos/centos:stream8"
+            self.builder = imageutils.build_centos_stream_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos-stream9":
+            self.docker_tag = "quay.io/centos/centos:stream9"
+            self.builder = imageutils.build_centos_stream_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian10":
+            self.docker_tag = "debian:10"
+            self.builder = imageutils.build_debian_10
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian11":
+            self.docker_tag = "debian:11"
+            self.builder = imageutils.build_debian_11
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian12":
+            self.docker_tag = "debian:12"
+            self.builder = imageutils.build_debian_12
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora37":
+            self.docker_tag = "fedora:37"
+            self.builder = imageutils.build_fedora_37
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora38":
+            self.docker_tag = "fedora:38"
+            self.builder = imageutils.build_fedora_38
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora39":
+            self.docker_tag = "fedora:39"
+            self.platforms = SUPPORTED_PLATFORMS
+            self.builder = imageutils.build_fedora_39
+        elif self.display_name == "opensuse15.4":
+            self.docker_tag = "opensuse/leap:15.4"
+            self.builder = imageutils.build_opensuse_15_4
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "opensuse15.5":
+            self.docker_tag = "opensuse/leap:15.5"
+            self.builder = imageutils.build_opensuse_15_5
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "opensusetumbleweed":
+            self.docker_tag = "opensuse/tumbleweed:latest"
+            self.builder = imageutils.build_opensuse_tumbleweed
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "oraclelinux8":
+            self.docker_tag = "oraclelinux:8"
+            self.builder = imageutils.build_oracle_linux_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "oraclelinux9":
+            self.docker_tag = "oraclelinux:9"
+            self.builder = imageutils.build_oracle_linux_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "rockylinux8":
+            self.docker_tag = "rockylinux:8"
+            self.builder = imageutils.build_rocky_linux_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "rockylinux9":
+            self.docker_tag = "rockylinux:9"
+            self.builder = imageutils.build_rocky_linux_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu20.04":
+            self.docker_tag = "ubuntu:20.04"
+            self.builder = imageutils.build_ubuntu_20_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu22.04":
+            self.docker_tag = "ubuntu:22.04"
+            self.builder = imageutils.build_ubuntu_22_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu23.04":
+            self.docker_tag = "ubuntu:23.04"
+            self.builder = imageutils.build_ubuntu_23_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu23.10":
+            self.docker_tag = "ubuntu:23.10"
+            self.builder = imageutils.build_ubuntu_23_10
+            self.platforms = SUPPORTED_PLATFORMS
+        else:
+            raise ValueError(f"Unknown distribution: {self.display_name}")
+
+    def _cache_volume(
+        self, client: dagger.Client, platform: dagger.Platform, path: str
+    ) -> dagger.CacheVolume:
+        tag = "_".join([self.display_name, Platform(platform).escaped()])
+        return client.cache_volume(f"{path}-{tag}")
+
+    def build(
+        self, client: dagger.Client, platform: dagger.Platform
+    ) -> dagger.Container:
+        if platform not in self.platforms:
+            raise ValueError(
+                f"Building {self.display_name} is not supported on {platform}."
+            )
+
+        ctr = self.builder(client, platform)
+        ctr = imageutils.install_cargo(ctr)
+
+        return ctr
+
+
+class FeatureFlags(enum.Flag):
+    DBEngine = enum.auto()
+    GoPlugin = enum.auto()
+    ExtendedBPF = enum.auto()
+    LogsManagement = enum.auto()
+    MachineLearning = enum.auto()
+    BundledProtobuf = enum.auto()
+
+
+class NetdataInstaller:
+    def __init__(
+        self,
+        platform: Platform,
+        distro: Distribution,
+        repo_root: pathlib.Path,
+        prefix: pathlib.Path,
+        features: FeatureFlags,
+    ):
+        self.platform = platform
+        self.distro = distro
+        self.repo_root = repo_root
+        self.prefix = prefix
+        self.features = features
+
+    def _mount_repo(
+        self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
+    ) -> dagger.Container:
+        host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
+        exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]
+
+        # The installer builds/stores intermediate artifacts under externaldeps/
+        # We add a volume to speed up rebuilds. The volume has to be unique
+        # per platform/distro in order to avoid mixing unrelated artifacts
+        # together.
+        externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")
+
+        ctr = (
+            ctr.with_directory(
+                self.repo_root.as_posix(), client.host().directory(host_repo_root)
+            )
+            .with_workdir(self.repo_root.as_posix())
+            .with_mounted_cache(
+                os.path.join(self.repo_root, "externaldeps"), externaldeps
+            )
+        )
+
+        return ctr
+
+    def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
+        args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]
+
+        if FeatureFlags.DBEngine not in self.features:
+            args.append("--disable-dbengine")
+
+        if FeatureFlags.GoPlugin not in self.features:
+            args.append("--disable-go")
+
+        if FeatureFlags.ExtendedBPF not in self.features:
+            args.append("--disable-ebpf")
+
+        if FeatureFlags.LogsManagement not in self.features:
+            args.append("--disable-logsmanagement")
+
+        if FeatureFlags.MachineLearning not in self.features:
+            args.append("--disable-ml")
+
+        if FeatureFlags.BundledProtobuf not in self.features:
+            args.append("--use-system-protobuf")
+
+        args.extend(["--install-prefix", self.prefix.parent.as_posix()])
+
+        ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())
+
+        ctr = ctr.with_env_variable(
+            "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
+        ).with_exec(["./netdata-installer.sh"] + args)
+
+        return ctr
+
+
+class Endpoint:
+    def __init__(self, hostname: str, port: int):
+        self.hostname = hostname
+        self.port = port
+
+    def __str__(self):
+        return ":".join([self.hostname, str(self.port)])
+
+
+class ChildStreamConf:
+    def __init__(
+        self,
+        installer: NetdataInstaller,
+        destinations: List[Endpoint],
+        api_key: uuid.UUID,
+    ):
+        self.installer = installer
+        self.substitutions = {
+            "enabled": "yes",
+            "destination": " ".join([str(dst) for dst in destinations]),
+            "api_key": api_key,
+            "timeout_seconds": 60,
+            "default_port": 19999,
+            "send_charts_matching": "*",
+            "buffer_size_bytes": 1024 * 1024,
+            "reconnect_delay_seconds": 5,
+            "initial_clock_resync_iterations": 60,
+        }
+
+    def render(self) -> str:
+        tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
+        with open(tmpl_path) as fp:
+            tmpl = jinja2.Template(fp.read())
+
+        return tmpl.render(**self.substitutions)
+
+
+class ParentStreamConf:
+    def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
+        self.installer = installer
+        self.substitutions = {
+            "api_key": str(api_key),
+            "enabled": "yes",
+            "allow_from": "*",
+            "default_history": 3600,
+            "health_enabled_by_default": "auto",
+            "default_postpone_alarms_on_connect_seconds": 60,
+            "multiple_connections": "allow",
+        }
+
+    def render(self) -> str:
+        tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
+        with open(tmpl_path) as fp:
+            tmpl = jinja2.Template(fp.read())
+
+        return tmpl.render(**self.substitutions)
+
+
+class StreamConf:
+    def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
+        self.child_conf = child_conf
+        self.parent_conf = parent_conf
+
+    def render(self) -> str:
+        child_section = self.child_conf.render() if self.child_conf else ""
+        parent_section = self.parent_conf.render() if self.parent_conf else ""
+        return "\n".join([child_section, parent_section])
+
+
+class AgentContext:
+    def __init__(
+        self,
+        client: dagger.Client,
+        platform: dagger.Platform,
+        distro: Distribution,
+        installer: NetdataInstaller,
+        endpoint: Endpoint,
+        api_key: uuid.UUID,
+        allow_children: bool,
+    ):
+        self.client = client
+        self.platform = platform
+        self.distro = distro
+        self.installer = installer
+        self.endpoint = endpoint
+        self.api_key = api_key
+        self.allow_children = allow_children
+
+        self.parent_contexts = []
+
+        self.built_distro = False
+        self.built_agent = False
+
+    def add_parent(self, parent_context: "AgentContext"):
+        self.parent_contexts.append(parent_context)
+
+    def build_container(self) -> dagger.Container:
+        ctr = self.distro.build(self.client, self.platform)
+        ctr = self.installer.install(self.client, ctr)
+
+        if len(self.parent_contexts) == 0 and not self.allow_children:
+            return ctr.with_exposed_port(self.endpoint.port)
+
+        destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
+        child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)
+
+        parent_stream_conf = None
+        if self.allow_children:
+            parent_stream_conf = ParentStreamConf(self.installer, self.api_key)
+
+        stream_conf = StreamConf(child_stream_conf, parent_stream_conf)
+
+        # write the stream conf to localhost and cp it in the container
+        host_stream_conf_path = pathlib.Path(
+            f"/tmp/{self.endpoint.hostname}_stream.conf"
+        )
+        with open(host_stream_conf_path, "w") as fp:
+            fp.write(stream_conf.render())
+
+        ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"
+
+        ctr = ctr.with_file(
+            ctr_stream_conf_path.as_posix(),
+            self.client.host().file(host_stream_conf_path.as_posix()),
+        )
+
+        ctr = ctr.with_exposed_port(self.endpoint.port)
+
+        return ctr

packaging/dag/test_command.py

@@ -0,0 +1,128 @@
+import click
+import asyncio
+import sys
+import pathlib
+import dagger
+import uuid
+import httpx
+
+from nd import Distribution, NetdataInstaller, FeatureFlags, Endpoint, AgentContext
+
+
+def run_async(func):
+    def wrapper(*args, **kwargs):
+        return asyncio.run(func(*args, **kwargs))
+
+    return wrapper
+
+
+@run_async
+async def simple_test():
+    config = dagger.Config(log_output=sys.stdout)
+
+    async with dagger.Connection(config) as client:
+        platform = dagger.Platform("linux/x86_64")
+        distro = Distribution("debian10")
+
+        repo_root = pathlib.Path("/netdata")
+        prefix_path = pathlib.Path("/opt/netdata")
+        installer = NetdataInstaller(
+            platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine
+        )
+
+        api_key = uuid.uuid4()
+
+        #
+        # parent
+        #
+        parent_endpoint = Endpoint("parent1", 22000)
+        parent_ctx = AgentContext(
+            client, platform, distro, installer, parent_endpoint, api_key, True
+        )
+        parent_cmd = installer.prefix / "usr/sbin/netdata"
+        parent_args = [
+            parent_cmd.as_posix(),
+            "-D",
+            "-i",
+            "0.0.0.0",
+            "-p",
+            str(parent_endpoint.port),
+        ]
+
+        parent_ctr = parent_ctx.build_container()
+        parent_ctr = parent_ctr.with_exec(parent_args)
+        parent_svc = parent_ctr.as_service()
+
+        #
+        # child
+        #
+        child_endpoint = Endpoint("child1", 21000)
+        child_ctx = AgentContext(
+            client, platform, distro, installer, child_endpoint, api_key, False
+        )
+        child_ctx.add_parent(parent_ctx)
+        child_cmd = installer.prefix / "usr/sbin/netdata"
+        child_args = [
+            child_cmd.as_posix(),
+            "-D",
+            "-i",
+            "0.0.0.0",
+            "-p",
+            str(child_endpoint.port),
+        ]
+
+        child_ctr = child_ctx.build_container()
+        child_ctr = child_ctr.with_service_binding(parent_endpoint.hostname, parent_svc)
+        child_ctr = child_ctr.with_exec(child_args)
+        child_svc = child_ctr.as_service()
+
+        #
+        # endpoints
+        #
+        parent_tunnel, child_tunnel = await asyncio.gather(
+            client.host().tunnel(parent_svc, native=True).start(),
+            client.host().tunnel(child_svc, native=True).start(),
+        )
+
+        parent_endpoint, child_endpoint = await asyncio.gather(
+            parent_tunnel.endpoint(),
+            child_tunnel.endpoint(),
+        )
+
+        await asyncio.sleep(10)
+
+        #
+        # run tests
+        #
+
+        async with httpx.AsyncClient() as http:
+            resp = await http.get(f"http://{parent_endpoint}/api/v1/info")
+
+        #
+        # Check that the child was connected
+        #
+        jd = resp.json()
+        assert (
+            "hosts-available" in jd
+        ), "Could not find 'host-available' key in api/v1/info"
+        assert jd["hosts-available"] == 2, "Child did not connect to parent"
+
+        #
+        # Check bearer protection
+        #
+        forbidden_urls = [
+            f"http://{parent_endpoint}/api/v2/bearer_protection",
+            f"http://{parent_endpoint}/api/v2/bearer_get_token",
+        ]
+
+        for url in forbidden_urls:
+            async with httpx.AsyncClient() as http:
+                resp = await http.get(url)
+            assert (
+                resp.status_code == httpx.codes.UNAVAILABLE_FOR_LEGAL_REASONS
+            ), "Bearer protection is broken"
+
+
+@click.command(help="Run a simple parent/child test")
+def test():
+    simple_test()