410 lines
14 KiB
Python
410 lines
14 KiB
Python
from typing import List
|
|
|
|
import enum
|
|
import os
|
|
import pathlib
|
|
import uuid
|
|
|
|
import dagger
|
|
import jinja2
|
|
|
|
import imageutils
|
|
|
|
|
|
class Platform:
|
|
def __init__(self, platform: str):
|
|
self.platform = dagger.Platform(platform)
|
|
|
|
def escaped(self) -> str:
|
|
return str(self.platform).removeprefix("linux/").replace("/", "_")
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, Platform):
|
|
return self.platform == other.platform
|
|
elif isinstance(other, dagger.Platform):
|
|
return self.platform == other
|
|
else:
|
|
return NotImplemented
|
|
|
|
def __ne__(self, other):
|
|
return not (self == other)
|
|
|
|
def __hash__(self):
|
|
return hash(self.platform)
|
|
|
|
def __str__(self) -> str:
|
|
return str(self.platform)
|
|
|
|
|
|
SUPPORTED_PLATFORMS = set(
|
|
[
|
|
Platform("linux/x86_64"),
|
|
Platform("linux/arm64"),
|
|
Platform("linux/i386"),
|
|
Platform("linux/arm/v7"),
|
|
Platform("linux/arm/v6"),
|
|
Platform("linux/ppc64le"),
|
|
Platform("linux/s390x"),
|
|
Platform("linux/riscv64"),
|
|
]
|
|
)
|
|
|
|
|
|
SUPPORTED_DISTRIBUTIONS = set(
|
|
[
|
|
"alpine_3_18",
|
|
"alpine_3_19",
|
|
"amazonlinux2",
|
|
"centos7",
|
|
"centos-stream8",
|
|
"centos-stream9",
|
|
"debian10",
|
|
"debian11",
|
|
"debian12",
|
|
"fedora37",
|
|
"fedora38",
|
|
"fedora39",
|
|
"opensuse15.4",
|
|
"opensuse15.5",
|
|
"opensusetumbleweed",
|
|
"oraclelinux8",
|
|
"oraclelinux9",
|
|
"rockylinux8",
|
|
"rockylinux9",
|
|
"ubuntu20.04",
|
|
"ubuntu22.04",
|
|
"ubuntu23.04",
|
|
"ubuntu23.10",
|
|
]
|
|
)
|
|
|
|
|
|
class Distribution:
|
|
def __init__(self, display_name):
|
|
self.display_name = display_name
|
|
|
|
if self.display_name == "alpine_3_18":
|
|
self.docker_tag = "alpine:3.18"
|
|
self.builder = imageutils.build_alpine_3_18
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "alpine_3_19":
|
|
self.docker_tag = "alpine:3.19"
|
|
self.builder = imageutils.build_alpine_3_19
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "amazonlinux2":
|
|
self.docker_tag = "amazonlinux:2"
|
|
self.builder = imageutils.build_amazon_linux_2
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "centos7":
|
|
self.docker_tag = "centos:7"
|
|
self.builder = imageutils.build_centos_7
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "centos-stream8":
|
|
self.docker_tag = "quay.io/centos/centos:stream8"
|
|
self.builder = imageutils.build_centos_stream_8
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "centos-stream9":
|
|
self.docker_tag = "quay.io/centos/centos:stream9"
|
|
self.builder = imageutils.build_centos_stream_9
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "debian10":
|
|
self.docker_tag = "debian:10"
|
|
self.builder = imageutils.build_debian_10
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "debian11":
|
|
self.docker_tag = "debian:11"
|
|
self.builder = imageutils.build_debian_11
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "debian12":
|
|
self.docker_tag = "debian:12"
|
|
self.builder = imageutils.build_debian_12
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "fedora37":
|
|
self.docker_tag = "fedora:37"
|
|
self.builder = imageutils.build_fedora_37
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "fedora38":
|
|
self.docker_tag = "fedora:38"
|
|
self.builder = imageutils.build_fedora_38
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "fedora39":
|
|
self.docker_tag = "fedora:39"
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
self.builder = imageutils.build_fedora_39
|
|
elif self.display_name == "opensuse15.4":
|
|
self.docker_tag = "opensuse/leap:15.4"
|
|
self.builder = imageutils.build_opensuse_15_4
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "opensuse15.5":
|
|
self.docker_tag = "opensuse/leap:15.5"
|
|
self.builder = imageutils.build_opensuse_15_5
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "opensusetumbleweed":
|
|
self.docker_tag = "opensuse/tumbleweed:latest"
|
|
self.builder = imageutils.build_opensuse_tumbleweed
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "oraclelinux8":
|
|
self.docker_tag = "oraclelinux:8"
|
|
self.builder = imageutils.build_oracle_linux_8
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "oraclelinux9":
|
|
self.docker_tag = "oraclelinux:9"
|
|
self.builder = imageutils.build_oracle_linux_9
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "rockylinux8":
|
|
self.docker_tag = "rockylinux:8"
|
|
self.builder = imageutils.build_rocky_linux_8
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "rockylinux9":
|
|
self.docker_tag = "rockylinux:9"
|
|
self.builder = imageutils.build_rocky_linux_9
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "ubuntu20.04":
|
|
self.docker_tag = "ubuntu:20.04"
|
|
self.builder = imageutils.build_ubuntu_20_04
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "ubuntu22.04":
|
|
self.docker_tag = "ubuntu:22.04"
|
|
self.builder = imageutils.build_ubuntu_22_04
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "ubuntu23.04":
|
|
self.docker_tag = "ubuntu:23.04"
|
|
self.builder = imageutils.build_ubuntu_23_04
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
elif self.display_name == "ubuntu23.10":
|
|
self.docker_tag = "ubuntu:23.10"
|
|
self.builder = imageutils.build_ubuntu_23_10
|
|
self.platforms = SUPPORTED_PLATFORMS
|
|
else:
|
|
raise ValueError(f"Unknown distribution: {self.display_name}")
|
|
|
|
def _cache_volume(
|
|
self, client: dagger.Client, platform: dagger.Platform, path: str
|
|
) -> dagger.CacheVolume:
|
|
tag = "_".join([self.display_name, Platform(platform).escaped()])
|
|
return client.cache_volume(f"{path}-{tag}")
|
|
|
|
def build(
|
|
self, client: dagger.Client, platform: dagger.Platform
|
|
) -> dagger.Container:
|
|
if platform not in self.platforms:
|
|
raise ValueError(
|
|
f"Building {self.display_name} is not supported on {platform}."
|
|
)
|
|
|
|
ctr = self.builder(client, platform)
|
|
ctr = imageutils.install_cargo(ctr)
|
|
|
|
return ctr
|
|
|
|
|
|
class FeatureFlags(enum.Flag):
|
|
DBEngine = enum.auto()
|
|
GoPlugin = enum.auto()
|
|
ExtendedBPF = enum.auto()
|
|
LogsManagement = enum.auto()
|
|
MachineLearning = enum.auto()
|
|
BundledProtobuf = enum.auto()
|
|
|
|
|
|
class NetdataInstaller:
|
|
def __init__(
|
|
self,
|
|
platform: Platform,
|
|
distro: Distribution,
|
|
repo_root: pathlib.Path,
|
|
prefix: pathlib.Path,
|
|
features: FeatureFlags,
|
|
):
|
|
self.platform = platform
|
|
self.distro = distro
|
|
self.repo_root = repo_root
|
|
self.prefix = prefix
|
|
self.features = features
|
|
|
|
def _mount_repo(
|
|
self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
|
|
) -> dagger.Container:
|
|
host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
|
|
exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]
|
|
|
|
# The installer builds/stores intermediate artifacts under externaldeps/
|
|
# We add a volume to speed up rebuilds. The volume has to be unique
|
|
# per platform/distro in order to avoid mixing unrelated artifacts
|
|
# together.
|
|
externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")
|
|
|
|
ctr = (
|
|
ctr.with_directory(
|
|
self.repo_root.as_posix(), client.host().directory(host_repo_root)
|
|
)
|
|
.with_workdir(self.repo_root.as_posix())
|
|
.with_mounted_cache(
|
|
os.path.join(self.repo_root, "externaldeps"), externaldeps
|
|
)
|
|
)
|
|
|
|
return ctr
|
|
|
|
def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
|
|
args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]
|
|
|
|
if FeatureFlags.DBEngine not in self.features:
|
|
args.append("--disable-dbengine")
|
|
|
|
if FeatureFlags.GoPlugin not in self.features:
|
|
args.append("--disable-go")
|
|
|
|
if FeatureFlags.ExtendedBPF not in self.features:
|
|
args.append("--disable-ebpf")
|
|
|
|
if FeatureFlags.LogsManagement not in self.features:
|
|
args.append("--disable-logsmanagement")
|
|
|
|
if FeatureFlags.MachineLearning not in self.features:
|
|
args.append("--disable-ml")
|
|
|
|
if FeatureFlags.BundledProtobuf not in self.features:
|
|
args.append("--use-system-protobuf")
|
|
|
|
args.extend(["--install-prefix", self.prefix.parent.as_posix()])
|
|
|
|
ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())
|
|
|
|
ctr = ctr.with_env_variable(
|
|
"NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
|
|
).with_exec(["./netdata-installer.sh"] + args)
|
|
|
|
return ctr
|
|
|
|
|
|
class Endpoint:
|
|
def __init__(self, hostname: str, port: int):
|
|
self.hostname = hostname
|
|
self.port = port
|
|
|
|
def __str__(self):
|
|
return ":".join([self.hostname, str(self.port)])
|
|
|
|
|
|
class ChildStreamConf:
|
|
def __init__(
|
|
self,
|
|
installer: NetdataInstaller,
|
|
destinations: List[Endpoint],
|
|
api_key: uuid.UUID,
|
|
):
|
|
self.installer = installer
|
|
self.substitutions = {
|
|
"enabled": "yes",
|
|
"destination": " ".join([str(dst) for dst in destinations]),
|
|
"api_key": api_key,
|
|
"timeout_seconds": 60,
|
|
"default_port": 19999,
|
|
"send_charts_matching": "*",
|
|
"buffer_size_bytes": 1024 * 1024,
|
|
"reconnect_delay_seconds": 5,
|
|
"initial_clock_resync_iterations": 60,
|
|
}
|
|
|
|
def render(self) -> str:
|
|
tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
|
|
with open(tmpl_path) as fp:
|
|
tmpl = jinja2.Template(fp.read())
|
|
|
|
return tmpl.render(**self.substitutions)
|
|
|
|
|
|
class ParentStreamConf:
|
|
def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
|
|
self.installer = installer
|
|
self.substitutions = {
|
|
"api_key": str(api_key),
|
|
"enabled": "yes",
|
|
"allow_from": "*",
|
|
"default_history": 3600,
|
|
"health_enabled_by_default": "auto",
|
|
"default_postpone_alarms_on_connect_seconds": 60,
|
|
"multiple_connections": "allow",
|
|
}
|
|
|
|
def render(self) -> str:
|
|
tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
|
|
with open(tmpl_path) as fp:
|
|
tmpl = jinja2.Template(fp.read())
|
|
|
|
return tmpl.render(**self.substitutions)
|
|
|
|
|
|
class StreamConf:
|
|
def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
|
|
self.child_conf = child_conf
|
|
self.parent_conf = parent_conf
|
|
|
|
def render(self) -> str:
|
|
child_section = self.child_conf.render() if self.child_conf else ""
|
|
parent_section = self.parent_conf.render() if self.parent_conf else ""
|
|
return "\n".join([child_section, parent_section])
|
|
|
|
|
|
class AgentContext:
|
|
def __init__(
|
|
self,
|
|
client: dagger.Client,
|
|
platform: dagger.Platform,
|
|
distro: Distribution,
|
|
installer: NetdataInstaller,
|
|
endpoint: Endpoint,
|
|
api_key: uuid.UUID,
|
|
allow_children: bool,
|
|
):
|
|
self.client = client
|
|
self.platform = platform
|
|
self.distro = distro
|
|
self.installer = installer
|
|
self.endpoint = endpoint
|
|
self.api_key = api_key
|
|
self.allow_children = allow_children
|
|
|
|
self.parent_contexts = []
|
|
|
|
self.built_distro = False
|
|
self.built_agent = False
|
|
|
|
def add_parent(self, parent_context: "AgentContext"):
|
|
self.parent_contexts.append(parent_context)
|
|
|
|
def build_container(self) -> dagger.Container:
|
|
ctr = self.distro.build(self.client, self.platform)
|
|
ctr = self.installer.install(self.client, ctr)
|
|
|
|
if len(self.parent_contexts) == 0 and not self.allow_children:
|
|
return ctr.with_exposed_port(self.endpoint.port)
|
|
|
|
destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
|
|
child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)
|
|
|
|
parent_stream_conf = None
|
|
if self.allow_children:
|
|
parent_stream_conf = ParentStreamConf(self.installer, self.api_key)
|
|
|
|
stream_conf = StreamConf(child_stream_conf, parent_stream_conf)
|
|
|
|
# write the stream conf to localhost and cp it in the container
|
|
host_stream_conf_path = pathlib.Path(
|
|
f"/tmp/{self.endpoint.hostname}_stream.conf"
|
|
)
|
|
with open(host_stream_conf_path, "w") as fp:
|
|
fp.write(stream_conf.render())
|
|
|
|
ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"
|
|
|
|
ctr = ctr.with_file(
|
|
ctr_stream_conf_path.as_posix(),
|
|
self.client.host().file(host_stream_conf_path.as_posix()),
|
|
)
|
|
|
|
ctr = ctr.with_exposed_port(self.endpoint.port)
|
|
|
|
return ctr
|