Integrate OpenTelemetry collector build into build system. (#19702)

* Fix Go version requirement detection to not have external deps.

Instead of relying on a UNIX-like environment with the `grep` and `cut`
commands, perform the required data extraction using CMake code. This
makes it more portable, but also should speed things up a tiny bit
because it doesn’t require invoking external commands.

* Preliminary integration of the new otel-collector into the build.

Adding `-DENABLE_PLUGIN_OTEL=On` to CMake options will enable building
the plugin and installing it.

Currently disabled by default, and does not include packaging
integration yet.

The plugin itself can be built independently of the primary build system
but using the same build infrastructure that is used to build it as
part of the regular build by using CMake in the src/go/otel-collector
directory.

* Minor cleanup.

* Fix build prefix.

* Restructure to not use a sub-project.

CMakeLists.txt

@@ -169,6 +169,7 @@ mark_as_advanced(ENABLE_DASHBOARD)
 
 # Data collection plugins
 option(ENABLE_PLUGIN_GO "Enable metric collectors written in Go" ${DEFAULT_FEATURE_STATE})
+option(ENABLE_PLUGIN_OTEL "Enable metrics collection via OpenTelemetry collector" False)
 option(ENABLE_PLUGIN_PYTHON "Enable metric collectors written in Python" ${DEFAULT_FEATURE_STATE})
 
 cmake_dependent_option(ENABLE_PLUGIN_APPS "Enable per-process resource usage monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX OR OS_FREEBSD OR OS_MACOS OR OS_WINDOWS" False)
@@ -303,7 +304,7 @@ if(ENABLE_JEMALLOC)
     endif()
 endif()
 
-if(ENABLE_PLUGIN_GO)
+if(ENABLE_PLUGIN_GO OR ENABLE_PLUGIN_OTEL)
     include(NetdataGoTools)
 
     find_min_go_version("${CMAKE_SOURCE_DIR}/src/go")
@@ -3053,6 +3054,18 @@ if(ENABLE_PLUGIN_GO)
             DESTINATION usr/libexec/netdata/plugins.d)
 endif()
 
+#
+# Handle OTel Collector plugin
+#
+
+if(ENABLE_PLUGIN_OTEL)
+  include(${CMAKE_SOURCE_DIR}/src/go/otel-collector/CMakeLists.txt)
+
+  install(PROGRAMS ${CMAKE_BINARY_DIR}/otel-collector/otelcol.plugin
+          COMPONENT plugin-otelcol
+          DESTINATION usr/libexec/netdata/plugins.d)
+endif()
+
 #
 # Generate config file
 #

packaging/cmake/Modules/NetdataGoTools.cmake

@@ -49,9 +49,6 @@ endmacro()
 # All files found will be checked for a `go` directive, and the
 # MIN_GO_VERSION variable will be set to the highest version
 # number found among these directives.
-#
-# Only works on UNIX-like systems, because it has to process the go.mod
-# files in ways that CMake can't do on it's own.
 function(find_min_go_version src_tree)
     message(STATUS "Determining minimum required version of Go for this build")
 
@@ -61,14 +58,10 @@ function(find_min_go_version src_tree)
 
     foreach(f IN ITEMS ${go_mod_files})
         message(VERBOSE "Checking Go version specified in ${f}")
-        execute_process(
+        file(STRINGS "${f}" match_line REGEX "^go .*$")
-            COMMAND grep -E "^go .*$" ${f}
-            COMMAND cut -f 2 -d " "
-            RESULT_VARIABLE version_check_result
-            OUTPUT_VARIABLE go_mod_version
-        )
 
-        if(version_check_result EQUAL 0)
+        if(match_line)
+            list(GET match_line 0 go_mod_version)
             string(REGEX MATCH "([0-9]+\\.[0-9]+(\\.[0-9]+)?)" go_mod_version "${go_mod_version}")
 
             if(go_mod_version VERSION_GREATER result)

src/go/otel-collector/CMakeLists.txt

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+function(_handle_otel)
+  if(CMAKE_BUILD_TYPE STREQUAL Debug)
+    set(DEBUG_BUILD True)
+  else()
+    set(DEBUG_BUILD False)
+  endif()
+
+  message(STATUS "Generating OpenTelemetry Collector Builder configuration")
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/release-config.yaml.in"
+                "${CMAKE_BINARY_DIR}/otel-build-config.yaml"
+                @ONLY)
+  message(STATUS "Generating OpenTelemetry Collector Builder configuration -- Done")
+
+  message(STATUS "Fetching OpenTelemetry Collector Builder")
+  set(OLD_GOBIN $ENV{GOBIN})
+  set(ENV{GOBIN} ${CMAKE_BINARY_DIR}/bin)
+  execute_process(
+    COMMAND ${GO_EXECUTABLE} install go.opentelemetry.io/collector/cmd/builder@latest
+    RESULT_VARIABLE otel_builder_install
+  )
+  set(ENV{GOBIN} ${OLD_GOBIN})
+
+  if(otel_builder_install)
+    message(FATAL_ERROR "Fetching OpenTelemetry Collector Builder --Failed")
+  else()
+    message(STATUS "Fetching OpenTelemetry Collector Builder -- Success")
+  endif()
+
+  set(DIRS "exporter/journaldexporter")
+  set(otelcol_deps "")
+
+  foreach(dir IN LISTS DIRS)
+    file(GLOB_RECURSE deps CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${dir}/*.go")
+    list(APPEND otelcol_deps "${deps}")
+    list(APPEND otelcol_deps
+      "${CMAKE_CURRENT_SOURCE_DIR}/${dir}/go.mod"
+      "${CMAKE_CURRENT_SOURCE_DIR}/${dir}/go.sum"
+    )
+  endforeach()
+
+  add_custom_command(
+    OUTPUT otel-collector/otelcol.plugin
+    COMMAND ${CMAKE_BINARY_DIR}/bin/builder --config=${CMAKE_BINARY_DIR}/otel-build-config.yaml
+    DEPENDS ${otelcol_deps}
+    COMMENT "Building otelcol.plugin"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    VERBATIM
+  )
+
+  add_custom_target(
+    plugin-otelcol ALL
+    DEPENDS otel-collector/otelcol.plugin
+  )
+endfunction()
+
+handle_otel()

src/go/otel-collector/release-config.yaml.in

@@ -0,0 +1,21 @@
+dist:
+  name: otelcol.plugin
+  module: github.com/netdata/netdata/otel-collector
+  description: OpenTelemetry Collector Distribution built for Netdata
+  output_path: @CMAKE_BINARY_DIR@/otel-collector
+  version: @NETDATA_VERSION_STRING@
+  go: @GO_EXECUTABLE@
+  debug_compilation: @DEBUG_BUILD@
+
+receivers:
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.120.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.120.0
+
+exporters:
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.120.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.120.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.120.0
+  - gomod: github.com/netdata/netdata/otel-collector/exporter/journaldexporter v0.0.0
+
+replaces:
+  - github.com/netdata/netdata/otel-collector/exporter/journaldexporter => @CMAKE_CURRENT_SOURCE_DIR@/exporter/journaldexporter