Setup sentry-native SDK. (#16798)

* Setup sentry-native SDK.

* Integrate Sentry into our CI

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* minor fix

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Include sentry field to the build matrix

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Update the bundle_sentry flag for all the distros

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* more changes

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* NON mergeable change

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* .

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Enable fetch content.

* Abort in profile plugin

* Update plugin_profile.cc

* NON MERGABLE COMMIT, just for testing purposes

* NON MERGEABLE CHANGE, jsut for testing purposes

* Bump

* Use breakpad backend

* Multiple changes

- Make DSN variable that we read from the CI from the CI
- Upload debug symbols
- Fix packaging workflow; include new env vars & fix shecllchecks

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Modify sentry dif command

* fix merge conf

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Fix merge conflict

* Undo file prefix map.

* Fix typo

* Cleanup stuff.

* Add 256-checksum

* Renable ML for debs

* Finalize CI changes

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Update rules

* final touches

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* merge the two if, no point to have them sep

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Update contrib/debian/rules

Co-authored-by: Tasos Katsoulas <12612986+tkatsoulas@users.noreply.github.com>

* Update contrib/debian/rules

Co-authored-by: Tasos Katsoulas <12612986+tkatsoulas@users.noreply.github.com>

* Add license

* Enable sentry for debian 12.

* Update .github/workflows/packaging.yml

Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com>

---------

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>
Co-authored-by: Tasos Katsoulas <tasos@netdata.cloud>
Co-authored-by: Tasos Katsoulas <12612986+tkatsoulas@users.noreply.github.com>
Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com>

.github/data/distros.yml

@@ -34,6 +34,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: true
+    bundle_sentry: false
   - <<: *alpine
     version: "3.18"
     support_type: Core
@@ -55,6 +56,7 @@ include:
     support_type: Intermediate
     notes: ''
     eol_check: false
+    bundle_sentry: false
     env_prep: |
       pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi
     test:
@@ -66,6 +68,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: 'amazon-linux'
+    bundle_sentry: false
     packages: &amzn_packages
       type: rpm
       repo_distro: amazonlinux/2
@@ -89,6 +92,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: false
+    bundle_sentry: false
     packages:
       type: rpm
       repo_distro: el/7
@@ -110,6 +114,7 @@ include:
     jsonc_removal: |
       dnf remove -y json-c-devel
     eol_check: true
+    bundle_sentry: false
     packages: &cs_packages
       type: rpm
       repo_distro: el/c9s
@@ -132,6 +137,7 @@ include:
     notes: ''
     base_image: debian:bookworm
     eol_check: true
+    bundle_sentry: true
     env_prep: |
       apt-get update
     jsonc_removal: |
@@ -149,6 +155,7 @@ include:
   - <<: *debian
     version: "11"
     base_image: debian:bullseye
+    bundle_sentry: false
     packages:
       <<: *debian_packages
       repo_distro: debian/bullseye
@@ -157,6 +164,7 @@ include:
   - <<: *debian
     version: "10"
     base_image: debian:buster
+    bundle_sentry: false
     packages:
       <<: *debian_packages
       repo_distro: debian/buster
@@ -169,6 +177,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: true
+    bundle_sentry: false
     jsonc_removal: |
       dnf remove -y json-c-devel
     packages: &fedora_packages
@@ -193,6 +202,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: true
+    bundle_sentry: false
     base_image: opensuse/leap:15.5
     jsonc_removal: |
       zypper rm -y libjson-c-devel
@@ -211,6 +221,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: true
+    bundle_sentry: false
     jsonc_removal: |
       dnf remove -y json-c-devel
     packages: &oracle_packages
@@ -235,6 +246,7 @@ include:
     jsonc_removal: |
       dnf remove -y json-c-devel
     eol_check: true
+    bundle_sentry: false
     packages: &rocky_packages
       type: rpm
       repo_distro: el/9
@@ -263,6 +275,7 @@ include:
     support_type: Core
     notes: ''
     eol_check: true
+    bundle_sentry: false
     env_prep: |
       rm -f /etc/apt/apt.conf.d/docker && apt-get update
     jsonc_removal: |
@@ -289,11 +302,13 @@ include:
 legacy: # Info for platforms we used to support and still need to handle packages for
   - <<: *fedora
     version: "37"
+    bundle_sentry: false
     packages:
       <<: *fedora_packages
       repo_distro: fedora/37
   - <<: *opensuse
     version: "15.4"
+    bundle_sentry: false
     packages:
       <<: *opensuse_packages
       repo_distro: opensuse/15.4

.github/scripts/gen-matrix-packaging.py

@@ -28,6 +28,7 @@ for i, v in enumerate(data['include']):
                     'format': data['include'][i]['packages']['type'],
                     'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]),
                     'platform': data['platform_map'][arch],
+                    'bundle_sentry': data['include'][i]['bundle_sentry'],
                     'arch': arch
                 })
 

.github/workflows/packaging.yml

@@ -205,6 +205,18 @@ jobs:
         with:
           fetch-depth: 0 # We need full history for versioning
           submodules: recursive
+      - name: Set Sentry telemetry env vars
+        id: set-telemetry-env-vars
+        run: |
+          if [ "${{ github.repository }}" = 'netdata/netdata' ] && \
+             [ "${{ matrix.bundle_sentry }}" = 'true' ] && \
+             [ "${{ github.event_name }}" = 'workflow_dispatch' ]; then
+              echo "RELEASE_PIPELINE=Production" >> "${GITHUB_ENV}"
+              echo "UPLOAD_SENTRY=true" >> "${GITHUB_ENV}"
+          else
+              echo "RELEASE_PIPELINE=Unknown" >> "${GITHUB_ENV}"
+              echo "UPLOAD_SENTRY=false" >> "${GITHUB_ENV}"
+          fi
       - name: Setup QEMU
         id: qemu
         if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' && needs.file-check.outputs.run == 'true'
@@ -233,6 +245,9 @@ jobs:
         shell: bash
         run: |
           docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \
+                    -e ENABLE_SENTRY=${{ matrix.bundle_sentry }} -e RELEASE_PIPELINE=${{ env.RELEASE_PIPELINE }} \
+                    -e BUILD_DESTINATION=${{ matrix.distro }}${{ matrix.version }}_${{ matrix.arch }} -e UPLOAD_SENTRY=${{ env.UPLOAD_SENTRY }} \
+                    -e SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_CLI_TOKEN }} -e SENTRY_DSN=${{ secrets.SENTRY_DSN }} \
                      --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1
       - name: Save Packages
         id: artifacts

CMakeLists.txt

@@ -115,8 +115,27 @@ option(ENABLE_BUNDLED_PROTOBUF "enable bundled protobuf" False)
 
 option(ENABLE_LOGS_MANAGEMENT_TESTS "enable logs management tests" True)
 
+option(ENABLE_SENTRY "enable sentry" False)
 option(ENABLE_WEBRTC "enable webrtc" False)
 
+if(ENABLE_SENTRY)
+        include(FetchContent)
+
+        # ignore debhelper
+        set(FETCHCONTENT_FULLY_DISCONNECTED Off)
+
+        set(SENTRY_VERSION 0.6.6)
+        set(SENTRY_BACKEND "breakpad")
+        set(SENTRY_BUILD_SHARED_LIBS OFF)
+
+        FetchContent_Declare(
+                sentry
+                URL https://github.com/getsentry/sentry-native/releases/download/${SENTRY_VERSION}/sentry-native.zip
+                URL_HASH SHA256=7a98467c0b2571380a3afc5e681cb13aa406a709529be12d74610b0015ccde0c
+        )
+        FetchContent_MakeAvailable(sentry)
+endif()
+
 if(ENABLE_WEBRTC)
         include(FetchContent)
 
@@ -1305,6 +1324,12 @@ if(LINUX)
                 ${PROC_PLUGIN_FILES}
                 ${TC_PLUGIN_FILES}
         )
+
+        if(ENABLE_SENTRY)
+            list(APPEND NETDATA_FILES
+                    daemon/sentry-native/sentry-native.c
+                    daemon/sentry-native/sentry-native.h)
+        endif()
 elseif(MACOS)
         list(APPEND NETDATA_FILES
                 daemon/static_threads_macos.c
@@ -2050,6 +2075,7 @@ target_link_libraries(netdata PRIVATE
         "$<$<BOOL:${ENABLE_EXPORTER_MONGODB}>:${MONGOC_LIBRARIES}>"
         "$<$<BOOL:${ENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE}>:${SNAPPY_LIBRARIES}>"
         "$<$<BOOL:${MACOS}>:${IOKIT};${FOUNDATION}>"
+        "$<$<BOOL:${ENABLE_SENTRY}>:sentry>"
         "$<$<BOOL:${ENABLE_WEBRTC}>:LibDataChannel::LibDataChannelStatic>"
 )
 

config.cmake.h.in

@@ -110,6 +110,13 @@
 #cmakedefine ENABLE_PLUGIN_CGROUP_NETWORK
 #cmakedefine ENABLE_PLUGIN_EBPF
 
+// enabled sentry
+
+#cmakedefine ENABLE_SENTRY
+#cmakedefine NETDATA_SENTRY_ENVIRONMENT "@NETDATA_SENTRY_ENVIRONMENT@"
+#cmakedefine NETDATA_SENTRY_RELEASE "@NETDATA_SENTRY_RELEASE@"
+#cmakedefine NETDATA_SENTRY_DIST "@NETDATA_SENTRY_DIST@"
+#cmakedefine NETDATA_SENTRY_DSN "@NETDATA_SENTRY_DSN@"
 // enabled bundling
 
 #cmakedefine ENABLE_BUNDLED_JSONC

contrib/debian/rules

@@ -2,6 +2,7 @@
 
 # Find the arch we are building for, as this determines
 # the location of plugins in /usr/lib
+SRC_DIR = /usr/src/netdata
 TOP = $(CURDIR)/debian/netdata
 TEMPTOP = $(CURDIR)/debian/tmp
 BUILDDIR = $(CURDIR)/debian/build
@@ -37,6 +38,20 @@ HAVE_XENSTAT = 1
 XENSTAT_CONFIG = -DENABLE_PLUGIN_XENSTAT=On
 endif
 
+ifeq ($(ENABLE_SENTRY),true)
+    RELEASE_PIPELINE ?= Unknown
+    VERSION ?= Unknown
+    BUILD_DESTINATION ?= Unknown
+    SENTRY_CONFIG := -DENABLE_SENTRY=On \
+        -DNETDATA_SENTRY_ENVIRONMENT=$(RELEASE_PIPELINE) \
+        -DNETDATA_SENTRY_RELEASE=$(VERSION) \
+        -DNETDATA_SENTRY_DIST=$(BUILD_DESTINATION) \
+        -DNETDATA_SENTRY_DSN=$(SENTRY_DSN)
+else
+    SENTRY_CONFIG := -DENABLE_SENTRY=Off
+endif
+
+
 %:
 	dh $@ --builddirectory=$(BUILDDIR)
 
@@ -56,6 +71,7 @@ override_dh_auto_configure:
 	dh_auto_configure -- -G Ninja \
 		-DCMAKE_INSTALL_PREFIX=/ \
 		-DWEB_DIR=/var/lib/netdata/www \
+		-DCMAKE_C_FLAGS='-ffile-prefix-map=${SRC_DIR}=${SRC_DIR}' \
 		-DENABLE_ACLK=On \
 		-DENABLE_CLOUD=On \
 		-DENABLE_DBENGINE=On \
@@ -77,11 +93,18 @@ override_dh_auto_configure:
 		-DENABLE_EXPORTER_MONGODB=On \
 		-DENABLE_BUNDLED_PROTOBUF=Off \
 		-DENABLE_BUNDLED_JSONC=Off \
+		${SENTRY_CONFIG} \
 		-DENABLE_BUNDLED_YAML=Off
 
 override_dh_auto_test:
 	true
 
+override_dh_strip:
+	if [ "${ENABLE_SENTRY}" = "true" ] && [ "${UPLOAD_SENTRY}" = "true" ]; then \
+		sentry-cli debug-files upload -o netdata-inc -p netdata-agent --force-foreground --log-level=debug --wait --include-sources /usr/src/netdata/debian/netdata/usr/sbin/netdata; \
+	fi
+	dh_strip
+
 override_dh_install:
 	cp -v $(BASE_CONFIG) debian/netdata.conf
 

daemon/main.c

@@ -6,6 +6,10 @@
 
 #include "database/engine/page_test.h"
 
+#ifdef ENABLE_SENTRY
+#include "sentry-native/sentry-native.h"
+#endif
+
 #if defined(ENV32BIT)
 #warning COMPILING 32BIT NETDATA
 #endif
@@ -510,6 +514,10 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 
     delta_shutdown_time("exit");
 
+#ifdef ENABLE_SENTRY
+    sentry_native_fini();
+#endif
+
     usec_t ended_ut = now_monotonic_usec();
     netdata_log_info("NETDATA SHUTDOWN: completed in %llu ms - netdata is now exiting - bye bye...", (ended_ut - started_ut) / USEC_PER_MS);
     exit(ret);
@@ -1890,7 +1898,6 @@ int main(int argc, char **argv) {
         for_each_open_fd(OPEN_FD_ACTION_CLOSE, OPEN_FD_EXCLUDE_STDIN | OPEN_FD_EXCLUDE_STDOUT | OPEN_FD_EXCLUDE_STDERR);
     }
 
-
     if(!config_loaded) {
         load_netdata_conf(NULL, 0, &user);
         load_cloud_conf(0);
@@ -2099,6 +2106,11 @@ int main(int argc, char **argv) {
     if(become_daemon(dont_fork, user) == -1)
         fatal("Cannot daemonize myself.");
 
+    // init sentry
+#ifdef ENABLE_SENTRY
+        sentry_native_init();
+#endif
+
     // The "HOME" env var points to the root's home dir because Netdata starts as root. Can't use "HOME".
     struct passwd *pw = getpwuid(getuid());
     if (config_exists(CONFIG_SECTION_DIRECTORIES, "home") || !pw || !pw->pw_dir) {

daemon/sentry-native/sentry-native.c

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sentry-native.h"
+#include "daemon/common.h"
+
+#include "sentry.h"
+
+static bool sentry_telemetry_disabled(void)
+{
+    char path[FILENAME_MAX + 1];
+    sprintf(path, "%s/%s", netdata_configured_user_config_dir, ".opt-out-from-anonymous-statistics");
+
+    struct stat buffer;
+    bool opt_out_file_exists = (stat(path, &buffer) == 0);
+
+    if (opt_out_file_exists)
+        return true;
+
+    return getenv("DISABLE_TELEMETRY") != NULL;
+}
+
+void sentry_native_init(void)
+{
+    if (sentry_telemetry_disabled())
+        return;
+
+    // path where sentry should save stuff
+    char path[FILENAME_MAX];
+    snprintfz(path, FILENAME_MAX - 1, "%s/%s", netdata_configured_cache_dir, ".sentry-native");
+
+    sentry_options_t *options = sentry_options_new();
+    sentry_options_set_dsn(options, NETDATA_SENTRY_DSN);
+    sentry_options_set_database_path(options, path);
+    sentry_options_set_auto_session_tracking(options, false);
+    sentry_options_set_environment(options, NETDATA_SENTRY_ENVIRONMENT);
+    sentry_options_set_release(options, NETDATA_SENTRY_RELEASE);
+    sentry_options_set_dist(options, NETDATA_SENTRY_DIST);
+#ifdef NETDATA_INTERNAL_CHECKS
+    sentry_options_set_debug(options, 1);
+#endif
+
+    sentry_init(options);
+}
+
+void sentry_native_fini(void)
+{
+    if (sentry_telemetry_disabled())
+        return;
+
+    sentry_close();
+}

daemon/sentry-native/sentry-native.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef SENTRY_NATIVE_H
+#define SENTRY_NATIVE_H
+
+void sentry_native_init(void);
+void sentry_native_fini(void);
+
+#endif /* SENTRY_NATIVE_H */

integrations/schemas/distros.json

@@ -77,6 +77,19 @@
             }
           ]
         },
+        "bundle_sentry": {
+          "description": "Set to true to build an Agent with Sentry telemetry from the CI.",
+          "oneOf": [
+            {
+              "type": "boolean",
+              "default": false
+            },
+            {
+              "type": "string",
+              "pattern": "^[a-z][a-z0-9._-]*$"
+            }
+          ]
+        },
         "base_image": {
           "type": "string",
           "description": "A string specifying the Docker image to be used for testing this platform.",
@@ -146,7 +159,8 @@
         "distro",
         "version",
         "support_type",
-        "notes"
+        "notes", 
+        "bundle_sentry"
       ]
     }
   }