annotate logs with stack trace when libunwind is available (#19334)

* annotate logs with stack trace when libunwind is available

* Update CMakeLists.txt

Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com>

* fix error when libunwind is not available

---------

Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com>

CMakeLists.txt

@@ -957,6 +957,7 @@ set(LIBNETDATA_FILES
         src/libnetdata/locks/benchmark.h
         src/libnetdata/locks/benchmark-rw.c
         src/libnetdata/locks/benchmark-rw.h
+        src/libnetdata/log/nd_log-libunwind.c
 )
 
 set(LIBH2O_FILES
@@ -2092,6 +2093,25 @@ netdata_add_jsonc_to_target(libnetdata)
 
 netdata_add_libyaml_to_target(libnetdata)
 
+# libunwind
+pkg_check_modules(LIBUNWIND libunwind IMPORTED_TARGET)
+if(TARGET PkgConfig::LIBUNWIND)
+    set(HAVE_LIBUNWIND On)
+
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(amd64)")
+        target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-x86_64)
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
+        target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-aarch64)
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
+        target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-arm)
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc|ppc")
+        target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-ppc64)
+    else()
+        message(WARNING "Unknown architecture ${CMAKE_SYSTEM_PROCESSOR} for libunwind. Stack traces may not work.")
+        target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND)
+    endif()
+endif()
+
 # zlib
 if(OS_MACOS)
         find_package(ZLIB REQUIRED)

packaging/cmake/config.cmake.h.in

@@ -75,6 +75,7 @@
 #cmakedefine HAVE_GETRANDOM
 #cmakedefine HAVE_SYSINFO
 
+#cmakedefine HAVE_LIBUNWIND
 #cmakedefine HAVE_BACKTRACE
 #cmakedefine HAVE_CLOSE_RANGE
 #cmakedefine HAVE_SCHED_GETSCHEDULER

src/libnetdata/log/nd_log-common.h

@@ -119,11 +119,13 @@ typedef enum __attribute__((__packed__)) {
     NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62,
     // NDF_ALERT_FLAGS,
 
+    NDF_STACK_TRACE = 63,                           // stack trace of the thread logging
+
     // put new items here
     // leave the request URL and the message last
 
-    NDF_REQUEST = 63,                               // the request we are currently working on
-    NDF_MESSAGE = 64,                               // the log message, if any
+    NDF_REQUEST = 64,                               // the request we are currently working on
+    NDF_MESSAGE = 65,                               // the log message, if any
 
     // terminator
     _NDF_MAX,

src/libnetdata/log/nd_log-internals.c

@@ -707,6 +707,12 @@ __thread struct log_field thread_log_fields[_NDF_MAX] = {
         .logfmt = "alert_notification_timestamp",
         .annotator = timestamp_usec_annotator,
     },
+    [NDF_STACK_TRACE] = {
+        .journal = "ND_STACK_TRACE",
+        .eventlog = "StackTrace",
+        .logfmt = NULL,
+        .annotator = stack_trace_annotator,
+    },
 
     // put new items here
     // leave the request URL and the message last

src/libnetdata/log/nd_log-internals.h

@@ -195,6 +195,7 @@ struct log_field;
 const char *errno_annotator(struct log_field *lf);
 const char *priority_annotator(struct log_field *lf);
 const char *timestamp_usec_annotator(struct log_field *lf);
+const char *stack_trace_annotator(struct log_field *lf);
 
 #if defined(OS_WINDOWS)
 const char *winerror_annotator(struct log_field *lf);

src/libnetdata/log/nd_log-libunwind.c

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+#ifdef HAVE_LIBUNWIND
+#include <libunwind.h>
+
+const char *stack_trace_annotator(struct log_field *lf __maybe_unused) {
+    static __thread char stack[4096];
+    static __thread bool in_stack_trace = false;
+
+    // prevent recursion
+    if(in_stack_trace)
+        return "stack trace recursion detected";
+
+    in_stack_trace = true;
+
+    unw_cursor_t cursor;
+    unw_context_t context;
+    char *d = stack;
+    size_t frames = 0;
+
+    // Initialize context for current thread
+    unw_getcontext(&context);
+    unw_init_local(&cursor, &context);
+
+    // Skip first 3 frames (our annotator and the logging infrastructure)
+    unw_step(&cursor);
+    unw_step(&cursor);
+    unw_step(&cursor);
+
+    while (unw_step(&cursor) > 0) {
+        unw_word_t offset, pc;
+        char sym[256];
+
+        unw_get_reg(&cursor, UNW_REG_IP, &pc);
+        if (pc == 0)
+            break;
+
+        const char *name = sym;
+        if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) == 0) {
+            if(frames++)
+                d += snprintfz(d, sizeof(stack) - (d - stack), "\n");
+            d += snprintfz(d, sizeof(stack) - (d - stack), "%s+0x%lx", name, (unsigned long)offset);
+        }
+        else {
+            if(frames++)
+                d += snprintfz(d, sizeof(stack) - (d - stack), "\n");
+            d += snprintfz(d, sizeof(stack) - (d - stack), "<unknown>");
+        }
+    }
+
+    in_stack_trace = false;
+    return stack;
+}
+
+#else
+const char *stack_trace_annotator(struct log_field *lf __maybe_unused) {
+    return "libunwind not available";
+}
+#endif

src/libnetdata/log/nd_log.c

@@ -232,6 +232,9 @@ static void nd_logger(const char *file, const char *function, const unsigned lon
 
     // set the common fields that are automatically set by the logging subsystem
 
+    if(likely(!thread_log_fields[NDF_STACK_TRACE].entry.set))
+        thread_log_fields[NDF_STACK_TRACE].entry = ND_LOG_FIELD_U64(NDF_STACK_TRACE, 1);
+
     if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
         thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);