diff --git a/CMakeLists.txt b/CMakeLists.txt
index 679ec290f6..cd62090152 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -670,10 +670,10 @@ set(LIBNETDATA_FILES
src/libnetdata/libnetdata.h
src/libnetdata/locks/locks.c
src/libnetdata/locks/locks.h
- src/libnetdata/log/journal.c
- src/libnetdata/log/journal.h
- src/libnetdata/log/log.c
- src/libnetdata/log/log.h
+ src/libnetdata/log/systemd-journal-helpers.c
+ src/libnetdata/log/systemd-journal-helpers.h
+ src/libnetdata/log/nd_log.c
+ src/libnetdata/log/nd_log.h
src/libnetdata/os/os.c
src/libnetdata/os/os.h
src/libnetdata/simple_hashtable.h
@@ -822,6 +822,25 @@ set(LIBNETDATA_FILES
src/libnetdata/sanitizers/sanitizers-functions.h
src/libnetdata/sanitizers/sanitizers-pluginsd.c
src/libnetdata/sanitizers/sanitizers-pluginsd.h
+ src/libnetdata/log/nd_log-internals.c
+ src/libnetdata/log/nd_log-internals.h
+ src/libnetdata/log/nd_log_limit.c
+ src/libnetdata/log/nd_log_limit.h
+ src/libnetdata/log/nd_log-config.c
+ src/libnetdata/log/nd_log-init.c
+ src/libnetdata/log/nd_log-to-syslog.c
+ src/libnetdata/log/nd_log-to-systemd-journal.c
+ src/libnetdata/log/nd_log-annotators.c
+ src/libnetdata/log/nd_log-field-formatters.c
+ src/libnetdata/log/nd_log-format-logfmt.c
+ src/libnetdata/log/nd_log-format-json.c
+ src/libnetdata/log/nd_log-to-file.c
+ src/libnetdata/log/nd_log-to-windows-events.c
+ src/libnetdata/string/utf8.c
+ src/libnetdata/spawn_server/log-forwarder.c
+ src/libnetdata/spawn_server/log-forwarder.h
+ src/libnetdata/log/nd_log-common.h
+ src/libnetdata/log/nd_log-to-windows-common.h
)
if(ENABLE_PLUGIN_EBPF)
@@ -1439,12 +1458,13 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
src/collectors/windows-events.plugin/windows-events-unicode.h
src/collectors/windows-events.plugin/windows-events-xml.c
src/collectors/windows-events.plugin/windows-events-xml.h
- src/collectors/windows-events.plugin/windows-events-publishers.c
- src/collectors/windows-events.plugin/windows-events-publishers.h
+ src/collectors/windows-events.plugin/windows-events-providers.c
+ src/collectors/windows-events.plugin/windows-events-providers.h
src/collectors/windows-events.plugin/windows-events-fields-cache.c
src/collectors/windows-events.plugin/windows-events-fields-cache.h
src/collectors/windows-events.plugin/windows-events-query-builder.c
src/collectors/windows-events.plugin/windows-events-query-builder.h
+ src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
)
set(WINDOWS_PLUGIN_FILES
@@ -1746,10 +1766,90 @@ target_include_directories(libnetdata BEFORE PUBLIC ${CONFIG_H_DIR} ${CMAKE_SOUR
target_link_libraries(libnetdata PUBLIC
"$<$<NOT:$<BOOL:${HAVE_BUILTIN_ATOMICS}>>:atomic>"
"$<$<OR:$<BOOL:${OS_LINUX}>,$<BOOL:${OS_FREEBSD}>>:pthread;rt>"
- "$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt>"
+ "$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt;wevtapi>"
"$<$<BOOL:${LINK_LIBM}>:m>"
"${SYSTEMD_LDFLAGS}")
+if(OS_WINDOWS)
+ set(HAVE_ETW True)
+ set(HAVE_WEL True)
+
+ # Output the results for debugging purposes
+ message(STATUS "Have Event Tracing for Windows (ETW): ${HAVE_ETW}")
+ message(STATUS "Have Windows Event Log (WEL): ${HAVE_WEL}")
+
+ if(HAVE_WEL OR HAVE_ETW)
+ # Define the source and generated file paths
+ set(WEVT_GEN_SRC_H_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/nd_log-to-windows-common.h")
+ set(WEVT_GEN_SRC_C_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_mc_generate.c")
+ set(WEVT_GEN_BIN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_mc_generate")
+
+ set(WEVT_BUILD_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_compile.sh")
+
+ set(WEVT_MC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.mc")
+ set(WEVT_MAN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.xml")
+ set(WEVT_RC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.rc")
+ set(WEVT_MC_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.h")
+ set(WEVT_MAN_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.h")
+ set(WEVT_RES_OBJECT "${CMAKE_BINARY_DIR}/wevt_netdata_res.o")
+
+ set(WEVT_DLL_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.dll")
+ set(WEVT_ETW_INSTALL_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_install.bat")
+
+ # we compile ${WEVT_GEN_BIN_FILE}, which generates the manifest, the .mc,
+ # and the headers required for compiling libnetdata/logs
+
+ if(HAVE_ETW)
+ # ETW method also supports WEL
+ # but it requires Microsoft tools mc, rc, and link
+ add_custom_command(
+ OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_MAN_H_FILE}" "${WEVT_DLL_FILE}"
+ COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
+ COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
+ COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
+ COMMAND "${WEVT_BUILD_SCRIPT}" "${CMAKE_SOURCE_DIR}/src/libnetdata/log" "${CMAKE_BINARY_DIR}"
+ DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
+ COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
+ )
+ else()
+ # WEL method can be built with windmc, windres and the normal linker
+ add_custom_command(
+ OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_DLL_FILE}"
+ COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
+ COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
+ COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
+ COMMAND windmc -r "${CMAKE_BINARY_DIR}" -h "${CMAKE_BINARY_DIR}" ${WEVT_MC_FILE}
+ COMMAND echo "1 2004" "wevt_netdata_manifest.xml" >> "${WEVT_RC_FILE}"
+ COMMAND windres ${WEVT_RC_FILE} -o ${WEVT_RES_OBJECT}
+ COMMAND ${CMAKE_LINKER} -dll --entry 0 -nostdlib -o ${WEVT_DLL_FILE} ${WEVT_RES_OBJECT}
+ DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
+ COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
+ )
+ endif()
+
+ # Create a custom target for the DLL
+ add_custom_target(wevt_netdata ALL DEPENDS ${WEVT_DLL_FILE})
+
+ set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
+ OBJECT_DEPENDS "${WEVT_MC_H_FILE}")
+
+ if(HAVE_ETW)
+ set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
+ OBJECT_DEPENDS "${WEVT_MAN_H_FILE}")
+
+ install(FILES "${WEVT_DLL_FILE}" "${WEVT_MAN_FILE}" "${WEVT_ETW_INSTALL_SCRIPT}"
+ COMPONENT wevt_netdata_dll
+ DESTINATION "${BINDIR}")
+ else()
+ # do not install the manifest in this case
+ # the nsi installer will skip registering the ETW publisher
+ install(FILES "${WEVT_DLL_FILE}"
+ COMPONENT wevt_netdata_dll
+ DESTINATION "${BINDIR}")
+ endif()
+ endif()
+endif()
+
# ebpf
if(ENABLE_PLUGIN_EBPF)
netdata_add_libbpf_to_target(libnetdata)
@@ -1925,8 +2025,7 @@ if(ENABLE_PLUGIN_APPS)
add_executable(apps.plugin ${APPS_PLUGIN_FILES})
target_link_libraries(apps.plugin libnetdata ${CAP_LIBRARIES}
- "$<$<BOOL:${OS_WINDOWS}>:Version>"
- "$<$<BOOL:${OS_WINDOWS}>:ntdll>")
+ "$<$<BOOL:${OS_WINDOWS}>:Version;ntdll>")
target_include_directories(apps.plugin PRIVATE ${CAP_INCLUDE_DIRS})
target_compile_options(apps.plugin PRIVATE ${CAP_CFLAGS_OTHER})
diff --git a/packaging/cmake/config.cmake.h.in b/packaging/cmake/config.cmake.h.in
index 603a57baba..d322826300 100644
--- a/packaging/cmake/config.cmake.h.in
+++ b/packaging/cmake/config.cmake.h.in
@@ -169,6 +169,8 @@
#cmakedefine HAVE_LIBYAML
#cmakedefine HAVE_LIBMNL
+#cmakedefine HAVE_WEL
+#cmakedefine HAVE_ETW
#cmakedefine RUN_UNDER_CLION
// /* Enable GNU extensions on systems that have them. */
diff --git a/packaging/utils/compile-on-windows.sh b/packaging/utils/compile-and-run-windows.sh
similarity index 94%
rename from packaging/utils/compile-on-windows.sh
rename to packaging/utils/compile-and-run-windows.sh
index 8867d10323..2d540eee94 100644
--- a/packaging/utils/compile-on-windows.sh
+++ b/packaging/utils/compile-and-run-windows.sh
@@ -70,15 +70,20 @@ then
${NULL}
fi
+ninja -v -C "${build}" || ninja -v -C "${build}" -j 1
+
+echo "Stopping service Netdata"
+sc stop "Netdata" || echo "Failed"
+
ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
+# register the event log publisher
+cmd.exe //c "$(cygpath -w -a "/opt/netdata/usr/bin/wevt_netdata_install.bat")"
+
#echo
#echo "Compile with:"
#echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
-echo "Stopping service Netdata"
-sc stop "Netdata" || echo "Failed"
-
echo "starting netdata..."
# enable JIT debug with gdb
export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"
diff --git a/packaging/windows/clion-msys-msys-environment.bat b/packaging/windows/clion-msys-msys-environment.bat
index 9f0c095d37..16934951d5 100644
--- a/packaging/windows/clion-msys-msys-environment.bat
+++ b/packaging/windows/clion-msys-msys-environment.bat
@@ -13,7 +13,9 @@ set MSYSTEM=MSYS
:: go exists only mingw64 / ucrt64 / etc, not under msys profile
set GOROOT=C:\msys64\mingw64
-set PATH="%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
+set "PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
+set "PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
+set "PATH=%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
::set PKG_CONFIG_EXECUTABLE=C:\msys64\mingw64\bin\pkg-config.exe
::set CMAKE_C_COMPILER=C:\msys64\mingw64\bin\gcc.exe
::set CMAKE_CC_COMPILER=C:\msys64\mingw64\bin\g++.exe
diff --git a/packaging/windows/find-sdk-path.sh b/packaging/windows/find-sdk-path.sh
new file mode 100644
index 0000000000..fb0eb600b3
--- /dev/null
+++ b/packaging/windows/find-sdk-path.sh
@@ -0,0 +1,217 @@
+#!/bin/bash
+
+# Function to output the path in Windows format (convert from MSYS2/Unix format using cygpath)
+convert_to_windows_format() {
+ cygpath -w -a "$1"
+}
+
+# Function to display help message
+display_help() {
+ echo "Usage: $0 [-s|--sdk] [-v|--visualstudio] [-w|--windows] [--help]"
+ echo
+ echo "Options:"
+ echo " -s, --sdk Search for tools in the Windows SDK."
+ echo " -v, --visualstudio Search for tools in Visual Studio."
+ echo " -w, --windows Output the path in Windows format (using cygpath)."
+ echo " --help Display this help message."
+ exit 0
+}
+
+# Function to find tools in the Windows SDK
+find_sdk_tools() {
+ sdk_base_path="/c/Program Files (x86)/Windows Kits/10/bin"
+
+ if [ ! -d "$sdk_base_path" ]; then
+ echo "ERROR: SDK base path \"$sdk_base_path\" does not exist. No SDK installations found." >&2
+ echo "$system_root"
+ return 1
+ fi
+
+ echo "SDK base path exists: \"$sdk_base_path\"" >&2
+
+ # Find all SDK versions
+ sdk_versions=($(ls "$sdk_base_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
+ echo "Found SDK versions: ${sdk_versions[*]}" >&2
+
+ if [ ${#sdk_versions[@]} -eq 0 ]; then
+ echo "ERROR: No valid Windows SDK versions found in \"$sdk_base_path\"." >&2
+ echo "$system_root"
+ return 1
+ fi
+
+ # Sort versions and pick the latest
+ sorted_versions=$(printf '%s\n' "${sdk_versions[@]}" | sort -V)
+ latest_sdk_version=$(echo "$sorted_versions" | tail -n 1)
+ sdk_tool_path="$sdk_base_path/$latest_sdk_version/x64"
+
+ echo "Latest SDK version: \"$latest_sdk_version\"" >&2
+
+ if [ ! -d "$sdk_tool_path" ]; then
+ echo "ERROR: Tool path \"$sdk_tool_path\" does not exist." >&2
+ echo "$system_root"
+ return 1
+ fi
+
+ # Check if required tools exist
+ tools=("mc.exe" "rc.exe")
+ for tool in "${tools[@]}"; do
+ if [ ! -f "$sdk_tool_path/$tool" ]; then
+ echo "ERROR: $tool not found in \"$sdk_tool_path\"" >&2
+ echo "$system_root"
+ return 1
+ else
+ echo "$tool found in \"$sdk_tool_path\"" >&2
+ fi
+ done
+
+ echo >&2
+ echo "DONE: All required tools found in \"$sdk_tool_path\"" >&2
+ echo >&2
+
+ echo "$sdk_tool_path"
+}
+
+# Function to find tools in Visual Studio
+find_visual_studio_tools() {
+ studio_base_path="/c/Program Files/Microsoft Visual Studio/2022"
+ echo "Checking for Visual Studio installations in: \"$studio_base_path\"" >&2
+
+ if [ ! -d "$studio_base_path" ]; then
+ echo "ERROR: Visual Studio base path \"$studio_base_path\" does not exist. No Visual Studio installations found." >&2
+ echo "$system_root"
+ return 1
+ fi
+
+ # Visual Studio editions we want to check
+ editions=("Enterprise" "Professional" "Community")
+ available_editions=()
+
+ # Loop through each edition and check for tools
+ for edition in "${editions[@]}"; do
+ edition_path="$studio_base_path/$edition/VC/Tools/MSVC"
+ if [ -d "$edition_path" ]; then
+ available_editions+=("$edition")
+ echo "Checking edition: $edition in $studio_base_path" >&2
+
+ # Find all MSVC versions and sort them
+ msvc_versions=($(ls "$edition_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
+ echo "Found MSVC versions in $edition: ${msvc_versions[*]}" >&2
+
+ if [ ${#msvc_versions[@]} -gt 0 ]; then
+ sorted_versions=$(printf '%s\n' "${msvc_versions[@]}" | sort -V)
+ latest_msvc_version=$(echo "${sorted_versions[@]}" | tail -n 1)
+ vs_tool_path="$edition_path/$latest_msvc_version/bin/Hostx64/x64"
+
+ echo "Latest MSVC version: \"$latest_msvc_version\" in $edition" >&2
+
+ if [ ! -d "$vs_tool_path" ]; then
+ echo "WARNING: Tool path \"$vs_tool_path\" does not exist." >&2
+ continue
+ fi
+
+ # Check if required tools exist
+ tools=("link.exe")
+ missing_tool=0
+
+ for tool in "${tools[@]}"; do
+ if [ ! -f "$vs_tool_path/$tool" ]; then
+ echo "WARNING: $tool not found in \"$vs_tool_path\" for $edition" >&2
+ missing_tool=1
+ else
+ echo "$tool found in \"$vs_tool_path\"" >&2
+ fi
+ done
+
+ if [ $missing_tool -eq 0 ]; then
+ echo >&2
+ echo "All required tools found in \"$vs_tool_path\"" >&2
+ echo >&2
+
+ echo "$vs_tool_path"
+ return 0
+ else
+ echo "WARNING: skipping edition '$edition', directory does not exist." >&2
+ fi
+ else
+ echo "WARNING: skipping edition '$edition', MSVC directory does not exist." >&2
+ fi
+ else
+ echo "WARNING: skipping edition '$edition', directory does not exist." >&2
+ fi
+ done
+
+ echo "ERROR: No valid Visual Studio editions found in \"$studio_base_path\"." >&2
+ echo "$system_root"
+ return 1
+}
+
+# Parse options using getopt
+TEMP=$(getopt -o svwh --long sdk,visualstudio,windows,help -- "$@")
+if [ $? != 0 ]; then
+ echo "ERROR: Invalid options provided." >&2
+ exit 1
+fi
+
+eval set -- "$TEMP"
+
+search_mode="sdk"
+windows_format=0
+system_root="/usr/bin"
+
+# Process getopt options
+while true; do
+ case "$1" in
+ -s|--sdk)
+ search_mode="sdk"
+ shift
+ ;;
+ -v|--visualstudio)
+ search_mode="visualstudio"
+ shift
+ ;;
+ -w|--windows)
+ system_root="%SYSTEMROOT%"
+ windows_format=1
+ shift
+ ;;
+ --help|-h)
+ display_help
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ echo "ERROR: Invalid option: $1" >&2
+ exit 1
+ ;;
+ esac
+done
+
+# Ensure that one of --sdk or --visualstudio is selected
+if [ -z "$search_mode" ]; then
+ echo "ERROR: You must specify either --sdk or --visualstudio." >&2
+ display_help
+fi
+
+# Determine which function to call based on the search mode
+if [ "$search_mode" = "sdk" ]; then
+ tool_path=$(find_sdk_tools)
+else
+ tool_path=$(find_visual_studio_tools)
+fi
+
+# If a valid path is found, output it
+if [ "$tool_path" != "$system_root" ]; then
+ if [ "$windows_format" -eq 1 ]; then
+ windows_tool_path=$(convert_to_windows_format "$tool_path")
+ echo "$windows_tool_path"
+ else
+ echo "$tool_path"
+ fi
+else
+ echo "$system_root"
+ exit 1
+fi
+
+exit 0
diff --git a/packaging/windows/installer.nsi b/packaging/windows/installer.nsi
index d57723ccce..b0796967a7 100644
--- a/packaging/windows/installer.nsi
+++ b/packaging/windows/installer.nsi
@@ -257,29 +257,97 @@ Function NetdataUninstallRegistry
end:
FunctionEnd
+Function InstallDLL
+ ; Check if certutil is available
+ nsExec::ExecToStack 'where certutil'
+ Pop $R0
+ StrCmp $R0 "" NoCertUtil FoundCertUtil
+
+ NoCertUtil:
+ DetailPrint "certutil not found, assuming files are different."
+ Goto CopyDLL
+
+ FoundCertUtil:
+ ; Calculate hash of the existing DLL
+ nsExec::ExecToStack 'certutil -hashfile "$SYSDIR\wevt_netdata.dll" MD5'
+ Pop $R0
+
+ ; Calculate hash of the new DLL
+ nsExec::ExecToStack 'certutil -hashfile "$INSTDIR\usr\bin\wevt_netdata.dll" MD5'
+ Pop $R1
+
+ StrCmp $R0 $R1 SetPermissions
+
+ CopyDLL:
+ ClearErrors
+ CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata.dll" "$SYSDIR"
+ IfErrors RetryPrompt SetPermissions
+
+ RetryPrompt:
+ MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata.dll probably because it is in use. Please close the Event Viewer (or other Event Log applications) and press Retry."
+ StrCmp $R0 IDRETRY CopyDLL
+ StrCmp $R0 IDCANCEL ExitInstall
+
+ Goto End
+
+ SetPermissions:
+ nsExec::ExecToLog 'icacls "$SYSDIR\wevt_netdata.dll" /grant "NT SERVICE\EventLog":R'
+ Goto End
+
+ ExitInstall:
+ Abort
+
+ End:
+FunctionEnd
+
+Function InstallManifest
+ IfFileExists "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" CopyManifest End
+
+ CopyManifest:
+ ClearErrors
+ CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" "$SYSDIR"
+ IfErrors RetryPrompt InstallManifest
+
+ RetryPrompt:
+ MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata_manifest.xml."
+ StrCmp $R0 IDRETRY CopyManifest
+ StrCmp $R0 IDCANCEL ExitInstall
+
+ InstallManifest:
+ nsExec::ExecToLog 'wevtutil im "$SYSDIR\wevt_netdata_manifest.xml" "/mf:$SYSDIR\wevt_netdata.dll" "/rf:$SYSDIR\wevt_netdata.dll"'
+ Goto End
+
+ ExitInstall:
+ Abort
+
+ End:
+FunctionEnd
+
Section "Install Netdata"
- SetOutPath $INSTDIR
- SetCompress off
+ SetOutPath $INSTDIR
+ SetCompress off
- File /r "C:\msys64\opt\netdata\*.*"
+ File /r "C:\msys64\opt\netdata\*.*"
- ClearErrors
+ ClearErrors
nsExec::ExecToLog '$SYSDIR\sc.exe create Netdata binPath= "$INSTDIR\usr\bin\netdata.exe" start= delayed-auto'
pop $0
${If} $0 != 0
- DetailPrint "Warning: Failed to create Netdata service."
+ DetailPrint "Warning: Failed to create Netdata service."
${EndIf}
- ClearErrors
+ ClearErrors
nsExec::ExecToLog '$SYSDIR\sc.exe description Netdata "Real-time system monitoring service"'
pop $0
${If} $0 != 0
- DetailPrint "Warning: Failed to add Netdata service description."
+ DetailPrint "Warning: Failed to add Netdata service description."
${EndIf}
WriteUninstaller "$INSTDIR\Uninstall.exe"
Call NetdataUninstallRegistry
+ Call InstallDLL
+ Call InstallManifest
StrLen $0 $cloudToken
StrLen $1 $cloudRooms
@@ -325,9 +393,24 @@ Section "Uninstall"
DetailPrint "Warning: Failed to delete Netdata service."
${EndIf}
- # https://nsis.sourceforge.io/Reference/RMDir
- RMDir /r /REBOOTOK "$INSTDIR"
+ ; Check if the manifest exists before uninstalling it
+ IfFileExists "$SYSDIR\wevt_netdata_manifest.xml" ManifestExistsForUninstall ManifestNotExistsForUninstall
+
+ManifestExistsForUninstall:
+ nsExec::ExecToLog 'wevtutil um "$SYSDIR\wevt_netdata_manifest.xml"'
+ pop $0
+ ${If} $0 != 0
+ DetailPrint "Warning: Failed to uninstall the event manifest."
+ ${EndIf}
+ Goto DoneUninstall
+
+ManifestNotExistsForUninstall:
+ DetailPrint "Manifest not found, skipping manifest uninstall."
+
+DoneUninstall:
+
+ ; Remove files
+ RMDir /r /REBOOTOK "$INSTDIR"
DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Netdata"
SectionEnd
-
diff --git a/src/collectors/apps.plugin/apps_os_windows.c b/src/collectors/apps.plugin/apps_os_windows.c
index f9098a4dd0..d2c30de76c 100644
--- a/src/collectors/apps.plugin/apps_os_windows.c
+++ b/src/collectors/apps.plugin/apps_os_windows.c
@@ -523,14 +523,10 @@ static char *wchar_to_utf8(WCHAR *s) {
static char *ansi_to_utf8(LPCSTR str) {
static __thread WCHAR unicode[PATH_MAX];
- static __thread int unicode_size = sizeof(unicode) / sizeof(*unicode);
// Step 1: Convert ANSI string (LPSTR) to wide string (UTF-16)
- int wideLength = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0);
- if (wideLength == 0 || wideLength > unicode_size)
- return NULL;
-
- MultiByteToWideChar(CP_ACP, 0, str, -1, unicode, wideLength);
+ size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1);
+ if (!count) return NULL;
return wchar_to_utf8(unicode);
}
diff --git a/src/collectors/apps.plugin/apps_pid.c b/src/collectors/apps.plugin/apps_pid.c
index 8d5666275c..949de92860 100644
--- a/src/collectors/apps.plugin/apps_pid.c
+++ b/src/collectors/apps.plugin/apps_pid.c
@@ -327,15 +327,11 @@ static bool is_filename(const char *s) {
(*s == '/' && s[1] == '/' && isalpha((uint8_t)s[2]) && s[3] == '/')) { // windows native "//x/"
WCHAR ws[FILENAME_MAX];
- int wlen = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0);
- if (wlen <= 0 || (size_t)wlen > sizeof(ws) / sizeof(*ws)) {
- return false; // Failed to convert UTF-8 to UTF-16
+ if(utf8_to_utf16(ws, _countof(ws), s, -1) > 0) {
+ DWORD attributes = GetFileAttributesW(ws);
+ if (attributes != INVALID_FILE_ATTRIBUTES)
+ return true;
}
-
- MultiByteToWideChar(CP_UTF8, 0, s, -1, ws, wlen);
- DWORD attributes = GetFileAttributesW(ws);
- if (attributes != INVALID_FILE_ATTRIBUTES)
- return true;
}
#endif
diff --git a/src/collectors/systemd-journal.plugin/systemd-journal.c b/src/collectors/systemd-journal.plugin/systemd-journal.c
index 56d37b807f..8d8055a8ff 100644
--- a/src/collectors/systemd-journal.plugin/systemd-journal.c
+++ b/src/collectors/systemd-journal.plugin/systemd-journal.c
@@ -85,6 +85,7 @@ struct lqs_extension {
#define LQS_SOURCE_TYPE SD_JOURNAL_FILE_SOURCE_TYPE
#define LQS_SOURCE_TYPE_ALL SDJF_ALL
#define LQS_SOURCE_TYPE_NONE SDJF_NONE
+#define LQS_PARAMETER_SOURCE_NAME "Journal Sources" // this is how it is shown to users
#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) get_internal_source_type(value)
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) available_journal_file_sources_to_json_array(wb)
#include "libnetdata/facets/logs_query_status.h"
diff --git a/src/collectors/windows-events.plugin/README.md b/src/collectors/windows-events.plugin/README.md
new file mode 100644
index 0000000000..8e05a79b6b
--- /dev/null
+++ b/src/collectors/windows-events.plugin/README.md
@@ -0,0 +1,289 @@
+# Windows Events plugin
+
+[KEY FEATURES](#key-features) | [EVENTS SOURCES](#events-sources) | [EVENT FIELDS](#event-fields) |
+[PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [PERFORMANCE](#query-performance) |
+[CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq)
+
+The Windows Events plugin by Netdata makes viewing, exploring and analyzing Windows Events simple and
+efficient.
+
+
+
+## Key features
+
+- Supports **Windows Event Logs (WEL)**.
+- Supports **Event Tracing for Windows (ETW)** and **TraceLogging (TL)**, when events are routed to Event Log.
+- Allows filtering on all System Events fields.
+- Allows **full text search** (`grep`) on all System and User fields.
+- Provides a **histogram** for log entries over time, with a break down per field-value, for any System Event field and any
+ time-frame.
+- Supports coloring log entries based on severity.
+- In PLAY mode it "tails" all the Events, showing new log entries immediately after they are received.
+
+### Prerequisites
+
+`windows-events.plugin` is a Netdata Function Plugin.
+
+To protect your privacy, as with all Netdata Functions, a free Netdata Cloud user account is required to access it.
+For more information check [this discussion](https://github.com/netdata/netdata/discussions/16136).
+
+## Events Sources
+
+The plugin automatically detects all the available channels and offers a list of "Event Channels".
+
+By default, it aggregates events from all event channels, providing a unified systems view of all events.
+
+> To improve query performance, we recommend selecting the relevant event channels, before doing more
+> analysis on the events.
+
+In the list of events channels, several shortcuts are added, aggregating events according to various attributes:
+
+- `All`, aggregates events from all available channels. This provides a holistic view of all events in the system.
+- `All-Admin`, `All-Operational`, `All-Analytic` and `All-Debug` aggregates events from channels marked `Admin`, `Operational`, `Analytic` and `Debug`, respectively.
+- `All-Windows`, aggregates events from `Application`, `Security`, `System` and `Setup`.
+- `All-Enabled` and `All-Disabled` aggregates events from channels depending on their status.
+- `All-Forwarded` aggregates events from channels owned by `Microsoft-Windows-EventCollector`.
+- `All-Classic` aggregates events from channels using the Classic Event Log API.
+- `All-Of-X`, where `X` is a provider name, is offered for all providers having more than a channel.
+- `All-In-X`, where `X` is `Backup-Mode`, `Overwrite-Mode`, `StopWhenFull-Mode` and `RetainAndBackup-Mode`, aggregate events based on their channel retention policy.
+
+Channels that are configured but are not queryable, and channels that do not have any events in them, are automatically excluded from the channels list.
+
+## Event Fields
+
+Windows Events are structured with both system-defined fields and user-defined fields.
+The Windows Events plugin primarily works with the system-defined fields, which are consistently available
+across all event types.
+
+### System-defined fields
+
+The system-defined fields are:
+
+1. **EventRecordID**
+ A unique, sequential identifier for the event within the channel. This ID increases as new events are logged.
+
+2. **Version**
+ The version of the event, indicating possible structural changes or updates to the event definition.
+
+ Netdata adds this field automatically when it is not zero.
+
+3. **Level**
+ The severity or importance of the event. Levels can include:
+ - 0: LogAlways (reserved)
+ - 1: Critical
+ - 2: Error
+ - 3: Warning
+ - 4: Informational
+ - 5: Verbose
+
+ Additionally, applications may define their own levels.
+
+ Netdata provides 2 fields: `Level` and `LevelID` for the text and numeric representation of it.
+
+4. **Opcode**
+ The action or state within a provider when the event was logged.
+
+ Netdata provides 2 fields: `Opcode` and `OpcodeID` for the text and numeric representation of it.
+
+5. **EventID**
+ This identifies the event template, linking it to a specific message or event type. Event IDs are provider-specific.
+
+6. **Task**
+ Defines a higher-level categorization or logical grouping for the event, often related to a specific function within the application or provider.
+
+ Netdata provides 2 fields: `Task` and `TaskID` for the text and numeric representation of it.
+
+7. **Qualifiers**
+ Provides additional detail for interpreting the event and is often specific to the event source.
+
+ Netdata adds this field automatically when it is not zero.
+
+8. **ProcessID**
+ The ID of the process that generated the event, useful for pinpointing the source of the event within the system.
+
+9. **ThreadID**
+ The ID of the thread within the process that generated the event, which helps in more detailed debugging scenarios.
+
+10. **Keywords**
+ A categorization field that can be used for event filtering. Keywords are bit flags that represent categories or purposes of the event, providing additional context.
+
+ Netdata provides 2 fields: `Keywords` and `keywordsID` for the text and numeric representation of it.
+
+11. **Provider**
+ The unique identifier (GUID) of the event provider. This is essential for knowing which application or system component generated the event.
+
+ Netdata provides 2 fields: `Provider` and `ProviderGUID` for its name and GUID of it.
+
+12. **ActivityID**
+ A GUID that correlates events generated as part of the same operation or transaction, helping to track activities across different components or stages.
+
+ Netdata adds this field automatically when it is not zero.
+
+13. **RelatedActivityID**
+ A GUID that links related operations or transactions, allowing for tracing complex workflows where one event triggers or relates to another.
+
+ Netdata adds this field automatically when it is not zero.
+
+14. **Timestamp**
+ The timestamp when the event was created. This provides precise timing information about when the event occurred.
+
+15. **User**
+ The system user who logged this event.
+
+ Netdata provides 3 fields: `UserAccount`, `UserDomain` and `UserSID`.
+
+### User-defined fields
+Each event log entry can include up to 100 user-defined fields (per event-id).
+
+Unfortunately, accessing these fields is significantly slower, to a level that is not practical to do so
+when there are more than few thousand log entries to explore. So, Netdata presents them
+with lazy loading.
+
+This prevents Netdata for offering filtering for user-defined fields, although Netdata does support
+full text search on user-defined field values.
+
+### Event fields as columns in the table
+
+The system fields mentioned above are offered as columns on the UI. Use the gear button above the table to
+select visible columns.
+
+### Event fields as filters
+
+The plugin presents the system fields as filters for the query, with counters for each of the possible values
+for the field. This list can be used to quickly check which fields and values are available for the entire
+time-frame of the query, across multiple providers and channels.
+
+### Event fields as histogram sources
+
+The histogram can be based on any of the system fields that are available as filters. This allows you to
+visualize the distribution of events over time based on different criteria such as Level, Provider, or EventID.
+
+## PLAY mode
+
+The PLAY mode in this plugin allows real-time monitoring of new events as they are added to the Windows Event
+Log. This feature works by continuously querying for new events and updating the display.
+
+## Full-text search
+
+The plugin supports searching for text within all system and user fields of the events. This means that while
+user-defined fields are not directly filterable, they are searchable through the full-text search feature.
+
+Keep in mind that query performance is slower while doing full text search, mainly because the plugin
+needs to ask from the system to provide all the user fields values.
+
+## Query performance
+
+The plugin is optimized to work efficiently with Event Logs. It uses several layers of caching and
+similar techniques to offload as much work as possible from the system, offering quick responses even when
+hundreds of thousands of events are within the visible timeframe.
+
+To achieve this level of efficiency, the plugin:
+
+- pre-loads ETW providers' manifests for resolving numeric Levels, Opcodes, Tasks and Keywords to text.
+- caches number to text maps for Levels, Opcodes, Tasks and Keywords per provider for WEL providers.
+- caches user SID to account and domain maps.
+- lazy loads the "expensive" event Message and XML, so that the system is queried only for the visible events.
+
+For Full Text Search:
+
+- requests only the Message and the values of the user-fields from the system, avoiding the "expensive" XML call (which is still lazy-loaded).
+
+The result is a system that is highly efficient for working with moderate volumes (hundreds of thousands) of events.
+
+## Configuration and maintenance
+
+This Netdata plugin does not require any specific configuration. It automatically detects available event logs
+on the system.
+
+## FAQ
+
+### Can I use this plugin on event centralization servers?
+
+Yes. You can centralize your Windows Events using Windows Event Forwarding (WEF) or other event collection
+mechanisms, and then install Netdata on this events centralization server to explore the events of all your
+infrastructure.
+
+This plugin will automatically provide multi-node views of your events and also give you the ability to
+combine the events of multiple servers, as you see fit.
+
+### Can I use this plugin from a parent Netdata?
+
+Yes. When your nodes are connected to a Netdata parent, all their functions are available via the parent's UI.
+So, from the parent UI, you can access the functions of all your nodes.
+
+Keep in mind that to protect your privacy, in order to access Netdata functions, you need a free Netdata Cloud
+account.
+
+### Is any of my data exposed to Netdata Cloud from this plugin?
+
+No. When you access the agent directly, none of your data passes through Netdata Cloud. You need a free Netdata
+Cloud account only to verify your identity and enable the use of Netdata Functions. Once this is done, all the
+data flow directly from your Netdata agent to your web browser.
+
+When you access Netdata via https://app.netdata.cloud, your data travel via Netdata Cloud, but they are not stored
+in Netdata Cloud. This is to allow you access your Netdata agents from anywhere. All communication from/to
+Netdata Cloud is encrypted.
+
+### What are the different types of event logs supported by this plugin?
+
+The plugin supports all the kinds of event logs currently supported by the Windows Event Viewer:
+
+- Windows Event Logs (WEL): The traditional event logging system in Windows.
+- Event Tracing for Windows (ETW): A more detailed and efficient event tracing system.
+- TraceLogging (TL): An extension of ETW that simplifies the process of adding events to your code.
+
+The plugin can access all of these when they are routed to the Windows Event Log.
+
+### How does this plugin handle user-defined fields in Windows Events?
+
+User-defined fields are not directly exposed as table columns or filters in the plugin interface. However,
+they are included in the XML representation of each event, which can be viewed in the info sidebar when
+clicking on an event entry. Additionally, the full-text search feature does search through these
+user-defined fields, allowing you to find specific information even if it's not in the main system fields.
+
+### Can I use this plugin to monitor real-time events?
+
+Yes, the plugin supports a PLAY mode that allows you to monitor events in real-time. When activated, it
+continuously updates to show new events as they are logged, similar to the "tail" functionality in
+Unix-like systems.
+
+### How does the plugin handle large volumes of events?
+
+The plugin is designed to handle moderate volumes of events (hundreds of thousands of events) efficiently.
+
+It is in our roadmap to port the `systemd-journal` sampling techniques to it, for working with very large
+datasets to provide quick responses while still giving accurate representations of the data. However, for
+the best performance, we recommend querying smaller time frames or using more specific filters when dealing
+with extremely large event volumes.
+
+### Can I use this plugin to analyze events from multiple servers?
+
+Yes, if you have set up Windows Event Forwarding (WEF) or another method of centralizing your Windows Events,
+you can use this plugin on the central server to analyze events from multiple sources. The plugin will
+automatically detect the available event sources.
+
+### How does the histogram feature work in this plugin?
+
+The histogram feature provides a visual representation of event frequency over time. You can base the
+histogram on any of the system fields available as filters (such as Level, Provider, or EventID). This
+allows you to quickly identify patterns or anomalies in your event logs.
+
+### Is it possible to export or share the results from this plugin?
+
+While the plugin doesn't have a direct export feature, you can use browser-based methods to save or share
+the results. This could include taking screenshots, using browser print/save as PDF functionality, or
+copying data from the table view. For more advanced data export needs, you might need to use the Windows
+Event Log API directly or other Windows administrative tools.
+
+### How often does the plugin update its data?
+
+The plugin updates its data in real-time when in PLAY mode. In normal mode, it refreshes data based on the
+query you've submitted. The plugin is designed to provide the most up-to-date information available in the
+Windows Event Logs at the time of the query.
+
+## TODO
+
+1. Support Sampling, so that the plugin can respond faster even on very busy systems (millions of events visible).
+2. Support exploring events from live Tracing sessions.
+3. Support exploring events in saved Event Trace Log files (`.etl` files).
+4. Support exploring events in saved Event Logs files (`.evtx` files).
diff --git a/src/collectors/windows-events.plugin/windows-events-fields-cache.c b/src/collectors/windows-events.plugin/windows-events-fields-cache.c
index f5f1f1fe4c..1e1e2a4e96 100644
--- a/src/collectors/windows-events.plugin/windows-events-fields-cache.c
+++ b/src/collectors/windows-events.plugin/windows-events-fields-cache.c
@@ -53,16 +53,16 @@ void field_cache_init(void) {
static inline bool should_zero_provider(WEVT_FIELD_TYPE type, uint64_t value) {
switch(type) {
case WEVT_FIELD_TYPE_LEVEL:
- return !is_valid_publisher_level(value, true);
+ return !is_valid_provider_level(value, true);
- case WEVT_FIELD_TYPE_KEYWORDS:
- return !is_valid_publisher_keywords(value, true);
+ case WEVT_FIELD_TYPE_KEYWORD:
+ return !is_valid_provider_keyword(value, true);
case WEVT_FIELD_TYPE_OPCODE:
- return !is_valid_publisher_opcode(value, true);
+ return !is_valid_provider_opcode(value, true);
case WEVT_FIELD_TYPE_TASK:
- return !is_valid_publisher_task(value, true);
+ return !is_valid_provider_task(value, true);
default:
return false;
diff --git a/src/collectors/windows-events.plugin/windows-events-fields-cache.h b/src/collectors/windows-events.plugin/windows-events-fields-cache.h
index 9b0866ab67..a76170d68a 100644
--- a/src/collectors/windows-events.plugin/windows-events-fields-cache.h
+++ b/src/collectors/windows-events.plugin/windows-events-fields-cache.h
@@ -8,7 +8,7 @@
typedef enum __attribute__((packed)) {
WEVT_FIELD_TYPE_LEVEL = 0,
WEVT_FIELD_TYPE_OPCODE,
- WEVT_FIELD_TYPE_KEYWORDS,
+ WEVT_FIELD_TYPE_KEYWORD,
WEVT_FIELD_TYPE_TASK,
// terminator
diff --git a/src/collectors/windows-events.plugin/windows-events-publishers.c b/src/collectors/windows-events.plugin/windows-events-providers.c
similarity index 61%
rename from src/collectors/windows-events.plugin/windows-events-publishers.c
rename to src/collectors/windows-events.plugin/windows-events-providers.c
index 5928cb8df4..a0400cab93 100644
--- a/src/collectors/windows-events.plugin/windows-events-publishers.c
+++ b/src/collectors/windows-events.plugin/windows-events-providers.c
@@ -1,17 +1,19 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#include "windows-events-publishers.h"
+#include "windows-events-providers.h"
-#define MAX_OPEN_HANDLES_PER_PUBLISHER 5
+#define MAX_OPEN_HANDLES_PER_PROVIDER 5
-struct publisher;
+struct provider;
// typedef as PROVIDER_META_HANDLE in include file
struct provider_meta_handle {
pid_t owner; // the owner of the handle, or zero
uint32_t locks; // the number of locks the owner has on this handle
EVT_HANDLE hMetadata; // the handle
- struct publisher *publisher; // a pointer back to the publisher
+ struct provider *provider; // a pointer back to the provider
+
+ usec_t created_monotonic_ut; // the monotonic timestamp this handle was created
// double linked list
PROVIDER_META_HANDLE *prev;
@@ -32,78 +34,156 @@ struct provider_list {
struct provider_data *array; // the array of entries, sorted (for binary search)
};
-typedef struct publisher {
+typedef struct provider_key {
ND_UUID uuid; // the Provider GUID
- const char *name; // the Provider name (UTF-8)
+ DWORD len; // the length of the Provider Name
+ const wchar_t *wname; // the Provider wide-string Name (UTF-16)
+} PROVIDER_KEY;
+
+typedef struct provider {
+ PROVIDER_KEY key;
+ const char *name; // the Provider Name (UTF-8)
uint32_t total_handles; // the number of handles allocated
uint32_t available_handles; // the number of available handles
uint32_t deleted_handles; // the number of deleted handles
PROVIDER_META_HANDLE *handles; // a double linked list of all the handles
- struct provider_list keywords;
+ WEVT_PROVIDER_PLATFORM platform;
+
+ struct provider_list keyword;
struct provider_list tasks;
struct provider_list opcodes;
struct provider_list levels;
-} PUBLISHER;
+} PROVIDER;
-// A hashtable implementation for publishers
-// using the provider GUID as key and PUBLISHER as value
-#define SIMPLE_HASHTABLE_NAME _PROVIDER_GUID
-#define SIMPLE_HASHTABLE_VALUE_TYPE PUBLISHER
-#define SIMPLE_HASHTABLE_KEY_TYPE ND_UUID
-#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION publisher_value_to_key
-#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION publisher_cache_compar
+// A hashtable implementation for Providers
+// using the Provider GUID as key and PROVIDER as value
+#define SIMPLE_HASHTABLE_NAME _PROVIDER
+#define SIMPLE_HASHTABLE_VALUE_TYPE PROVIDER
+#define SIMPLE_HASHTABLE_KEY_TYPE PROVIDER_KEY
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION provider_value_to_key
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION provider_cache_compar
#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
#include "libnetdata/simple_hashtable.h"
static struct {
SPINLOCK spinlock;
- uint32_t total_publishers;
+ uint32_t total_providers;
uint32_t total_handles;
uint32_t deleted_handles;
- struct simple_hashtable_PROVIDER_GUID hashtable;
- ARAL *aral_publishers;
+ struct simple_hashtable_PROVIDER hashtable;
+ ARAL *aral_providers;
ARAL *aral_handles;
} pbc = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
};
-static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
+static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
-static inline ND_UUID *publisher_value_to_key(PUBLISHER *p) {
- return &p->uuid;
+const char *provider_get_name(PROVIDER_META_HANDLE *p) {
+ return (p && p->provider && p->provider->name) ? p->provider->name : "__UNKNOWN PROVIDER__";
}
-static inline bool publisher_cache_compar(ND_UUID *a, ND_UUID *b) {
- return UUIDeq(*a, *b);
+ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p) {
+ return (p && p->provider) ? p->provider->key.uuid : UUID_ZERO;
}
-void publisher_cache_init(void) {
- simple_hashtable_init_PROVIDER_GUID(&pbc.hashtable, 100000);
- pbc.aral_publishers = aral_create("wevt_publishers", sizeof(PUBLISHER), 0, 4096, NULL, NULL, NULL, false, true);
+static inline PROVIDER_KEY *provider_value_to_key(PROVIDER *p) {
+ return &p->key;
+}
+
+static inline bool provider_cache_compar(PROVIDER_KEY *a, PROVIDER_KEY *b) {
+ return a->len == b->len && UUIDeq(a->uuid, b->uuid) && memcmp(a->wname, b->wname, a->len) == 0;
+}
+
+void provider_cache_init(void) {
+ simple_hashtable_init_PROVIDER(&pbc.hashtable, 100000);
+ pbc.aral_providers = aral_create("wevt_providers", sizeof(PROVIDER), 0, 4096, NULL, NULL, NULL, false, true);
pbc.aral_handles = aral_create("wevt_handles", sizeof(PROVIDER_META_HANDLE), 0, 4096, NULL, NULL, NULL, false, true);
}
-PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
- if(!providerName || !providerName[0] || UUIDiszero(uuid))
+static bool provider_property_get(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
+ DWORD bufferUsed = 0;
+
+ if(!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
+ DWORD status = GetLastError();
+ if (status != ERROR_INSUFFICIENT_BUFFER) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
+ goto cleanup;
+ }
+ }
+
+ wevt_variant_resize(content, bufferUsed);
+ if (!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
+ goto cleanup;
+ }
+
+ return true;
+
+cleanup:
+ return false;
+}
+
+static bool provider_string_property_exists(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
+ if(!provider_property_get(h, content, property_id))
+ return false;
+
+ if(content->data->Type != EvtVarTypeString)
+ return false;
+
+ if(!content->data->StringVal[0])
+ return false;
+
+ return true;
+}
+
+static void provider_detect_platform(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content) {
+ if(UUIDiszero(h->provider->key.uuid))
+ h->provider->platform = WEVT_PLATFORM_WEL;
+ else if(h->hMetadata) {
+ if (provider_string_property_exists(h, content, EvtPublisherMetadataMessageFilePath) ||
+ provider_string_property_exists(h, content, EvtPublisherMetadataResourceFilePath) ||
+ provider_string_property_exists(h, content, EvtPublisherMetadataParameterFilePath))
+ h->provider->platform = WEVT_PLATFORM_ETW;
+ else
+ // The provider cannot be opened, does not have any resource files (message, resource, parameter)
+ h->provider->platform = WEVT_PLATFORM_TL;
+ }
+ else h->provider->platform = WEVT_PLATFORM_ETW;
+}
+
+WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p) {
+ return p->provider->platform;
+}
+
+PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName) {
+ if(!providerName || !providerName[0])
return NULL;
- // XXH64_hash_t hash = XXH3_64bits(&uuid, sizeof(uuid));
- uint64_t hash = uuid.parts.low64 + uuid.parts.hig64;
+ PROVIDER_KEY key = {
+ .uuid = uuid,
+ .len = wcslen(providerName),
+ .wname = providerName,
+ };
+ XXH64_hash_t hash = XXH3_64bits(providerName, wcslen(key.wname) * sizeof(*key.wname));
spinlock_lock(&pbc.spinlock);
- SIMPLE_HASHTABLE_SLOT_PROVIDER_GUID *slot =
- simple_hashtable_get_slot_PROVIDER_GUID(&pbc.hashtable, hash, &uuid, true);
+ SIMPLE_HASHTABLE_SLOT_PROVIDER *slot =
+ simple_hashtable_get_slot_PROVIDER(&pbc.hashtable, hash, &key, true);
bool load_it = false;
- PUBLISHER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+ PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
if(!p) {
- p = aral_callocz(pbc.aral_publishers);
- p->uuid = uuid;
- simple_hashtable_set_slot_PROVIDER_GUID(&pbc.hashtable, slot, hash, p);
+ p = aral_callocz(pbc.aral_providers);
+ p->key.uuid = key.uuid;
+ p->key.len = key.len;
+ p->key.wname = wcsdup(key.wname);
+ p->name = strdupz(provider2utf8(key.wname));
+ simple_hashtable_set_slot_PROVIDER(&pbc.hashtable, slot, hash, p);
load_it = true;
- pbc.total_publishers++;
+ pbc.total_providers++;
}
pid_t me = gettid_cached();
@@ -117,7 +197,8 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
if(!h) {
h = aral_callocz(pbc.aral_handles);
- h->publisher = p;
+ h->provider = p;
+ h->created_monotonic_ut = now_monotonic_usec();
h->hMetadata = EvtOpenPublisherMetadata(
NULL, // Local machine
providerName, // Provider name
@@ -147,10 +228,11 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
WEVT_VARIANT property = { 0 };
TXT_UNICODE unicode = { 0 };
- publisher_load_list(h, &content, &property, &unicode, &p->keywords, EvtPublisherMetadataKeywords);
- publisher_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
- publisher_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
- publisher_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
+ provider_detect_platform(h, &content);
+ provider_load_list(h, &content, &property, &unicode, &p->keyword, EvtPublisherMetadataKeywords);
+ provider_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
+ provider_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
+ provider_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
txt_unicode_cleanup(&unicode);
wevt_variant_cleanup(&content);
@@ -162,42 +244,72 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
return h;
}
-EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h) {
+EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h) {
return h ? h->hMetadata : NULL;
}
-PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h) {
+PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h) {
if(h) h->locks++;
return h;
}
-void publisher_release(PROVIDER_META_HANDLE *h) {
+static void provider_meta_handle_delete(PROVIDER_META_HANDLE *h) {
+ PROVIDER *p = h->provider;
+
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
+
+ if(h->hMetadata)
+ EvtClose(h->hMetadata);
+
+ aral_freez(pbc.aral_handles, h);
+
+ fatal_assert(pbc.total_handles && p->total_handles && p->available_handles);
+
+ pbc.total_handles--;
+ p->total_handles--;
+
+ pbc.deleted_handles++;
+ p->deleted_handles++;
+
+ p->available_handles--;
+}
+
+void providers_release_unused_handles(void) {
+ usec_t now_ut = now_monotonic_usec();
+
+ spinlock_lock(&pbc.spinlock);
+ for(size_t i = 0; i < pbc.hashtable.size ; i++) {
+ SIMPLE_HASHTABLE_SLOT_PROVIDER *slot = &pbc.hashtable.hashtable[i];
+ PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+ if(!p) continue;
+
+ PROVIDER_META_HANDLE *h = p->handles;
+ while(h) {
+ PROVIDER_META_HANDLE *next = h->next;
+
+ if(!h->locks && (now_ut - h->created_monotonic_ut) >= WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT)
+ provider_meta_handle_delete(h);
+
+ h = next;
+ }
+ }
+ spinlock_unlock(&pbc.spinlock);
+}
+
+void provider_release(PROVIDER_META_HANDLE *h) {
if(!h) return;
pid_t me = gettid_cached();
fatal_assert(h->owner == me);
fatal_assert(h->locks > 0);
if(--h->locks == 0) {
- PUBLISHER *p = h->publisher;
+ PROVIDER *p = h->provider;
spinlock_lock(&pbc.spinlock);
h->owner = 0;
- if(++p->available_handles > MAX_OPEN_HANDLES_PER_PUBLISHER) {
- // there are multiple handles on this publisher
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
-
- if(h->hMetadata)
- EvtClose(h->hMetadata);
-
- aral_freez(pbc.aral_handles, h);
-
- pbc.total_handles--;
- p->total_handles--;
-
- pbc.deleted_handles++;
- p->deleted_handles++;
-
- p->available_handles--;
+ if(++p->available_handles > MAX_OPEN_HANDLES_PER_PROVIDER) {
+ // there are too many idle handles on this provider
+ provider_meta_handle_delete(h);
}
else if(h->next) {
// it is not the last, put it at the end
@@ -210,7 +322,7 @@ void publisher_release(PROVIDER_META_HANDLE *h) {
}
// --------------------------------------------------------------------------------------------------------------------
-// load publisher lists
+// load provider lists
static bool wevt_get_property_from_array(WEVT_VARIANT *property, EVT_HANDLE handle, DWORD dwIndex, EVT_PUBLISHER_METADATA_PROPERTY_ID PropertyId) {
DWORD used = 0;
@@ -253,7 +365,7 @@ static int compare_ascending(const void *a, const void *b) {
// return 0;
//}
-static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
+static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
if(!h || !h->hMetadata) return;
EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id;
@@ -268,7 +380,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
value_id = EvtPublisherMetadataLevelValue;
value_bits = 32;
compare_func = compare_ascending;
- is_valid = is_valid_publisher_level;
+ is_valid = is_valid_provider_level;
break;
case EvtPublisherMetadataOpcodes:
@@ -276,7 +388,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataOpcodeMessageID;
value_id = EvtPublisherMetadataOpcodeValue;
value_bits = 32;
- is_valid = is_valid_publisher_opcode;
+ is_valid = is_valid_provider_opcode;
compare_func = compare_ascending;
break;
@@ -285,7 +397,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataTaskMessageID;
value_id = EvtPublisherMetadataTaskValue;
value_bits = 32;
- is_valid = is_valid_publisher_task;
+ is_valid = is_valid_provider_task;
compare_func = compare_ascending;
break;
@@ -294,7 +406,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataKeywordMessageID;
value_id = EvtPublisherMetadataKeywordValue;
value_bits = 64;
- is_valid = is_valid_publisher_keywords;
+ is_valid = is_valid_provider_keyword;
compare_func = NULL;
break;
@@ -305,23 +417,11 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
EVT_HANDLE hMetadata = h->hMetadata;
EVT_HANDLE hArray = NULL;
- DWORD bufferUsed = 0;
DWORD itemCount = 0;
// Get the metadata array for the list (e.g., opcodes, tasks, or levels)
- if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
- DWORD status = GetLastError();
- if (status != ERROR_INSUFFICIENT_BUFFER) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
- goto cleanup;
- }
- }
-
- wevt_variant_resize(content, bufferUsed);
- if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
+ if(!provider_property_get(h, content, property_id))
goto cleanup;
- }
// Get the number of items (e.g., levels, tasks, or opcodes)
hArray = content->data->EvtHandleVal;
@@ -375,7 +475,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
uint32_t messageID = wevt_field_get_uint32(property->data);
if (messageID != (uint32_t)-1) {
- if (wevt_get_message_unicode(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
+ if (EvtFormatMessage_utf16(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
size_t len;
d->name = unicode2utf8_strdupz(dst->data, &len);
d->len = len;
@@ -414,7 +514,7 @@ cleanup:
// lookup functions
// lookup bitmap metdata (returns a comma separated list of strings)
-static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+static bool provider_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
if(!(value & l->mask) || !l->total || !l->array || l->exceeds_data_type)
return false;
@@ -445,7 +545,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
memcpy(&dst->data[dst->used], s, slen);
dst->used += slen;
- dst->src = TXT_SOURCE_PUBLISHER;
+ dst->src = TXT_SOURCE_PROVIDER;
added++;
}
}
@@ -460,7 +560,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
}
//// lookup a single value (returns its string)
-//static bool publisher_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+//static bool provider_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
// if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
// return false;
//
@@ -477,7 +577,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
//
// memcpy(dst->data, s, slen);
// dst->used = slen;
-// dst->src = TXT_SOURCE_PUBLISHER;
+// dst->src = TXT_SOURCE_PROVIDER;
//
// break;
// }
@@ -493,11 +593,11 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
// return (dst->used > 0);
//}
-static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+static bool provider_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
return false;
- // if(l->total < 3) return publisher_value_metadata_linear(dst, l, value);
+ // if(l->total < 3) return provider_value_metadata_linear(dst, l, value);
dst->used = 0;
@@ -519,7 +619,7 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
memcpy(dst->data, s, slen);
dst->used = slen;
dst->data[dst->used++] = 0;
- dst->src = TXT_SOURCE_PUBLISHER;
+ dst->src = TXT_SOURCE_PROVIDER;
}
break;
}
@@ -539,38 +639,38 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
// --------------------------------------------------------------------------------------------------------------------
// public API to lookup metadata
-bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h) {
- return h && !h->publisher->keywords.exceeds_data_type;
+bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h) {
+ return h && !h->provider->keyword.exceeds_data_type;
}
-bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h) {
- return h && !h->publisher->tasks.exceeds_data_type;
+bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h) {
+ return h && !h->provider->tasks.exceeds_data_type;
}
-bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h) {
- return h && !h->publisher->levels.exceeds_data_type;
+bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h) {
+ return h && !h->provider->levels.exceeds_data_type;
}
-bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
- return h && !h->publisher->opcodes.exceeds_data_type;
+bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
+ return h && !h->provider->opcodes.exceeds_data_type;
}
-bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
- return publisher_bitmap_metadata(dst, &h->publisher->keywords, value);
+ return provider_bitmap_metadata(dst, &h->provider->keyword, value);
}
-bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
- return publisher_value_metadata(dst, &h->publisher->levels, value);
+ return provider_value_metadata(dst, &h->provider->levels, value);
}
-bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
- return publisher_value_metadata(dst, &h->publisher->tasks, value);
+ return provider_value_metadata(dst, &h->provider->tasks, value);
}
-bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
- return publisher_value_metadata(dst, &h->publisher->opcodes, value);
+ return provider_value_metadata(dst, &h->provider->opcodes, value);
}
diff --git a/src/collectors/windows-events.plugin/windows-events-providers.h b/src/collectors/windows-events.plugin/windows-events-providers.h
new file mode 100644
index 0000000000..b6d476c5c3
--- /dev/null
+++ b/src/collectors/windows-events.plugin/windows-events-providers.h
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_WINDOWS_EVENTS_PROVIDERS_H
+#define NETDATA_WINDOWS_EVENTS_PROVIDERS_H
+
+typedef enum __attribute__((packed)) {
+ WEVT_PLATFORM_UNKNOWN = 0,
+ WEVT_PLATFORM_WEL,
+ WEVT_PLATFORM_ETW,
+ WEVT_PLATFORM_TL,
+} WEVT_PROVIDER_PLATFORM;
+
+#include "windows-events.h"
+
+struct provider_meta_handle;
+typedef struct provider_meta_handle PROVIDER_META_HANDLE;
+
+PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName);
+void provider_release(PROVIDER_META_HANDLE *h);
+EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h);
+PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h);
+
+void providers_release_unused_handles(void);
+
+const char *provider_get_name(PROVIDER_META_HANDLE *p);
+ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p);
+
+void provider_cache_init(void);
+
+bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h);
+bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h);
+bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h);
+bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h);
+
+bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p);
+
+#endif //NETDATA_WINDOWS_EVENTS_PROVIDERS_H
diff --git a/src/collectors/windows-events.plugin/windows-events-publishers.h b/src/collectors/windows-events.plugin/windows-events-publishers.h
deleted file mode 100644
index b9269dcadb..0000000000
--- a/src/collectors/windows-events.plugin/windows-events-publishers.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
-#define NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
-
-#include "windows-events.h"
-
-struct provider_meta_handle;
-typedef struct provider_meta_handle PROVIDER_META_HANDLE;
-
-PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName);
-void publisher_release(PROVIDER_META_HANDLE *h);
-EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h);
-PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h);
-
-void publisher_cache_init(void);
-
-bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h);
-bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h);
-bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h);
-bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h);
-
-bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
-bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
-bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
-bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
-
-#endif //NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
diff --git a/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c b/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
new file mode 100644
index 0000000000..9d18f81fcf
--- /dev/null
+++ b/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "windows-events.h"
+#include <sddl.h> // For SID string conversion
+
+// Function to append the separator if the buffer is not empty
+static inline void append_separator_if_needed(BUFFER *b, const char *separator) {
+ if (buffer_strlen(b) > 0 && separator != NULL)
+ buffer_strcat(b, separator);
+}
+
+// Helper function to convert UTF16 strings to UTF8 and append to the buffer
+static inline void append_utf16(BUFFER *b, LPCWSTR utf16Str, const char *separator) {
+ if (!utf16Str || !*utf16Str) return;
+
+ append_separator_if_needed(b, separator);
+
+ size_t remaining = b->size - b->len;
+ if(remaining < 128) {
+ buffer_need_bytes(b, 128);
+ remaining = b->size - b->len;
+ }
+
+ size_t used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
+ if(used >= remaining) {
+ // oops, we need to resize
+ size_t needed = utf16_to_utf8(NULL, 0, utf16Str, -1); // find the size needed
+ buffer_need_bytes(b, needed);
+ remaining = b->size - b->len;
+ used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
+ }
+
+ if(used) {
+ b->len += used - 1;
+
+ internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
+ "Buffer length mismatch.");
+ }
+}
+
+// Function to append binary data to the buffer
+static inline void append_binary(BUFFER *b, PBYTE data, DWORD size, const char *separator) {
+ if (data == NULL || size == 0) return;
+
+ append_separator_if_needed(b, separator);
+
+ buffer_need_bytes(b, size * 4);
+ for (DWORD i = 0; i < size; i++) {
+ uint8_t value = data[i];
+ b->buffer[b->len++] = hex_digits[(value & 0xf0) >> 4];
+ b->buffer[b->len++] = hex_digits[(value & 0x0f)];
+ }
+}
+
+// Function to append size_t to the buffer
+static inline void append_size_t(BUFFER *b, size_t size, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64(b, size);
+}
+
+// Function to append HexInt32 in hexadecimal format
+static inline void append_uint32_hex(BUFFER *b, UINT32 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64_hex(b, n);
+}
+
+// Function to append HexInt64 in hexadecimal format
+static inline void append_uint64_hex(BUFFER *b, UINT64 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64_hex(b, n);
+}
+
+// Function to append various data types to the buffer
+static inline void append_uint64(BUFFER *b, UINT64 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64(b, n);
+}
+
+static inline void append_int64(BUFFER *b, INT64 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_int64(b, n);
+}
+
+static inline void append_double(BUFFER *b, double n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_netdata_double(b, n);
+}
+
+static inline void append_guid(BUFFER *b, GUID *guid, const char *separator) {
+ fatal_assert(sizeof(GUID) == sizeof(nd_uuid_t));
+
+ append_separator_if_needed(b, separator);
+
+ ND_UUID *uuid = (ND_UUID *)guid;
+ buffer_need_bytes(b, UUID_STR_LEN);
+ uuid_unparse_lower(uuid->uuid, &b->buffer[b->len]);
+ b->len += UUID_STR_LEN - 1;
+
+ internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
+ "Buffer length mismatch.");
+}
+
+static inline void append_systime(BUFFER *b, SYSTEMTIME *st, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_sprintf(b, "%04d-%02d-%02d %02d:%02d:%02d",
+ st->wYear, st->wMonth, st->wDay, st->wHour, st->wMinute, st->wSecond);
+}
+
+static inline void append_filetime(BUFFER *b, FILETIME *ft, const char *separator) {
+ SYSTEMTIME st;
+ if (FileTimeToSystemTime(ft, &st))
+ append_systime(b, &st, separator);
+}
+
+static inline void append_sid(BUFFER *b, PSID sid, const char *separator) {
+ buffer_sid_to_sid_str_and_name(sid, b, separator);
+}
+
+static inline void append_sbyte(BUFFER *b, INT8 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_int64(b, n);
+}
+
+static inline void append_byte(BUFFER *b, UINT8 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64(b, n);
+}
+
+static inline void append_int16(BUFFER *b, INT16 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_int64(b, n);
+}
+
+static inline void append_uint16(BUFFER *b, UINT16 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64(b, n);
+}
+
+static inline void append_int32(BUFFER *b, INT32 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_int64(b, n);
+}
+
+static inline void append_uint32(BUFFER *b, UINT32 n, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64(b, n);
+}
+
+// Function to append EVT_HANDLE to the buffer
+static inline void append_evt_handle(BUFFER *b, EVT_HANDLE h, const char *separator) {
+ append_separator_if_needed(b, separator);
+ buffer_print_uint64_hex(b, (uintptr_t)h);
+}
+
+// Function to append XML data (UTF-16) to the buffer
+static inline void append_evt_xml(BUFFER *b, LPCWSTR xmlData, const char *separator) {
+ append_utf16(b, xmlData, separator); // XML data is essentially UTF-16 string
+}
+
+void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator) {
+ if(ev->Type == EvtVarTypeNull) return;
+
+ if (ev->Type & EVT_VARIANT_TYPE_ARRAY) {
+ for (DWORD i = 0; i < ev->Count; i++) {
+ switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
+ case EvtVarTypeString:
+ append_utf16(b, ev->StringArr[i], separator);
+ break;
+
+ case EvtVarTypeAnsiString:
+ if (ev->AnsiStringArr[i] != NULL) {
+ append_utf16(b, (LPCWSTR)ev->AnsiStringArr[i], separator);
+ }
+ break;
+
+ case EvtVarTypeSByte:
+ append_sbyte(b, ev->SByteArr[i], separator);
+ break;
+
+ case EvtVarTypeByte:
+ append_byte(b, ev->ByteArr[i], separator);
+ break;
+
+ case EvtVarTypeInt16:
+ append_int16(b, ev->Int16Arr[i], separator);
+ break;
+
+ case EvtVarTypeUInt16:
+ append_uint16(b, ev->UInt16Arr[i], separator);
+ break;
+
+ case EvtVarTypeInt32:
+ append_int32(b, ev->Int32Arr[i], separator);
+ break;
+
+ case EvtVarTypeUInt32:
+ append_uint32(b, ev->UInt32Arr[i], separator);
+ break;
+
+ case EvtVarTypeInt64:
+ append_int64(b, ev->Int64Arr[i], separator);
+ break;
+
+ case EvtVarTypeUInt64:
+ append_uint64(b, ev->UInt64Arr[i], separator);
+ break;
+
+ case EvtVarTypeSingle:
+ append_double(b, ev->SingleArr[i], separator);
+ break;
+
+ case EvtVarTypeDouble:
+ append_double(b, ev->DoubleArr[i], separator);
+ break;
+
+ case EvtVarTypeGuid:
+ append_guid(b, &ev->GuidArr[i], separator);
+ break;
+
+ case EvtVarTypeFileTime:
+ append_filetime(b, &ev->FileTimeArr[i], separator);
+ break;
+
+ case EvtVarTypeSysTime:
+ append_systime(b, &ev->SysTimeArr[i], separator);
+ break;
+
+ case EvtVarTypeSid:
+ append_sid(b, ev->SidArr[i], separator);
+ break;
+
+ case EvtVarTypeBinary:
+ append_binary(b, ev->BinaryVal, ev->Count, separator);
+ break;
+
+ case EvtVarTypeSizeT:
+ append_size_t(b, ev->SizeTArr[i], separator);
+ break;
+
+ case EvtVarTypeHexInt32:
+ append_uint32_hex(b, ev->UInt32Arr[i], separator);
+ break;
+
+ case EvtVarTypeHexInt64:
+ append_uint64_hex(b, ev->UInt64Arr[i], separator);
+ break;
+
+ case EvtVarTypeEvtHandle:
+ append_evt_handle(b, ev->EvtHandleVal, separator);
+ break;
+
+ case EvtVarTypeEvtXml:
+ append_evt_xml(b, ev->XmlValArr[i], separator);
+ break;
+
+ default:
+ // Skip unknown array types
+ break;
+ }
+ }
+ } else {
+ switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
+ case EvtVarTypeNull:
+ // Do nothing for null types
+ break;
+
+ case EvtVarTypeString:
+ append_utf16(b, ev->StringVal, separator);
+ break;
+
+ case EvtVarTypeAnsiString:
+ append_utf16(b, (LPCWSTR)ev->AnsiStringVal, separator);
+ break;
+
+ case EvtVarTypeSByte:
+ append_sbyte(b, ev->SByteVal, separator);
+ break;
+
+ case EvtVarTypeByte:
+ append_byte(b, ev->ByteVal, separator);
+ break;
+
+ case EvtVarTypeInt16:
+ append_int16(b, ev->Int16Val, separator);
+ break;
+
+ case EvtVarTypeUInt16:
+ append_uint16(b, ev->UInt16Val, separator);
+ break;
+
+ case EvtVarTypeInt32:
+ append_int32(b, ev->Int32Val, separator);
+ break;
+
+ case EvtVarTypeUInt32:
+ append_uint32(b, ev->UInt32Val, separator);
+ break;
+
+ case EvtVarTypeInt64:
+ append_int64(b, ev->Int64Val, separator);
+ break;
+
+ case EvtVarTypeUInt64:
+ append_uint64(b, ev->UInt64Val, separator);
+ break;
+
+ case EvtVarTypeSingle:
+ append_double(b, ev->SingleVal, separator);
+ break;
+
+ case EvtVarTypeDouble:
+ append_double(b, ev->DoubleVal, separator);
+ break;
+
+ case EvtVarTypeBoolean:
+ append_separator_if_needed(b, separator);
+ buffer_strcat(b, ev->BooleanVal ? "true" : "false");
+ break;
+
+ case EvtVarTypeGuid:
+ append_guid(b, ev->GuidVal, separator);
+ break;
+
+ case EvtVarTypeBinary:
+ append_binary(b, ev->BinaryVal, ev->Count, separator);
+ break;
+
+ case EvtVarTypeSizeT:
+ append_size_t(b, ev->SizeTVal, separator);
+ break;
+
+ case EvtVarTypeHexInt32:
+ append_uint32_hex(b, ev->UInt32Val, separator);
+ break;
+
+ case EvtVarTypeHexInt64:
+ append_uint64_hex(b, ev->UInt64Val, separator);
+ break;
+
+ case EvtVarTypeEvtHandle:
+ append_evt_handle(b, ev->EvtHandleVal, separator);
+ break;
+
+ case EvtVarTypeEvtXml:
+ append_evt_xml(b, ev->XmlVal, separator);
+ break;
+
+ default:
+ // Skip unknown types
+ break;
+ }
+ }
+}
diff --git a/src/collectors/windows-events.plugin/windows-events-query.c b/src/collectors/windows-events.plugin/windows-events-query.c
index d4c660c14a..12734fe9d5 100644
--- a/src/collectors/windows-events.plugin/windows-events-query.c
+++ b/src/collectors/windows-events.plugin/windows-events-query.c
@@ -2,48 +2,13 @@
#include "windows-events.h"
+static void wevt_event_done(WEVT_LOG *log);
+
static uint64_t wevt_log_file_size(const wchar_t *channel);
-#define FIELD_RECORD_NUMBER (0)
-#define FIELD_EVENT_ID (1)
-#define FIELD_LEVEL (2)
-#define FIELD_OPCODE (3)
-#define FIELD_KEYWORDS (4)
-#define FIELD_VERSION (5)
-#define FIELD_TASK (6)
-#define FIELD_PROCESS_ID (7)
-#define FIELD_THREAD_ID (8)
-#define FIELD_TIME_CREATED (9)
-#define FIELD_CHANNEL (10)
-#define FIELD_COMPUTER_NAME (11)
-#define FIELD_PROVIDER_NAME (12)
-#define FIELD_EVENT_SOURCE_NAME (13)
-#define FIELD_PROVIDER_GUID (14)
-#define FIELD_CORRELATION_ACTIVITY_ID (15)
-#define FIELD_USER_ID (16)
+// --------------------------------------------------------------------------------------------------------------------
-// These are the fields we extract from the logs
-static const wchar_t *RENDER_ITEMS[] = {
- L"/Event/System/EventRecordID",
- L"/Event/System/EventID",
- L"/Event/System/Level",
- L"/Event/System/Opcode",
- L"/Event/System/Keywords",
- L"/Event/System/Version",
- L"/Event/System/Task",
- L"/Event/System/Execution/@ProcessID",
- L"/Event/System/Execution/@ThreadID",
- L"/Event/System/TimeCreated/@SystemTime",
- L"/Event/System/Channel",
- L"/Event/System/Computer",
- L"/Event/System/Provider/@Name",
- L"/Event/System/Provider/@EventSourceName",
- L"/Event/System/Provider/@Guid",
- L"/Event/System/Correlation/@ActivityID",
- L"/Event/System/Security/@UserID",
-};
-
-static const char *wevt_extended_status(void) {
+static const char *EvtGetExtendedStatus_utf8(void) {
static __thread wchar_t wbuf[4096];
static __thread char buf[4096];
DWORD wbuf_used = 0;
@@ -62,7 +27,9 @@ static const char *wevt_extended_status(void) {
return buf;
}
-bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
+// --------------------------------------------------------------------------------------------------------------------
+
+bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
dst->used = 0;
DWORD size = 0;
@@ -107,56 +74,44 @@ cleanup:
return false;
}
-static bool wevt_get_field_from_events_log(
- WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
- TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
+static bool EvtFormatMessage_utf8(
+ TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
+ TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
dst->src = TXT_SOURCE_EVENT_LOG;
- if(wevt_get_message_unicode(&log->ops.unicode, publisher_handle(p), hEvent, 0, flags))
- return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
+ if(EvtFormatMessage_utf16(tmp, provider_handle(p), hEvent, 0, flags))
+ return wevt_str_unicode_to_utf8(dst, tmp);
wevt_utf8_empty(dst);
return false;
}
-bool wevt_get_event_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
- return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageEvent);
+bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
+ return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageEvent);
}
-bool wevt_get_xml_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
- return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageXml);
+bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
+ return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageXml);
}
-static inline void wevt_event_done(WEVT_LOG *log) {
- if (log->publisher) {
- publisher_release(log->publisher);
- log->publisher = NULL;
- }
-
- if (log->hEvent) {
- EvtClose(log->hEvent);
- log->hEvent = NULL;
- }
-
- log->ops.level.src = TXT_SOURCE_UNKNOWN;
- log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
- log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
- log->ops.task.src = TXT_SOURCE_UNKNOWN;
-}
+// --------------------------------------------------------------------------------------------------------------------
static void wevt_get_field_from_cache(
- WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
- TXT_UTF8 *dst, const ND_UUID *provider,
- WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
+ WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
+ TXT_UTF8 *dst, const ND_UUID *provider,
+ WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
if (field_cache_get(cache_type, provider, value, dst))
return;
- wevt_get_field_from_events_log(log, h, log->hEvent, dst, flags);
+ EvtFormatMessage_utf8(&log->ops.unicode, h, log->hEvent, dst, flags);
field_cache_set(cache_type, provider, value, dst);
}
+// --------------------------------------------------------------------------------------------------------------------
+// Level
+
#define SET_LEN_AND_RETURN(constant) *len = sizeof(constant) - 1; return constant
static inline const char *wevt_level_hardcoded(uint64_t level, size_t *len) {
@@ -179,9 +134,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageLevel;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_LEVEL;
- bool is_publisher = is_valid_publisher_level(value, true);
+ bool is_provider = is_valid_provider_level(value, true);
- if(!is_publisher) {
+ if(!is_provider) {
size_t len;
const char *hardcoded = wevt_level_hardcoded(value, &len);
if(hardcoded) {
@@ -189,12 +144,12 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
dst->src = TXT_SOURCE_HARDCODED;
}
else {
- // since this is not a publisher value
+ // since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
- else if (!publisher_get_level(dst, h, value)) {
+ else if (!provider_get_level(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@@ -203,6 +158,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
dst, WEVT_PREFIX_LEVEL, sizeof(WEVT_PREFIX_LEVEL) - 1, ev->level);
}
+// --------------------------------------------------------------------------------------------------------------------
+// Opcode
+
static inline const char *wevt_opcode_hardcoded(uint64_t opcode, size_t *len) {
switch(opcode) {
case WEVT_OPCODE_INFO: SET_LEN_AND_RETURN(WEVT_OPCODE_NAME_INFO);
@@ -228,9 +186,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageOpcode;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_OPCODE;
- bool is_publisher = is_valid_publisher_opcode(value, true);
+ bool is_provider = is_valid_provider_opcode(value, true);
- if(!is_publisher) {
+ if(!is_provider) {
size_t len;
const char *hardcoded = wevt_opcode_hardcoded(value, &len);
if(hardcoded) {
@@ -238,12 +196,12 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
dst->src = TXT_SOURCE_HARDCODED;
}
else {
- // since this is not a publisher value
+ // since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
- else if (!publisher_get_opcode(dst, h, value)) {
+ else if (!provider_get_opcode(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@@ -252,6 +210,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
dst, WEVT_PREFIX_OPCODE, sizeof(WEVT_PREFIX_OPCODE) - 1, ev->opcode);
}
+// --------------------------------------------------------------------------------------------------------------------
+// Task
+
static const char *wevt_task_hardcoded(uint64_t task, size_t *len) {
switch(task) {
case WEVT_TASK_NONE: SET_LEN_AND_RETURN(WEVT_TASK_NAME_NONE);
@@ -267,9 +228,9 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageTask;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_TASK;
- bool is_publisher = is_valid_publisher_task(value, true);
+ bool is_provider = is_valid_provider_task(value, true);
- if(!is_publisher) {
+ if(!is_provider) {
size_t len;
const char *hardcoded = wevt_task_hardcoded(value, &len);
if(hardcoded) {
@@ -277,12 +238,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
dst->src = TXT_SOURCE_HARDCODED;
}
else {
- // since this is not a publisher value
+ // since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
- else if (!publisher_get_task(dst, h, value)) {
+ else if (!provider_get_task(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@@ -291,9 +252,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
dst, WEVT_PREFIX_TASK, sizeof(WEVT_PREFIX_TASK) - 1, ev->task);
}
+// --------------------------------------------------------------------------------------------------------------------
+// Keyword
+
#define SET_BITS(msk, txt) { .mask = msk, .name = txt, .len = sizeof(txt) - 1, }
-static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
+static uint64_t wevt_keyword_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
struct {
uint64_t mask;
const char *name;
@@ -324,7 +288,7 @@ static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
return value & 0x0000FFFFFFFFFFFF;
}
-static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
+static void wevt_get_keyword(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
TXT_UTF8 *dst = &log->ops.keywords;
if(ev->keywords == WEVT_KEYWORD_NONE) {
@@ -332,18 +296,18 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
dst->src = TXT_SOURCE_HARDCODED;
}
- uint64_t value = wevt_keywords_handle_reserved(ev->keywords, dst);
+ uint64_t value = wevt_keyword_handle_reserved(ev->keywords, dst);
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageKeyword;
- WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORDS;
+ WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORD;
if(!value && dst->used <= 1) {
// no hardcoded info in the buffer, make it None
txt_utf8_set(dst, WEVT_KEYWORD_NAME_NONE, sizeof(WEVT_KEYWORD_NAME_NONE) - 1);
dst->src = TXT_SOURCE_HARDCODED;
}
- else if (value && !publisher_get_keywords(dst, h, value) && dst->used <= 1) {
- // the publisher did not provide any info and the description is still empty.
+ else if (value && !provider_get_keywords(dst, h, value) && dst->used <= 1) {
+ // the provider did not provide any info and the description is still empty.
// the system returns 1 keyword, the highest bit, not a list
// so, when we call the system, we pass the original value (ev->keywords)
wevt_get_field_from_cache(log, ev->keywords, h, dst, &ev->provider, cache_type, flags);
@@ -353,57 +317,86 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
dst, WEVT_PREFIX_KEYWORDS, sizeof(WEVT_PREFIX_KEYWORDS) - 1, ev->keywords);
}
-bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
- bool ret = false;
+// --------------------------------------------------------------------------------------------------------------------
+// Fetching Events
- // obtain the information from selected events
+static inline bool wEvtRender(WEVT_LOG *log, EVT_HANDLE context, WEVT_VARIANT *raw) {
DWORD bytes_used = 0, property_count = 0;
- if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
+ if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
// information exceeds the allocated space
if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, hRenderContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %zu, extended info: %s",
- (uintptr_t)log->hRenderContext, (uintptr_t)log->hEvent, (uintptr_t)log->ops.content.data, log->ops.content.size, wevt_extended_status());
- goto cleanup;
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ "EvtRender() failed, hRenderSystemContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %u, extended info: %s",
+ (uintptr_t)context, (uintptr_t)log->hEvent, (uintptr_t)raw->data, raw->size,
+ EvtGetExtendedStatus_utf8());
+ return false;
}
- wevt_variant_resize(&log->ops.content, bytes_used);
- if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after bytes_used increase, extended info: %s",
- wevt_extended_status());
- goto cleanup;
+ wevt_variant_resize(raw, bytes_used);
+ if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ "EvtRender() failed, after bytes_used increase, extended info: %s",
+ EvtGetExtendedStatus_utf8());
+ return false;
}
}
- log->ops.content.used = bytes_used;
+ raw->used = bytes_used;
+ raw->count = property_count;
- EVT_VARIANT *content = log->ops.content.data;
+ return true;
+}
- ev->id = wevt_field_get_uint64(&content[FIELD_RECORD_NUMBER]);
- ev->event_id = wevt_field_get_uint16(&content[FIELD_EVENT_ID]);
- ev->level = wevt_field_get_uint8(&content[FIELD_LEVEL]);
- ev->opcode = wevt_field_get_uint8(&content[FIELD_OPCODE]);
- ev->keywords = wevt_field_get_uint64_hex(&content[FIELD_KEYWORDS]);
- ev->version = wevt_field_get_uint8(&content[FIELD_VERSION]);
- ev->task = wevt_field_get_uint16(&content[FIELD_TASK]);
- ev->process_id = wevt_field_get_uint32(&content[FIELD_PROCESS_ID]);
- ev->thread_id = wevt_field_get_uint32(&content[FIELD_THREAD_ID]);
- ev->created_ns = wevt_field_get_filetime_to_ns(&content[FIELD_TIME_CREATED]);
+static bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev) {
+ bool ret = false;
- if(full) {
- wevt_field_get_string_utf8(&content[FIELD_CHANNEL], &log->ops.channel);
- wevt_field_get_string_utf8(&content[FIELD_COMPUTER_NAME], &log->ops.computer);
- wevt_field_get_string_utf8(&content[FIELD_PROVIDER_NAME], &log->ops.provider);
- wevt_field_get_string_utf8(&content[FIELD_EVENT_SOURCE_NAME], &log->ops.source);
- wevt_get_uuid_by_type(&content[FIELD_PROVIDER_GUID], &ev->provider);
- wevt_get_uuid_by_type(&content[FIELD_CORRELATION_ACTIVITY_ID], &ev->correlation_activity_id);
- wevt_field_get_sid(&content[FIELD_USER_ID], &log->ops.user);
+ if(!wEvtRender(log, log->hRenderSystemContext, &log->ops.raw.system))
+ goto cleanup;
- PROVIDER_META_HANDLE *h = log->publisher =
- publisher_get(ev->provider, content[FIELD_PROVIDER_NAME].StringVal);
+ EVT_VARIANT *content = log->ops.raw.system.data;
- wevt_get_level(log, ev, h);
- wevt_get_task(log, ev, h);
- wevt_get_opcode(log, ev, h);
- wevt_get_keywords(log, ev, h);
+ ev->id = wevt_field_get_uint64(&content[EvtSystemEventRecordId]);
+ ev->event_id = wevt_field_get_uint16(&content[EvtSystemEventID]);
+ ev->level = wevt_field_get_uint8(&content[EvtSystemLevel]);
+ ev->opcode = wevt_field_get_uint8(&content[EvtSystemOpcode]);
+ ev->keywords = wevt_field_get_uint64_hex(&content[EvtSystemKeywords]);
+ ev->version = wevt_field_get_uint8(&content[EvtSystemVersion]);
+ ev->task = wevt_field_get_uint16(&content[EvtSystemTask]);
+ ev->qualifiers = wevt_field_get_uint16(&content[EvtSystemQualifiers]);
+ ev->process_id = wevt_field_get_uint32(&content[EvtSystemProcessID]);
+ ev->thread_id = wevt_field_get_uint32(&content[EvtSystemThreadID]);
+ ev->created_ns = wevt_field_get_filetime_to_ns(&content[EvtSystemTimeCreated]);
+
+ if(log->type & WEVT_QUERY_EXTENDED) {
+ wevt_field_get_string_utf8(&content[EvtSystemChannel], &log->ops.channel);
+ wevt_field_get_string_utf8(&content[EvtSystemComputer], &log->ops.computer);
+ wevt_field_get_string_utf8(&content[EvtSystemProviderName], &log->ops.provider);
+ wevt_get_uuid_by_type(&content[EvtSystemProviderGuid], &ev->provider);
+ wevt_get_uuid_by_type(&content[EvtSystemActivityID], &ev->activity_id);
+ wevt_get_uuid_by_type(&content[EvtSystemRelatedActivityID], &ev->related_activity_id);
+ wevt_field_get_sid(&content[EvtSystemUserID], &log->ops.account, &log->ops.domain, &log->ops.sid);
+
+ PROVIDER_META_HANDLE *p = log->provider =
+ provider_get(ev->provider, content[EvtSystemProviderName].StringVal);
+
+ ev->platform = provider_get_platform(p);
+
+ wevt_get_level(log, ev, p);
+ wevt_get_task(log, ev, p);
+ wevt_get_opcode(log, ev, p);
+ wevt_get_keyword(log, ev, p);
+
+ if(log->type & WEVT_QUERY_EVENT_DATA && wEvtRender(log, log->hRenderUserContext, &log->ops.raw.user)) {
+#if (ON_FTS_PRELOAD_MESSAGE == 1)
+ EvtFormatMessage_Event_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.event);
+#endif
+#if (ON_FTS_PRELOAD_XML == 1)
+ EvtFormatMessage_Xml_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.xml);
+#endif
+#if (ON_FTS_PRELOAD_EVENT_DATA == 1)
+ for(size_t i = 0; i < log->ops.raw.user.count ;i++)
+ evt_variant_to_buffer(log->ops.event_data, &log->ops.raw.user.data[i], " ||| ");
+#endif
+ }
}
ret = true;
@@ -412,11 +405,11 @@ cleanup:
return ret;
}
-bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
- DWORD size = full ? BATCH_NEXT_EVENT : 1;
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev) {
+ DWORD size = (log->type & WEVT_QUERY_EXTENDED) ? BATCH_NEXT_EVENT : 1;
DWORD max_failures = 10;
- fatal_assert(log && log->hQuery && log->hRenderContext);
+ fatal_assert(log && log->hQuery && log->hRenderSystemContext);
while(max_failures > 0) {
if (log->batch.used >= log->batch.size) {
@@ -433,7 +426,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
if(size == 1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtNext() failed, hQuery: 0x%lx, size: %zu, extended info: %s",
- (uintptr_t)log->hQuery, (size_t)size, wevt_extended_status());
+ (uintptr_t)log->hQuery, (size_t)size, EvtGetExtendedStatus_utf8());
return false;
}
@@ -455,7 +448,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
log->batch.hEvents[log->batch.used] = NULL;
log->batch.used++;
- if(wevt_get_next_event_one(log, ev, full))
+ if(wevt_get_next_event_one(log, ev))
return true;
else {
log->query_stats.failed_count++;
@@ -467,6 +460,69 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
return false;
}
+static void wevt_event_done(WEVT_LOG *log) {
+ if (log->provider) {
+ provider_release(log->provider);
+ log->provider = NULL;
+ }
+
+ if (log->hEvent) {
+ EvtClose(log->hEvent);
+ log->hEvent = NULL;
+ }
+
+ log->ops.channel.src = TXT_SOURCE_UNKNOWN;
+ log->ops.provider.src = TXT_SOURCE_UNKNOWN;
+ log->ops.computer.src = TXT_SOURCE_UNKNOWN;
+ log->ops.account.src = TXT_SOURCE_UNKNOWN;
+ log->ops.domain.src = TXT_SOURCE_UNKNOWN;
+ log->ops.sid.src = TXT_SOURCE_UNKNOWN;
+
+ log->ops.event.src = TXT_SOURCE_UNKNOWN;
+ log->ops.level.src = TXT_SOURCE_UNKNOWN;
+ log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
+ log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
+ log->ops.task.src = TXT_SOURCE_UNKNOWN;
+ log->ops.xml.src = TXT_SOURCE_UNKNOWN;
+
+ log->ops.channel.used = 0;
+ log->ops.provider.used = 0;
+ log->ops.computer.used = 0;
+ log->ops.account.used = 0;
+ log->ops.domain.used = 0;
+ log->ops.sid.used = 0;
+
+ log->ops.event.used = 0;
+ log->ops.level.used = 0;
+ log->ops.keywords.used = 0;
+ log->ops.opcode.used = 0;
+ log->ops.task.used = 0;
+ log->ops.xml.used = 0;
+
+ if(log->ops.event_data)
+ log->ops.event_data->len = 0;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Query management
+
+bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
+ wevt_query_done(log);
+ log->log_stats.queries_count++;
+
+ EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
+ if (!hQuery) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
+ query2utf8(query), EvtGetExtendedStatus_utf8());
+
+ log->log_stats.queries_failed++;
+ return false;
+ }
+
+ log->hQuery = hQuery;
+ return true;
+}
+
void wevt_query_done(WEVT_LOG *log) {
// close the last working hEvent
wevt_event_done(log);
@@ -490,19 +546,59 @@ void wevt_query_done(WEVT_LOG *log) {
log->query_stats.failed_count = 0;
}
+// --------------------------------------------------------------------------------------------------------------------
+// Log management
+
+WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type) {
+ WEVT_LOG *log = callocz(1, sizeof(*log));
+ log->type = type;
+
+ // create the system render
+ log->hRenderSystemContext = EvtCreateRenderContext(0, NULL, EvtRenderContextSystem);
+ if (!log->hRenderSystemContext) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ "EvtCreateRenderContext() on system context failed, extended info: %s",
+ EvtGetExtendedStatus_utf8());
+ goto cleanup;
+ }
+
+ if(type & WEVT_QUERY_EVENT_DATA) {
+ log->hRenderUserContext = EvtCreateRenderContext(0, NULL, EvtRenderContextUser);
+ if (!log->hRenderUserContext) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ "EvtCreateRenderContext failed, on user context failed, extended info: %s",
+ EvtGetExtendedStatus_utf8());
+ goto cleanup;
+ }
+
+ log->ops.event_data = buffer_create(4096, NULL);
+ }
+
+ return log;
+
+cleanup:
+ wevt_closelog6(log);
+ return NULL;
+}
+
void wevt_closelog6(WEVT_LOG *log) {
wevt_query_done(log);
- if (log->hRenderContext)
- EvtClose(log->hRenderContext);
+ if (log->hRenderSystemContext)
+ EvtClose(log->hRenderSystemContext);
- wevt_variant_cleanup(&log->ops.content);
+ if (log->hRenderUserContext)
+ EvtClose(log->hRenderUserContext);
+
+ wevt_variant_cleanup(&log->ops.raw.system);
+ wevt_variant_cleanup(&log->ops.raw.user);
txt_unicode_cleanup(&log->ops.unicode);
txt_utf8_cleanup(&log->ops.channel);
txt_utf8_cleanup(&log->ops.provider);
- txt_utf8_cleanup(&log->ops.source);
txt_utf8_cleanup(&log->ops.computer);
- txt_utf8_cleanup(&log->ops.user);
+ txt_utf8_cleanup(&log->ops.account);
+ txt_utf8_cleanup(&log->ops.domain);
+ txt_utf8_cleanup(&log->ops.sid);
txt_utf8_cleanup(&log->ops.event);
txt_utf8_cleanup(&log->ops.level);
@@ -510,9 +606,15 @@ void wevt_closelog6(WEVT_LOG *log) {
txt_utf8_cleanup(&log->ops.opcode);
txt_utf8_cleanup(&log->ops.task);
txt_utf8_cleanup(&log->ops.xml);
+
+ buffer_free(log->ops.event_data);
+
freez(log);
}
+// --------------------------------------------------------------------------------------------------------------------
+// Retention
+
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention) {
bool ret = false;
@@ -525,15 +627,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
if (!log->hQuery) {
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, cannot get retention, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
else
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, cannot get retention, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
- if (!wevt_get_next_event(log, &retention->first_event, false))
+ if (!wevt_get_next_event(log, &retention->first_event))
goto cleanup;
if (!retention->first_event.id) {
@@ -548,15 +650,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
if (!log->hQuery) {
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
else
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
- if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
+ if (!wevt_get_next_event(log, &retention->last_event) || retention->last_event.id == 0) {
// no data in eventlog
retention->last_event = retention->first_event;
}
@@ -582,24 +684,6 @@ cleanup:
return ret;
}
-WEVT_LOG *wevt_openlog6(void) {
- size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
-
- WEVT_LOG *log = callocz(1, sizeof(*log));
-
- // create the system render
- log->hRenderContext = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
- if (!log->hRenderContext) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed, extended info: %s", wevt_extended_status());
- freez(log);
- log = NULL;
- goto cleanup;
- }
-
-cleanup:
- return log;
-}
-
static uint64_t wevt_log_file_size(const wchar_t *channel) {
EVT_HANDLE hLog = NULL;
EVT_VARIANT evtVariant;
@@ -610,14 +694,14 @@ static uint64_t wevt_log_file_size(const wchar_t *channel) {
hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
if (!hLog) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
// Get the file size of the log
if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed, extended info: %s",
- channel2utf8(channel), wevt_extended_status());
+ channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
@@ -630,20 +714,3 @@ cleanup:
return file_size;
}
-
-bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
- wevt_query_done(log);
- log->log_stats.queries_count++;
-
- EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
- if (!hQuery) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
- query2utf8(query), wevt_extended_status());
-
- log->log_stats.queries_failed++;
- return false;
- }
-
- log->hQuery = hQuery;
- return true;
-}
diff --git a/src/collectors/windows-events.plugin/windows-events-query.h b/src/collectors/windows-events.plugin/windows-events-query.h
index c822da74b8..c43c3c9b50 100644
--- a/src/collectors/windows-events.plugin/windows-events-query.h
+++ b/src/collectors/windows-events.plugin/windows-events-query.h
@@ -4,6 +4,7 @@
#define NETDATA_WINDOWS_EVENTS_QUERY_H
#include "libnetdata/libnetdata.h"
+#include "windows-events.h"
#define BATCH_NEXT_EVENT 500
@@ -11,23 +12,27 @@ typedef struct wevt_event {
uint64_t id; // EventRecordId (unique and sequential per channel)
uint8_t version;
uint8_t level; // The severity of event
- uint8_t opcode; // we receive this as 8bit, but publishers use 32bit
+ uint8_t opcode; // we receive this as 8bit, but providers use 32bit
uint16_t event_id; // This is the template that defines the message to be shown
uint16_t task;
+ uint16_t qualifiers;
uint32_t process_id;
uint32_t thread_id;
uint64_t keywords; // Categorization of the event
ND_UUID provider;
- ND_UUID correlation_activity_id;
+ ND_UUID activity_id;
+ ND_UUID related_activity_id;
nsec_t created_ns;
+ WEVT_PROVIDER_PLATFORM platform;
} WEVT_EVENT;
#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
typedef struct {
EVT_VARIANT *data;
- size_t size;
- size_t used;
+ DWORD size;
+ DWORD used;
+ DWORD count;
} WEVT_VARIANT;
typedef struct {
@@ -41,6 +46,16 @@ typedef struct {
struct provider_meta_handle;
+typedef enum __attribute__((packed)) {
+ WEVT_QUERY_BASIC = (1 << 0),
+ WEVT_QUERY_EXTENDED = (1 << 1),
+ WEVT_QUERY_EVENT_DATA = (1 << 2),
+} WEVT_QUERY_TYPE;
+
+#define WEVT_QUERY_RETENTION WEVT_QUERY_BASIC
+#define WEVT_QUERY_NORMAL (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED)
+#define WEVT_QUERY_FTS (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED | WEVT_QUERY_EVENT_DATA)
+
typedef struct wevt_log {
struct {
DWORD size;
@@ -50,13 +65,19 @@ typedef struct wevt_log {
EVT_HANDLE hEvent;
EVT_HANDLE hQuery;
- EVT_HANDLE hRenderContext;
- struct provider_meta_handle *publisher;
+ EVT_HANDLE hRenderSystemContext;
+ EVT_HANDLE hRenderUserContext;
+ struct provider_meta_handle *provider;
+
+ WEVT_QUERY_TYPE type;
struct {
- // temp buffer used for rendering event log messages
- // never use directly
- WEVT_VARIANT content;
+ struct {
+ // temp buffer used for rendering event log messages
+ // never use directly
+ WEVT_VARIANT system;
+ WEVT_VARIANT user;
+ } raw;
// temp buffer used for fetching and converting UNICODE and UTF-8
// every string operation overwrites it, multiple times per event log entry
@@ -75,16 +96,19 @@ typedef struct wevt_log {
TXT_UTF8 channel;
TXT_UTF8 provider;
- TXT_UTF8 source;
TXT_UTF8 computer;
- TXT_UTF8 user;
+ TXT_UTF8 account;
+ TXT_UTF8 domain;
+ TXT_UTF8 sid;
- TXT_UTF8 event;
+ TXT_UTF8 event; // the message to be shown to the user
TXT_UTF8 level;
TXT_UTF8 keywords;
TXT_UTF8 opcode;
TXT_UTF8 task;
TXT_UTF8 xml;
+
+ BUFFER *event_data;
} ops;
struct {
@@ -102,7 +126,7 @@ typedef struct wevt_log {
} WEVT_LOG;
-WEVT_LOG *wevt_openlog6(void);
+WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type);
void wevt_closelog6(WEVT_LOG *log);
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention);
@@ -110,12 +134,14 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction);
void wevt_query_done(WEVT_LOG *log);
-bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev);
-bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
+bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
-bool wevt_get_event_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
-bool wevt_get_xml_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
+bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
+bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
+
+void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator);
static inline void wevt_variant_cleanup(WEVT_VARIANT *v) {
freez(v->data);
@@ -130,6 +156,10 @@ static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) {
v->data = mallocz(v->size);
}
+static inline void wevt_variant_count_from_used(WEVT_VARIANT *v) {
+ v->count = v->used / sizeof(*v->data);
+}
+
static inline uint8_t wevt_field_get_uint8(EVT_VARIANT *ev) {
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull)
return 0;
@@ -180,16 +210,17 @@ static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) {
return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1);
}
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
-
-static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst) {
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
+static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
- wevt_utf8_empty(dst);
+ wevt_utf8_empty(dst_account);
+ wevt_utf8_empty(dst_domain);
+ wevt_utf8_empty(dst_sid_str);
return false;
}
fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid);
- return wevt_convert_user_id_to_name(ev->SidVal, dst);
+ return wevt_convert_user_id_to_name(ev->SidVal, dst_account, dst_domain, dst_sid_str);
}
static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) {
@@ -222,42 +253,42 @@ static inline bool wevt_get_uuid_by_type(EVT_VARIANT *ev, ND_UUID *dst) {
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-severity-levels
-static inline bool is_valid_publisher_level(uint64_t level, bool strict) {
+static inline bool is_valid_provider_level(uint64_t level, bool strict) {
if(strict)
- // when checking if the name is publisher independent
+ // when checking if the name is provider independent
return level >= 16 && level <= 255;
else
- // when checking acceptable values in publisher manifests
+ // when checking acceptable values in provider manifests
return level <= 255;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
-static inline bool is_valid_publisher_opcode(uint64_t opcode, bool strict) {
+static inline bool is_valid_provider_opcode(uint64_t opcode, bool strict) {
if(strict)
- // when checking if the name is publisher independent
+ // when checking if the name is provider independent
return opcode >= 10 && opcode <= 239;
else
- // when checking acceptable values in publisher manifests
+ // when checking acceptable values in provider manifests
return opcode <= 255;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
-static inline bool is_valid_publisher_task(uint64_t task, bool strict) {
+static inline bool is_valid_provider_task(uint64_t task, bool strict) {
if(strict)
- // when checking if the name is publisher independent
+ // when checking if the name is provider independent
return task > 0 && task <= 0xFFFF;
else
- // when checking acceptable values in publisher manifests
+ // when checking acceptable values in provider manifests
return task <= 0xFFFF;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-keywords-used-to-classify-types-of-events
-static inline bool is_valid_publisher_keywords(uint64_t keyword, bool strict) {
+static inline bool is_valid_provider_keyword(uint64_t keyword, bool strict) {
if(strict)
- // when checking if the name is publisher independent
+ // when checking if the name is provider independent
return keyword > 0 && keyword <= 0x0000FFFFFFFFFFFF;
else
- // when checking acceptable values in publisher manifests
+ // when checking acceptable values in provider manifests
return true;
}
diff --git a/src/collectors/windows-events.plugin/windows-events-sid.c b/src/collectors/windows-events.plugin/windows-events-sid.c
index 6aefd22698..8f92fb495d 100644
--- a/src/collectors/windows-events.plugin/windows-events-sid.c
+++ b/src/collectors/windows-events.plugin/windows-events-sid.c
@@ -9,8 +9,18 @@ typedef struct {
} SID_KEY;
typedef struct {
- const char *user;
- size_t user_len;
+ // IMPORTANT:
+ // This is malloc'd ! You have to manually set fields to zero.
+
+ const char *account;
+ const char *domain;
+ const char *full;
+ const char *sid_str;
+
+ uint32_t account_len;
+ uint32_t domain_len;
+ uint32_t full_len;
+ uint32_t sid_str_len;
// this needs to be last, because of its variable size
SID_KEY key;
@@ -43,48 +53,46 @@ void sid_cache_init(void) {
simple_hashtable_init_SID(&sid_globals.hashtable, 100);
}
-static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
- if(found && found->user) {
- txt_utf8_resize(dst, found->user_len + 1, false);
- memcpy(dst->data, found->user, found->user_len + 1);
- dst->used = found->user_len + 1;
- return true;
- }
-
- txt_utf8_resize(dst, 1, false);
- dst->data[0] = '\0';
- dst->used = 1;
- return false;
-}
-
-static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
+static void lookup_user(SID_VALUE *sv) {
static __thread wchar_t account_unicode[256];
static __thread wchar_t domain_unicode[256];
+ static __thread char tmp[512 + 2];
+
DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
SID_NAME_USE sid_type;
- txt_utf8_resize(dst, 1024, false);
-
- if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
- const char *user = account2utf8(account_unicode);
+ if (LookupAccountSidW(NULL, sv->key.sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
+ const char *account = account2utf8(account_unicode);
const char *domain = domain2utf8(domain_unicode);
- dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
+ snprintfz(tmp, sizeof(tmp), "%s\\%s", domain, account);
+ sv->domain = strdupz(domain); sv->domain_len = strlen(sv->domain);
+ sv->account = strdupz(account); sv->account_len = strlen(sv->account);
+ sv->full = strdupz(tmp); sv->full_len = strlen(sv->full);
}
else {
- wchar_t *sid_string = NULL;
- if (ConvertSidToStringSidW(sid, &sid_string)) {
- const char *user = account2utf8(sid_string);
- dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
- }
- else
- dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
+ sv->domain = NULL;
+ sv->account = NULL;
+ sv->full = NULL;
+ sv->domain_len = 0;
+ sv->account_len = 0;
+ sv->full_len = 0;
+ }
+
+ wchar_t *sid_string = NULL;
+ if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) {
+ sv->sid_str = strdupz(account2utf8(sid_string));
+ sv->sid_str_len = strlen(sv->sid_str);
+ }
+ else {
+ sv->sid_str = NULL;
+ sv->sid_str_len = 0;
}
}
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
+static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) {
if(!sid || !IsValidSid(sid))
- return update_user(NULL, dst);
+ return NULL;
size_t size = GetLengthSid(sid);
@@ -98,21 +106,76 @@ bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
spinlock_lock(&sid_globals.spinlock);
SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
spinlock_unlock(&sid_globals.spinlock);
- if(found) return update_user(found, dst);
+ if(found) return found;
// allocate the SID_VALUE
found = mallocz(tmp_size);
memcpy(found, buf, tmp_size);
- // lookup the user
- lookup_user(sid, dst);
- found->user = strdupz(dst->data);
- found->user_len = dst->used - 1;
+ lookup_user(found);
// add it to the cache
spinlock_lock(&sid_globals.spinlock);
simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
spinlock_unlock(&sid_globals.spinlock);
- return update_user(found, dst);
+ return found;
+}
+
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
+ SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
+
+ if(found) {
+ if (found->account) {
+ txt_utf8_resize(dst_account, found->account_len + 1, false);
+ memcpy(dst_account->data, found->account, found->account_len + 1);
+ dst_account->used = found->account_len + 1;
+ }
+ else wevt_utf8_empty(dst_account);
+
+ if (found->domain) {
+ txt_utf8_resize(dst_domain, found->domain_len + 1, false);
+ memcpy(dst_domain->data, found->domain, found->domain_len + 1);
+ dst_domain->used = found->domain_len + 1;
+ }
+ else wevt_utf8_empty(dst_domain);
+
+ if (found->sid_str) {
+ txt_utf8_resize(dst_sid_str, found->sid_str_len + 1, false);
+ memcpy(dst_sid_str->data, found->sid_str, found->sid_str_len + 1);
+ dst_sid_str->used = found->sid_str_len + 1;
+ }
+ else wevt_utf8_empty(dst_sid_str);
+
+ return true;
+ }
+
+ wevt_utf8_empty(dst_account);
+ wevt_utf8_empty(dst_domain);
+ wevt_utf8_empty(dst_sid_str);
+ return false;
+}
+
+bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix) {
+ SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
+ size_t added = 0;
+
+ if(found) {
+ if (found->full) {
+ if (prefix && *prefix)
+ buffer_strcat(dst, prefix);
+
+ buffer_fast_strcat(dst, found->full, found->full_len);
+ added++;
+ }
+ if (found->sid_str) {
+ if (prefix && *prefix)
+ buffer_strcat(dst, prefix);
+
+ buffer_fast_strcat(dst, found->sid_str, found->sid_str_len);
+ added++;
+ }
+ }
+
+ return added > 0;
}
diff --git a/src/collectors/windows-events.plugin/windows-events-sid.h b/src/collectors/windows-events.plugin/windows-events-sid.h
index 8723ade58c..aca8e77e34 100644
--- a/src/collectors/windows-events.plugin/windows-events-sid.h
+++ b/src/collectors/windows-events.plugin/windows-events-sid.h
@@ -6,7 +6,8 @@
#include "windows-events.h"
struct wevt_log;
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
+bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix);
void sid_cache_init(void);
#endif //NETDATA_WINDOWS_EVENTS_SID_H
diff --git a/src/collectors/windows-events.plugin/windows-events-sources.c b/src/collectors/windows-events.plugin/windows-events-sources.c
index 204e01ca9a..b931ed059f 100644
--- a/src/collectors/windows-events.plugin/windows-events-sources.c
+++ b/src/collectors/windows-events.plugin/windows-events-sources.c
@@ -137,41 +137,32 @@
// }
//};
+ENUM_STR_MAP_DEFINE(WEVT_SOURCE_TYPE) = {
+ { .id = WEVTS_ALL, .name = WEVT_SOURCE_ALL_NAME },
+ { .id = WEVTS_ADMIN, .name = WEVT_SOURCE_ALL_ADMIN_NAME },
+ { .id = WEVTS_OPERATIONAL, .name = WEVT_SOURCE_ALL_OPERATIONAL_NAME },
+ { .id = WEVTS_ANALYTIC, .name = WEVT_SOURCE_ALL_ANALYTIC_NAME },
+ { .id = WEVTS_DEBUG, .name = WEVT_SOURCE_ALL_DEBUG_NAME },
+ { .id = WEVTS_WINDOWS, .name = WEVT_SOURCE_ALL_WINDOWS_NAME },
+ { .id = WEVTS_ENABLED, .name = WEVT_SOURCE_ALL_ENABLED_NAME },
+ { .id = WEVTS_DISABLED, .name = WEVT_SOURCE_ALL_DISABLED_NAME },
+ { .id = WEVTS_FORWARDED, .name = WEVT_SOURCE_ALL_FORWARDED_NAME },
+ { .id = WEVTS_CLASSIC, .name = WEVT_SOURCE_ALL_CLASSIC_NAME },
+ { .id = WEVTS_BACKUP_MODE, .name = WEVT_SOURCE_ALL_BACKUP_MODE_NAME },
+ { .id = WEVTS_OVERWRITE_MODE, .name = WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME },
+ { .id = WEVTS_STOP_WHEN_FULL_MODE, .name = WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME },
+ { .id = WEVTS_RETAIN_AND_BACKUP_MODE, .name = WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME },
+
+ // terminator
+ { . id = 0, .name = NULL }
+};
+
+BITMAP_STR_DEFINE_FUNCTIONS(WEVT_SOURCE_TYPE, WEVTS_NONE, "");
+
DICTIONARY *wevt_sources = NULL;
DICTIONARY *used_hashes_registry = NULL;
static usec_t wevt_session = 0;
-WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
- if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
- return WEVTS_ALL;
-
- if(strcmp(value, WEVT_SOURCE_ALL_ADMIN_NAME) == 0)
- return WEVTS_ADMIN;
-
- if(strcmp(value, WEVT_SOURCE_ALL_OPERATIONAL_NAME) == 0)
- return WEVTS_OPERATIONAL;
-
- if(strcmp(value, WEVT_SOURCE_ALL_ANALYTIC_NAME) == 0)
- return WEVTS_ANALYTIC;
-
- if(strcmp(value, WEVT_SOURCE_ALL_DEBUG_NAME) == 0)
- return WEVTS_DEBUG;
-
- if(strcmp(value, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME) == 0)
- return WEVTS_DIAGNOSTIC;
-
- if(strcmp(value, WEVT_SOURCE_ALL_TRACING_NAME) == 0)
- return WEVTS_TRACING;
-
- if(strcmp(value, WEVT_SOURCE_ALL_PERFORMANCE_NAME) == 0)
- return WEVTS_PERFORMANCE;
-
- if(strcmp(value, WEVT_SOURCE_ALL_WINDOWS_NAME) == 0)
- return WEVTS_WINDOWS;
-
- return WEVTS_NONE;
-}
-
void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
LOGS_QUERY_SOURCE *src = value;
freez((void *)src->fullname);
@@ -265,7 +256,14 @@ int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
// --------------------------------------------------------------------------------------------------------------------
+typedef enum {
+ wevt_source_type_internal,
+ wevt_source_type_provider,
+ wevt_source_type_channel,
+} wevt_source_type;
+
struct wevt_source {
+ wevt_source_type type;
usec_t first_ut;
usec_t last_ut;
size_t count;
@@ -279,6 +277,15 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
const char *name = dictionary_acquired_item_name(item);
+ if(s->count == 1 && strncmp(name, WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX, sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) - 1) == 0)
+ // do not include "All-Of-X" when there is only 1 channel
+ return 0;
+
+ bool default_selected = (s->type == wevt_source_type_channel);
+ if(default_selected && (strcmp(name, "NetdataWEL") == 0 || strcmp(name, "Netdata/Access") == 0))
+ // do not select Netdata Access logs by default
+ default_selected = false;
+
buffer_json_add_array_item_object(wb);
{
char size_for_humans[128];
@@ -300,6 +307,7 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
buffer_json_member_add_string(wb, "name", name);
buffer_json_member_add_string(wb, "pill", size_for_humans);
buffer_json_member_add_string(wb, "info", info);
+ buffer_json_member_add_boolean(wb, "default_selected", default_selected);
}
buffer_json_object_close(wb); // options object
@@ -337,61 +345,142 @@ void wevt_sources_to_json_array(BUFFER *wb) {
t.size = src->size;
t.entries = src->entries;
- dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
+ src->source_type |= WEVTS_ALL;
+ t.type = wevt_source_type_internal;
+ for(size_t i = 0; WEVT_SOURCE_TYPE_names[i].name ;i++) {
+ if(src->source_type & WEVT_SOURCE_TYPE_names[i].id)
+ dictionary_set(dict, WEVT_SOURCE_TYPE_names[i].name, &t, sizeof(t));
+ }
- if(src->source_type & WEVTS_ADMIN)
- dictionary_set(dict, WEVT_SOURCE_ALL_ADMIN_NAME, &t, sizeof(t));
+ if(src->provider) {
+ t.type = wevt_source_type_provider;
+ dictionary_set(dict, string2str(src->provider), &t, sizeof(t));
+ }
- if(src->source_type & WEVTS_OPERATIONAL)
- dictionary_set(dict, WEVT_SOURCE_ALL_OPERATIONAL_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_ANALYTIC)
- dictionary_set(dict, WEVT_SOURCE_ALL_ANALYTIC_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_DEBUG)
- dictionary_set(dict, WEVT_SOURCE_ALL_DEBUG_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_DIAGNOSTIC)
- dictionary_set(dict, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_TRACING)
- dictionary_set(dict, WEVT_SOURCE_ALL_TRACING_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_PERFORMANCE)
- dictionary_set(dict, WEVT_SOURCE_ALL_PERFORMANCE_NAME, &t, sizeof(t));
-
- if(src->source_type & WEVTS_WINDOWS)
- dictionary_set(dict, WEVT_SOURCE_ALL_WINDOWS_NAME, &t, sizeof(t));
-
- if(src->source)
+ if(src->source) {
+ t.type = wevt_source_type_channel;
dictionary_set(dict, string2str(src->source), &t, sizeof(t));
+ }
}
dfe_done(jf);
dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
}
-static bool check_and_remove_suffix(char *name, size_t len, const char *suffix) {
- char s[strlen(suffix) + 2];
- s[0] = '/';
- memcpy(&s[1], suffix, sizeof(s) - 1);
- size_t slen = sizeof(s) - 1;
-
- if(slen + 1 >= len) return false;
-
- char *match = &name[len - slen];
- if(strcasecmp(match, s) == 0) {
- *match = '\0';
- return true;
+static bool ndEvtGetChannelConfigProperty(EVT_HANDLE hChannelConfig, WEVT_VARIANT *pr, EVT_CHANNEL_CONFIG_PROPERTY_ID id) {
+ if (!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
+ DWORD status = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER == status) {
+ wevt_variant_resize(pr, pr->used);
+ if(!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
+ pr->used = 0;
+ pr->count = 0;
+ return false;
+ }
+ }
}
- s[0] = '-';
- if(strcasecmp(match, s) == 0) {
- *match = '\0';
- return true;
+ wevt_variant_count_from_used(pr);
+ return true;
+}
+
+WEVT_SOURCE_TYPE categorize_channel(const wchar_t *channel_path, const char **provider, WEVT_VARIANT *property) {
+ EVT_HANDLE hChannelConfig = NULL;
+ WEVT_SOURCE_TYPE result = WEVTS_ALL;
+
+ // Open the channel configuration
+ hChannelConfig = EvtOpenChannelConfig(NULL, channel_path, 0);
+ if (!hChannelConfig)
+ goto cleanup;
+
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigType) &
+ property->count &&
+ property->data[0].Type == EvtVarTypeUInt32) {
+ switch (property->data[0].UInt32Val) {
+ case EvtChannelTypeAdmin:
+ result |= WEVTS_ADMIN;
+ break;
+
+ case EvtChannelTypeOperational:
+ result |= WEVTS_OPERATIONAL;
+ break;
+
+ case EvtChannelTypeAnalytic:
+ result |= WEVTS_ANALYTIC;
+ break;
+
+ case EvtChannelTypeDebug:
+ result |= WEVTS_DEBUG;
+ break;
+
+ default:
+ break;
+ }
}
- return false;
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigClassicEventlog) &&
+ property->count &&
+ property->data[0].Type == EvtVarTypeBoolean &&
+ property->data[0].BooleanVal)
+ result |= WEVTS_CLASSIC;
+
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigOwningPublisher) &&
+ property->count &&
+ property->data[0].Type == EvtVarTypeString) {
+ *provider = provider2utf8(property->data[0].StringVal);
+ if(wcscasecmp(property->data[0].StringVal, L"Microsoft-Windows-EventCollector") == 0)
+ result |= WEVTS_FORWARDED;
+ }
+ else
+ *provider = NULL;
+
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigEnabled) &&
+ property->count &&
+ property->data[0].Type == EvtVarTypeBoolean) {
+ if(property->data[0].BooleanVal)
+ result |= WEVTS_ENABLED;
+ else
+ result |= WEVTS_DISABLED;
+ }
+
+ bool got_retention = false;
+ bool retained = false;
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigRetention) &&
+ property->count &&
+ property->data[0].Type == EvtVarTypeBoolean) {
+ got_retention = true;
+ retained = property->data[0].BooleanVal;
+ }
+
+ bool got_auto_backup = false;
+ bool auto_backup = false;
+ if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigAutoBackup) &&
+ property->count &&
+ property->data[0].Type == EvtVarTypeBoolean) {
+ got_auto_backup = true;
+ auto_backup = property->data[0].BooleanVal;
+ }
+
+ if(got_retention && got_auto_backup) {
+ if(!retained) {
+ if(auto_backup)
+ result |= WEVTS_BACKUP_MODE;
+ else
+ result |= WEVTS_OVERWRITE_MODE;
+ }
+ else {
+ if(auto_backup)
+ result |= WEVTS_STOP_WHEN_FULL_MODE;
+ else
+ result |= WEVTS_RETAIN_AND_BACKUP_MODE;
+ }
+ }
+
+cleanup:
+ if (hChannelConfig)
+ EvtClose(hChannelConfig);
+
+ return result;
}
void wevt_sources_scan(void) {
@@ -400,8 +489,9 @@ void wevt_sources_scan(void) {
EVT_HANDLE hChannelEnum = NULL;
if(spinlock_trylock(&spinlock)) {
- const usec_t now_monotonic_ut = now_monotonic_usec();
+ const usec_t started_ut = now_monotonic_usec();
+ WEVT_VARIANT property = { 0 };
DWORD dwChannelBufferSize = 0;
DWORD dwChannelBufferUsed = 0;
DWORD status = ERROR_SUCCESS;
@@ -414,7 +504,7 @@ void wevt_sources_scan(void) {
goto cleanup;
}
- WEVT_LOG *log = wevt_openlog6();
+ WEVT_LOG *log = wevt_openlog6(WEVT_QUERY_RETENTION);
if(!log) goto cleanup;
while (true) {
@@ -438,32 +528,26 @@ void wevt_sources_scan(void) {
if(!wevt_channel_retention(log, channel, NULL, &retention))
continue;
+ LOGS_QUERY_SOURCE *found = dictionary_get(wevt_sources, channel2utf8(channel));
+ if(found) {
+ // we just need to update its retention
+
+ found->last_scan_monotonic_ut = now_monotonic_usec();
+ found->msg_first_id = retention.first_event.id;
+ found->msg_last_id = retention.last_event.id;
+ found->msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC;
+ found->msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC;
+ found->size = retention.size_bytes;
+ continue;
+ }
+
const char *name = channel2utf8(channel);
const char *fullname = strdupz(name);
+ const char *provider;
- WEVT_SOURCE_TYPE sources = WEVTS_ALL;
- size_t len = strlen(fullname);
- if(check_and_remove_suffix((char *)name, len, "Admin"))
- sources |= WEVTS_ADMIN;
- else if(check_and_remove_suffix((char *)name, len, "Operational"))
- sources |= WEVTS_OPERATIONAL;
- else if(check_and_remove_suffix((char *)name, len, "Analytic"))
- sources |= WEVTS_ANALYTIC;
- else if(check_and_remove_suffix((char *)name, len, "Debug") ||
- check_and_remove_suffix((char *)name, len, "Verbose"))
- sources |= WEVTS_DEBUG;
- else if(check_and_remove_suffix((char *)name, len, "Diagnostic"))
- sources |= WEVTS_DIAGNOSTIC;
- else if(check_and_remove_suffix((char *)name, len, "Trace") ||
- check_and_remove_suffix((char *)name, len, "Tracing"))
- sources |= WEVTS_TRACING;
- else if(check_and_remove_suffix((char *)name, len, "Performance") ||
- check_and_remove_suffix((char *)name, len, "Perf"))
- sources |= WEVTS_PERFORMANCE;
-
+ WEVT_SOURCE_TYPE sources = categorize_channel(channel, &provider, &property);
char *slash = strchr(name, '/');
- if(slash)
- *slash = '\0';
+ if(slash) *slash = '\0';
if(strcasecmp(name, "Application") == 0)
sources |= WEVTS_WINDOWS;
@@ -485,9 +569,27 @@ void wevt_sources_scan(void) {
.msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
.size = retention.size_bytes,
.source_type = sources,
- .source = string_strdupz(name),
+ .source = string_strdupz(fullname),
};
+ if(strncmp(fullname, "Netdata", 7) == 0)
+ // WEL based providers of Netdata are named NetdataX
+ provider = "Netdata";
+
+ if(provider && *provider) {
+ char buf[sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) + strlen(provider)]; // sizeof() includes terminator
+ snprintf(buf, sizeof(buf), WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "%s", provider);
+
+ if(trim_all(buf) != NULL) {
+ for (size_t i = 0; i < sizeof(buf) - 1; i++) {
+ // remove character that may interfere with our parsing
+ if (isspace((uint8_t) buf[i]) || buf[i] == '%' || buf[i] == '+' || buf[i] == '|' || buf[i] == ':')
+ buf[i] = '_';
+ }
+ src.provider = string_strdupz(buf);
+ }
+ }
+
dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
}
@@ -519,13 +621,21 @@ void wevt_sources_scan(void) {
LOGS_QUERY_SOURCE *src;
dfe_start_write(wevt_sources, src)
{
- if(src->last_scan_monotonic_ut < now_monotonic_ut)
+ if(src->last_scan_monotonic_ut < started_ut) {
+ src->msg_first_id = 0;
+ src->msg_last_id = 0;
+ src->msg_first_ut = 0;
+ src->msg_last_ut = 0;
+ src->size = 0;
dictionary_del(wevt_sources, src->fullname);
+ }
}
dfe_done(src);
dictionary_garbage_collect(wevt_sources);
spinlock_unlock(&spinlock);
+
+ wevt_variant_cleanup(&property);
}
cleanup:
diff --git a/src/collectors/windows-events.plugin/windows-events-sources.h b/src/collectors/windows-events.plugin/windows-events-sources.h
index 6ee6ee4c03..4ad4880d7d 100644
--- a/src/collectors/windows-events.plugin/windows-events-sources.h
+++ b/src/collectors/windows-events.plugin/windows-events-sources.h
@@ -6,18 +6,42 @@
#include "libnetdata/libnetdata.h"
typedef enum {
- WEVTS_NONE = 0,
- WEVTS_ALL = (1 << 0),
- WEVTS_ADMIN = (1 << 1),
- WEVTS_OPERATIONAL = (1 << 2),
- WEVTS_ANALYTIC = (1 << 3),
- WEVTS_DEBUG = (1 << 4),
- WEVTS_DIAGNOSTIC = (1 << 5),
- WEVTS_TRACING = (1 << 6),
- WEVTS_PERFORMANCE = (1 << 7),
- WEVTS_WINDOWS = (1 << 8),
+ WEVTS_NONE = 0,
+ WEVTS_ALL = (1 << 0),
+ WEVTS_ADMIN = (1 << 1),
+ WEVTS_OPERATIONAL = (1 << 2),
+ WEVTS_ANALYTIC = (1 << 3),
+ WEVTS_DEBUG = (1 << 4),
+ WEVTS_WINDOWS = (1 << 5),
+ WEVTS_ENABLED = (1 << 6),
+ WEVTS_DISABLED = (1 << 7),
+ WEVTS_FORWARDED = (1 << 8),
+ WEVTS_CLASSIC = (1 << 9),
+ WEVTS_BACKUP_MODE = (1 << 10),
+ WEVTS_OVERWRITE_MODE = (1 << 11),
+ WEVTS_STOP_WHEN_FULL_MODE = (1 << 12),
+ WEVTS_RETAIN_AND_BACKUP_MODE = (1 << 13),
} WEVT_SOURCE_TYPE;
+BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(WEVT_SOURCE_TYPE)
+
+#define WEVT_SOURCE_ALL_NAME "All"
+#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
+#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
+#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
+#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
+#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
+#define WEVT_SOURCE_ALL_ENABLED_NAME "All-Enabled"
+#define WEVT_SOURCE_ALL_DISABLED_NAME "All-Disabled"
+#define WEVT_SOURCE_ALL_FORWARDED_NAME "All-Forwarded"
+#define WEVT_SOURCE_ALL_CLASSIC_NAME "All-Classic"
+#define WEVT_SOURCE_ALL_BACKUP_MODE_NAME "All-In-Backup-Mode"
+#define WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME "All-In-Overwrite-Mode"
+#define WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME "All-In-StopWhenFull-Mode"
+#define WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME "All-In-RetainAndBackup-Mode"
+
+#define WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "All-Of-"
+
typedef struct {
const char *fullname;
size_t fullname_len;
@@ -25,6 +49,7 @@ typedef struct {
const wchar_t *custom_query;
STRING *source;
+ STRING *provider;
WEVT_SOURCE_TYPE source_type;
usec_t msg_first_ut;
usec_t msg_last_ut;
@@ -40,16 +65,6 @@ typedef struct {
extern DICTIONARY *wevt_sources;
extern DICTIONARY *used_hashes_registry;
-#define WEVT_SOURCE_ALL_NAME "All"
-#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
-#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
-#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
-#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
-#define WEVT_SOURCE_ALL_DIAGNOSTIC_NAME "All-Diagnostic"
-#define WEVT_SOURCE_ALL_TRACING_NAME "All-Tracing"
-#define WEVT_SOURCE_ALL_PERFORMANCE_NAME "All-Performance"
-#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
-
void wevt_sources_init(void);
void wevt_sources_scan(void);
void buffer_json_wevt_versions(BUFFER *wb);
diff --git a/src/collectors/windows-events.plugin/windows-events-unicode.c b/src/collectors/windows-events.plugin/windows-events-unicode.c
index 3208ee774f..f84418dc76 100644
--- a/src/collectors/windows-events.plugin/windows-events-unicode.c
+++ b/src/collectors/windows-events.plugin/windows-events-unicode.c
@@ -5,7 +5,7 @@
inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
if (src) {
// Convert from UTF-8 to wide char (UTF-16)
- if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
+ if (utf8_to_utf16(dst, dst_size, src, -1) == 0)
wcsncpy(dst, L"[failed conv.]", dst_size - 1);
}
else
@@ -41,7 +41,7 @@ char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) {
wchar_t *channel2unicode(const char *utf8str) {
static __thread wchar_t buffer[1024];
- utf82unicode(buffer, sizeof(buffer) / sizeof(wchar_t), utf8str);
+ utf82unicode(buffer, _countof(buffer), utf8str);
return buffer;
}
@@ -69,6 +69,12 @@ char *query2utf8(const wchar_t *query) {
return buffer;
}
+char *provider2utf8(const wchar_t *provider) {
+ static __thread char buffer[256];
+ unicode2utf8(buffer, sizeof(buffer), provider);
+ return buffer;
+}
+
bool wevt_str_wchar_to_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len_with_null) {
if(!src || !src_len_with_null)
goto cleanup;
diff --git a/src/collectors/windows-events.plugin/windows-events-unicode.h b/src/collectors/windows-events.plugin/windows-events-unicode.h
index fe11d04a15..fa2f4a49eb 100644
--- a/src/collectors/windows-events.plugin/windows-events-unicode.h
+++ b/src/collectors/windows-events.plugin/windows-events-unicode.h
@@ -9,7 +9,7 @@
typedef enum __attribute__((packed)) {
TXT_SOURCE_UNKNOWN = 0,
- TXT_SOURCE_PUBLISHER,
+ TXT_SOURCE_PROVIDER,
TXT_SOURCE_FIELD_CACHE,
TXT_SOURCE_EVENT_LOG,
TXT_SOURCE_HARDCODED,
@@ -177,6 +177,7 @@ char *channel2utf8(const wchar_t *channel);
wchar_t *channel2unicode(const char *utf8str);
char *query2utf8(const wchar_t *query);
+char *provider2utf8(const wchar_t *provider);
char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len);
diff --git a/src/collectors/windows-events.plugin/windows-events.c b/src/collectors/windows-events.plugin/windows-events.c
index 24f8f59a59..14ca9d6cea 100644
--- a/src/collectors/windows-events.plugin/windows-events.c
+++ b/src/collectors/windows-events.plugin/windows-events.c
@@ -18,14 +18,17 @@ static bool plugin_should_exit = false;
#define WEVT_KEYS_INCLUDED_IN_FACETS \
"|" WEVT_FIELD_COMPUTER \
"|" WEVT_FIELD_PROVIDER \
- "|" WEVT_FIELD_SOURCE \
"|" WEVT_FIELD_LEVEL \
"|" WEVT_FIELD_KEYWORDS \
"|" WEVT_FIELD_OPCODE \
"|" WEVT_FIELD_TASK \
- "|" WEVT_FIELD_USER \
+ "|" WEVT_FIELD_ACCOUNT \
+ "|" WEVT_FIELD_DOMAIN \
+ "|" WEVT_FIELD_SID \
""
+#define query_has_fts(lqs) ((lqs)->rq.query != NULL)
+
static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
nd_log(NDLS_COLLECTORS, NDLP_INFO, "Function has been cancelled");
@@ -66,9 +69,10 @@ FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused,
struct wevt_bin_data {
bool rendered;
+ WEVT_EVENT ev;
WEVT_LOG *log;
EVT_HANDLE hEvent;
- PROVIDER_META_HANDLE *publisher;
+ PROVIDER_META_HANDLE *provider;
};
static void wevt_cleanup_bin_data(void *data) {
@@ -77,21 +81,30 @@ static void wevt_cleanup_bin_data(void *data) {
if(d->hEvent)
EvtClose(d->hEvent);
- publisher_release(d->publisher);
+ provider_release(d->provider);
freez(d);
}
-static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev __maybe_unused) {
+static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev) {
struct wevt_bin_data *d = mallocz(sizeof(struct wevt_bin_data));
+#ifdef NETDATA_INTERNAL_CHECKS
+ internal_fatal(strcmp(log->ops.provider.data, provider_get_name(log->provider)) != 0,
+ "Provider name mismatch in data!");
+
+ internal_fatal(!UUIDeq(ev->provider, provider_get_uuid(log->provider)),
+ "Provider UUID mismatch in data!");
+#endif
+
+ d->ev = *ev;
d->log = log;
d->rendered = false;
// take the bookmark
d->hEvent = log->hEvent; log->hEvent = NULL;
- // dup the publisher
- d->publisher = publisher_dup(log->publisher);
+ // dup the provider
+ d->provider = provider_dup(log->provider);
facets_row_bin_data_set(facets, wevt_cleanup_bin_data, d);
}
@@ -99,12 +112,26 @@ static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets,
static void wevt_lazy_loading_event_and_xml(struct wevt_bin_data *d, FACET_ROW *row __maybe_unused) {
if(d->rendered) return;
- wevt_get_xml_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.xml);
- wevt_get_event_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.event);
+#ifdef NETDATA_INTERNAL_CHECKS
+ const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
+ internal_fatal(!provider_rkv || strcmp(buffer_tostring(provider_rkv->wb), provider_get_name(d->provider)) != 0,
+ "Provider of row does not match the bin data associated with it");
+
+ uint64_t event_record_id = UINT64_MAX;
+ const FACET_ROW_KEY_VALUE *event_record_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTRECORDID);
+ if(event_record_id_rkv)
+ event_record_id = str2uint64_t(buffer_tostring(event_record_id_rkv->wb), NULL);
+ internal_fatal(event_record_id != d->ev.id,
+ "Event Record ID of row does not match the bin data associated with it");
+#endif
+
+ // the message needs the xml
+ EvtFormatMessage_Xml_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.xml);
+ EvtFormatMessage_Event_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.event);
d->rendered = true;
}
-static void wevt_render_xml(
+static void wevt_lazy_load_xml(
FACETS *facets,
BUFFER *json_array,
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@@ -121,7 +148,7 @@ static void wevt_render_xml(
buffer_json_add_array_item_string(json_array, d->log->ops.xml.data);
}
-static void wevt_render_message(
+static void wevt_lazy_load_message(
FACETS *facets,
BUFFER *json_array,
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@@ -217,18 +244,22 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
facets_register_key_name(
facets, WEVT_FIELD_CHANNEL,
- FACET_KEY_OPTION_FTS);
+ rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_PROVIDER,
rq->default_facet | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
facets_register_key_name(
- facets, WEVT_FIELD_SOURCE,
+ facets, WEVT_FIELD_ACCOUNT,
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
- facets, WEVT_FIELD_USER,
+ facets, WEVT_FIELD_DOMAIN,
+ rq->default_facet | FACET_KEY_OPTION_FTS);
+
+ facets_register_key_name(
+ facets, WEVT_FIELD_SID,
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
@@ -236,6 +267,11 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
rq->default_facet |
FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
+ facets_register_key_name(
+ facets, WEVT_FIELD_EVENTS_API,
+ rq->default_facet |
+ FACET_KEY_OPTION_FTS);
+
facets_register_key_name(
facets, WEVT_FIELD_LEVEL,
rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_EXPANDED_FILTER);
@@ -277,14 +313,32 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
FACET_KEY_OPTION_NONE);
facets_register_dynamic_key_name(
- facets, WEVT_FIELD_MESSAGE,
+ facets,
+ WEVT_FIELD_MESSAGE,
FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
- wevt_render_message, NULL);
+ wevt_lazy_load_message,
+ NULL);
facets_register_dynamic_key_name(
- facets, WEVT_FIELD_XML,
- FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
- wevt_render_xml, NULL);
+ facets,
+ WEVT_FIELD_XML,
+ FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
+ wevt_lazy_load_xml,
+ NULL);
+
+ if(query_has_fts(lqs)) {
+ facets_register_key_name(
+ facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN,
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
+
+ facets_register_key_name(
+ facets, WEVT_FIELD_EVENT_XML_HIDDEN,
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
+
+ facets_register_key_name(
+ facets, WEVT_FIELD_EVENT_DATA_HIDDEN,
+ FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
+ }
#ifdef NETDATA_INTERNAL_CHECKS
facets_register_key_name(
@@ -315,8 +369,8 @@ static const char *source_to_str(TXT_UTF8 *txt) {
case TXT_SOURCE_EVENT_LOG:
return "event-log";
- case TXT_SOURCE_PUBLISHER:
- return "publisher";
+ case TXT_SOURCE_PROVIDER:
+ return "provider";
case TXT_SOURCE_FIELD_CACHE:
return "fields-cache";
@@ -327,21 +381,80 @@ static const char *source_to_str(TXT_UTF8 *txt) {
}
#endif
+static const char *events_api_to_str(WEVT_PROVIDER_PLATFORM platform) {
+ switch(platform) {
+ case WEVT_PLATFORM_WEL:
+ return "Windows Event Log";
+
+ case WEVT_PLATFORM_ETW:
+ return "Event Tracing for Windows";
+
+ case WEVT_PLATFORM_TL:
+ return "TraceLogging";
+
+ default:
+ return "Unknown";
+ }
+}
+
static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *ev) {
- size_t len, bytes = log->ops.content.used;
+ static __thread char uuid_str[UUID_STR_LEN];
+
+ size_t len, bytes = log->ops.raw.system.used + log->ops.raw.user.used;
+
+ if(!UUIDiszero(ev->provider)) {
+ uuid_unparse_lower(ev->provider.uuid, uuid_str);
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_PROVIDER_GUID, sizeof(WEVT_FIELD_PROVIDER_GUID) - 1,
+ uuid_str, sizeof(uuid_str) - 1);
+ }
+
+ if(!UUIDiszero(ev->activity_id)) {
+ uuid_unparse_lower(ev->activity_id.uuid, uuid_str);
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_ACTIVITY_ID, sizeof(WEVT_FIELD_ACTIVITY_ID) - 1,
+ uuid_str, sizeof(uuid_str) - 1);
+ }
+
+ if(!UUIDiszero(ev->related_activity_id)) {
+ uuid_unparse_lower(ev->related_activity_id.uuid, uuid_str);
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_RELATED_ACTIVITY_ID, sizeof(WEVT_FIELD_RELATED_ACTIVITY_ID) - 1,
+ uuid_str, sizeof(uuid_str) - 1);
+ }
+
+ if(ev->qualifiers) {
+ static __thread char qualifiers[UINT64_HEX_MAX_LENGTH];
+ len = print_uint64_hex(qualifiers, ev->qualifiers);
+ bytes += len;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_QUALIFIERS, sizeof(WEVT_FIELD_QUALIFIERS) - 1,
+ qualifiers, len);
+ }
+
+ {
+ static __thread char event_record_id_str[UINT64_MAX_LENGTH];
+ len = print_uint64(event_record_id_str, ev->id);
+ bytes += len;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
+ event_record_id_str, len);
+ }
+
+ if(ev->version) {
+ static __thread char version[UINT64_MAX_LENGTH];
+ len = print_uint64(version, ev->version);
+ bytes += len;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_VERSION, sizeof(WEVT_FIELD_VERSION) - 1,
+ version, len);
+ }
if(log->ops.provider.used > 1) {
bytes += log->ops.provider.used * 2; // unicode is double
facets_add_key_value_length(
- facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
- log->ops.provider.data, log->ops.provider.used - 1);
- }
-
- if(log->ops.source.used > 1) {
- bytes += log->ops.source.used * 2;
- facets_add_key_value_length(
- facets, WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
- log->ops.source.data, log->ops.source.used - 1);
+ facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
+ log->ops.provider.data, log->ops.provider.used - 1);
}
if(log->ops.channel.used > 1) {
@@ -357,15 +470,6 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
src->fullname, src->fullname_len);
}
- {
- static __thread char event_record_id_str[UINT64_MAX_LENGTH];
- len = print_uint64(event_record_id_str, ev->id);
- bytes += len;
- facets_add_key_value_length(
- facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
- event_record_id_str, len);
- }
-
if(log->ops.level.used > 1) {
bytes += log->ops.level.used * 2;
facets_add_key_value_length(
@@ -401,11 +505,28 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
log->ops.task.data, log->ops.task.used - 1);
}
- if(log->ops.user.used > 1) {
- bytes += log->ops.user.used * 2;
+ if(log->ops.account.used > 1) {
+ bytes += log->ops.account.used * 2;
facets_add_key_value_length(
- facets, WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
- log->ops.user.data, log->ops.user.used - 1);
+ facets,
+ WEVT_FIELD_ACCOUNT, sizeof(WEVT_FIELD_ACCOUNT) - 1,
+ log->ops.account.data, log->ops.account.used - 1);
+ }
+
+ if(log->ops.domain.used > 1) {
+ bytes += log->ops.domain.used * 2;
+ facets_add_key_value_length(
+ facets,
+ WEVT_FIELD_DOMAIN, sizeof(WEVT_FIELD_DOMAIN) - 1,
+ log->ops.domain.data, log->ops.domain.used - 1);
+ }
+
+ if(log->ops.sid.used > 1) {
+ bytes += log->ops.sid.used * 2;
+ facets_add_key_value_length(
+ facets,
+ WEVT_FIELD_SID, sizeof(WEVT_FIELD_SID) - 1,
+ log->ops.sid.data, log->ops.sid.used - 1);
}
{
@@ -417,6 +538,12 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
event_id_str, len);
}
+ {
+ const char *s = events_api_to_str(ev->platform);
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_EVENTS_API, sizeof(WEVT_FIELD_EVENTS_API) - 1, s, strlen(s));
+ }
+
if(ev->process_id) {
static __thread char process_id_str[UINT64_MAX_LENGTH];
len = print_uint64(process_id_str, ev->process_id);
@@ -467,6 +594,30 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
facets, WEVT_FIELD_TASK "ID", sizeof(WEVT_FIELD_TASK) + 2 - 1, str, len);
}
+ if(log->type & WEVT_QUERY_EVENT_DATA) {
+ // the query has full text-search
+ if(log->ops.event.used > 1) {
+ bytes += log->ops.event.used;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN, sizeof(WEVT_FIELD_EVENT_MESSAGE_HIDDEN) - 1,
+ log->ops.event.data, log->ops.event.used - 1);
+ }
+
+ if(log->ops.xml.used > 1) {
+ bytes += log->ops.xml.used;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_EVENT_XML_HIDDEN, sizeof(WEVT_FIELD_EVENT_XML_HIDDEN) - 1,
+ log->ops.xml.data, log->ops.xml.used - 1);
+ }
+
+ if(log->ops.event_data->len) {
+ bytes += log->ops.event_data->len;
+ facets_add_key_value_length(
+ facets, WEVT_FIELD_EVENT_DATA_HIDDEN, sizeof(WEVT_FIELD_EVENT_DATA_HIDDEN) - 1,
+ buffer_tostring(log->ops.event_data), buffer_strlen(log->ops.event_data));
+ }
+ }
+
wevt_facets_register_bin_data(log, facets, ev);
#ifdef NETDATA_INTERNAL_CHECKS
@@ -536,7 +687,7 @@ static WEVT_QUERY_STATUS wevt_query_backward(
facets_rows_begin(facets);
WEVT_EVENT e;
- while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
+ while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
if(unlikely(!msg_ut)) {
@@ -650,7 +801,7 @@ static WEVT_QUERY_STATUS wevt_query_forward(
facets_rows_begin(facets);
WEVT_EVENT e;
- while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
+ while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
if(unlikely(!msg_ut)) {
@@ -754,8 +905,20 @@ static WEVT_QUERY_STATUS wevt_query_one_channel(
}
static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
- if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
- (lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
+ if(
+ // no source is requested
+ (lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) ||
+
+ // matches our internal source types
+ (src->source_type & lqs->rq.source_type) ||
+
+ // matches the source name
+ (lqs->rq.sources && src->source && simple_pattern_matches(lqs->rq.sources, string2str(src->source))) ||
+
+ // matches the provider (providers start with a special prefix to avoid mix and match)
+ (lqs->rq.sources && src->provider && simple_pattern_matches(lqs->rq.sources, string2str(src->provider)))
+
+ ) {
if(!src->msg_last_ut)
// the file is not scanned yet, or the timestamps have not been updated,
@@ -837,7 +1000,7 @@ static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs _
usec_t ended_ut = started_ut;
usec_t duration_ut, max_duration_ut = 0;
- WEVT_LOG *log = wevt_openlog6();
+ WEVT_LOG *log = wevt_openlog6(query_has_fts(lqs) ? WEVT_QUERY_FTS : WEVT_QUERY_NORMAL);
if(!log) {
// release the files
for(size_t f = 0; f < files_used ;f++)
@@ -1135,7 +1298,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
// initialization
wevt_sources_init();
- publisher_cache_init();
+ provider_cache_init();
sid_cache_init();
field_cache_init();
@@ -1209,6 +1372,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
const usec_t step_ut = 100 * USEC_PER_MS;
usec_t send_newline_ut = 0;
usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
+ usec_t since_last_providers_release_ut = 0;
const bool tty = isatty(fileno(stdout)) == 1;
heartbeat_t hb;
@@ -1220,7 +1384,13 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
since_last_scan_ut = 0;
}
+ if(since_last_providers_release_ut > WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT) {
+ providers_release_unused_handles();
+ since_last_providers_release_ut = 0;
+ }
+
usec_t dt_ut = heartbeat_next(&hb, step_ut);
+ since_last_providers_release_ut += dt_ut;
since_last_scan_ut += dt_ut;
send_newline_ut += dt_ut;
diff --git a/src/collectors/windows-events.plugin/windows-events.h b/src/collectors/windows-events.plugin/windows-events.h
index 631855824d..4cb5b50cad 100644
--- a/src/collectors/windows-events.plugin/windows-events.h
+++ b/src/collectors/windows-events.plugin/windows-events.h
@@ -128,10 +128,15 @@ typedef enum {
#include "windows-events-unicode.h"
#include "windows-events-sid.h"
#include "windows-events-xml.h"
-#include "windows-events-publishers.h"
+#include "windows-events-providers.h"
#include "windows-events-fields-cache.h"
#include "windows-events-query.h"
+// enable or disable preloading on full-text-search
+#define ON_FTS_PRELOAD_MESSAGE 1
+#define ON_FTS_PRELOAD_XML 0
+#define ON_FTS_PRELOAD_EVENT_DATA 1
+
#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
#define WEVT_FUNCTION_NAME "windows-events"
@@ -139,26 +144,40 @@ typedef enum {
#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
-
#define FUNCTION_PROGRESS_EVERY_ROWS (2000)
#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1000)
#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
+// run providers release every 5 mins
+#define WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT (5 * 60 * USEC_PER_SEC)
+// release idle handles that are older than 5 mins
+#define WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT (5 * 60 * USEC_PER_SEC)
+
#define WEVT_FIELD_COMPUTER "Computer"
#define WEVT_FIELD_CHANNEL "Channel"
#define WEVT_FIELD_PROVIDER "Provider"
-#define WEVT_FIELD_SOURCE "Source"
+#define WEVT_FIELD_PROVIDER_GUID "ProviderGUID"
#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
+#define WEVT_FIELD_VERSION "Version"
+#define WEVT_FIELD_QUALIFIERS "Qualifiers"
#define WEVT_FIELD_EVENTID "EventID"
#define WEVT_FIELD_LEVEL "Level"
#define WEVT_FIELD_KEYWORDS "Keywords"
#define WEVT_FIELD_OPCODE "Opcode"
-#define WEVT_FIELD_USER "User"
+#define WEVT_FIELD_ACCOUNT "UserAccount"
+#define WEVT_FIELD_DOMAIN "UserDomain"
+#define WEVT_FIELD_SID "UserSID"
#define WEVT_FIELD_TASK "Task"
#define WEVT_FIELD_PROCESSID "ProcessID"
#define WEVT_FIELD_THREADID "ThreadID"
+#define WEVT_FIELD_ACTIVITY_ID "ActivityID"
+#define WEVT_FIELD_RELATED_ACTIVITY_ID "RelatedActivityID"
#define WEVT_FIELD_XML "XML"
#define WEVT_FIELD_MESSAGE "Message"
+#define WEVT_FIELD_EVENTS_API "EventsAPI"
+#define WEVT_FIELD_EVENT_DATA_HIDDEN "__HIDDEN__EVENT__DATA__"
+#define WEVT_FIELD_EVENT_MESSAGE_HIDDEN "__HIDDEN__MESSAGE__DATA__"
+#define WEVT_FIELD_EVENT_XML_HIDDEN "__HIDDEN__XML__DATA__"
// functions needed by LQS
@@ -237,7 +256,8 @@ struct lqs_extension {
#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
-#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
+#define LQS_PARAMETER_SOURCE_NAME "Event Channels" // this is how it is shown to users
+#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) WEVT_SOURCE_TYPE_2id_one(value)
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
#include "libnetdata/facets/logs_query_status.h"
diff --git a/src/daemon/buildinfo.c b/src/daemon/buildinfo.c
index 905009a52b..3cbbe90350 100644
--- a/src/daemon/buildinfo.c
+++ b/src/daemon/buildinfo.c
@@ -1331,7 +1331,7 @@ char *get_value_from_key(char *buffer, char *key) {
return s;
}
-void get_install_type(char **install_type, char **prebuilt_arch, char **prebuilt_dist) {
+void get_install_type(char **install_type, char **prebuilt_arch __maybe_unused, char **prebuilt_dist __maybe_unused) {
#ifndef OS_WINDOWS
char *install_type_filename;
diff --git a/src/daemon/main.c b/src/daemon/main.c
index 98f32a88d2..4c199f1602 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -847,34 +847,60 @@ static void log_init(void) {
nd_log_set_priority_level(config_get(CONFIG_SECTION_LOGS, "level", netdata_log_level));
char filename[FILENAME_MAX + 1];
+ char* os_default_method = NULL;
+#if defined(OS_LINUX)
+ os_default_method = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */ ? "journal" : NULL;
+#elif defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ os_default_method = "etw";
+#elif defined(HAVE_WEL)
+ os_default_method = "wel";
+#endif
+#endif
+
+#if defined(OS_WINDOWS)
+ // on windows, debug log goes to windows events
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir);
+#endif
+
nd_log_set_user_settings(NDLS_DEBUG, config_get(CONFIG_SECTION_LOGS, "debug", filename));
- bool with_journal = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */;
- if(with_journal)
- snprintfz(filename, FILENAME_MAX, "journal");
+ if(os_default_method)
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/daemon.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_DAEMON, config_get(CONFIG_SECTION_LOGS, "daemon", filename));
- if(with_journal)
- snprintfz(filename, FILENAME_MAX, "journal");
+ if(os_default_method)
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/collector.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_COLLECTORS, config_get(CONFIG_SECTION_LOGS, "collector", filename));
+#if defined(OS_WINDOWS)
+ // on windows, access log goes to windows events
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
+#endif
nd_log_set_user_settings(NDLS_ACCESS, config_get(CONFIG_SECTION_LOGS, "access", filename));
- if(with_journal)
- snprintfz(filename, FILENAME_MAX, "journal");
+ if(os_default_method)
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/health.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_HEALTH, config_get(CONFIG_SECTION_LOGS, "health", filename));
aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO);
if (aclklog_enabled) {
+#if defined(OS_WINDOWS)
+ // on windows, aclk log goes to windows events
+ snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
snprintfz(filename, FILENAME_MAX, "%s/aclk.log", netdata_configured_log_dir);
+#endif
nd_log_set_user_settings(NDLS_ACLK, config_get(CONFIG_SECTION_CLOUD, "conversation log file", filename));
}
diff --git a/src/daemon/winsvc.cc b/src/daemon/winsvc.cc
index 23ade2895e..ca6929a618 100644
--- a/src/daemon/winsvc.cc
+++ b/src/daemon/winsvc.cc
@@ -219,7 +219,11 @@ static bool update_path() {
int main(int argc, char *argv[])
{
+#if defined(OS_WINDOWS) && defined(RUN_UNDER_CLION)
+ bool tty = true;
+#else
bool tty = isatty(fileno(stdin)) == 1;
+#endif
if (!update_path()) {
return 1;
diff --git a/src/libnetdata/buffer/buffer.h b/src/libnetdata/buffer/buffer.h
index 962ba25839..01a24be35a 100644
--- a/src/libnetdata/buffer/buffer.h
+++ b/src/libnetdata/buffer/buffer.h
@@ -521,13 +521,13 @@ static inline size_t print_int64(char *dst, int64_t value) {
#define UINT64_MAX_LENGTH (24) // 21 should be enough
static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
- buffer_need_bytes(wb, 50);
+ buffer_need_bytes(wb, UINT64_MAX_LENGTH);
wb->len += print_uint64(&wb->buffer[wb->len], value);
buffer_overflow_check(wb);
}
static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
- buffer_need_bytes(wb, 50);
+ buffer_need_bytes(wb, UINT64_MAX_LENGTH);
wb->len += print_int64(&wb->buffer[wb->len], value);
buffer_overflow_check(wb);
}
diff --git a/src/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c
index 0ea3640334..50ddf79ef4 100644
--- a/src/libnetdata/facets/facets.c
+++ b/src/libnetdata/facets/facets.c
@@ -2363,8 +2363,8 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w
if(with_keys) {
FACET_KEY *k;
- foreach_key_in_facets(facets, k){
- if (!k->values.enabled)
+ foreach_key_in_facets(facets, k) {
+ if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
buffer_json_add_array_item_string(wb, facets_key_id(k));
@@ -2591,7 +2591,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
CLEAN_BUFFER *tb = buffer_create(0, NULL);
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
- if(!k->values.enabled)
+ if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
facets_sort_and_reorder_values(k);
@@ -2677,37 +2677,40 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
- RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
- RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
- RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
+ if(k->options & FACET_KEY_OPTION_HIDDEN)
+ continue;
- if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
- ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
- simple_pattern_matches(facets->visible_keys, k->name))
- options |= RRDF_FIELD_OPTS_VISIBLE;
+ RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
+ RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
+ RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
- if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
- options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
+ if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
+ ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
+ simple_pattern_matches(facets->visible_keys, k->name))
+ options |= RRDF_FIELD_OPTS_VISIBLE;
- if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
- options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
+ if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
+ options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
- if (k->options & FACET_KEY_OPTION_PRETTY_XML)
- transform = RRDF_FIELD_TRANSFORM_XML;
+ if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
+ options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
- const char *key_id = facets_key_id(k);
+ if (k->options & FACET_KEY_OPTION_PRETTY_XML)
+ transform = RRDF_FIELD_TRANSFORM_XML;
- buffer_rrdf_table_add_field(
- wb, field_id++,
- key_id, k->name ? k->name : key_id,
- RRDF_FIELD_TYPE_STRING,
- visual, transform, 0, NULL, NAN,
- RRDF_FIELD_SORT_FIXED,
- NULL,
- RRDF_FIELD_SUMMARY_COUNT,
- (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
- options, FACET_VALUE_UNSET);
- }
+ const char *key_id = facets_key_id(k);
+
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ key_id, k->name ? k->name : key_id,
+ RRDF_FIELD_TYPE_STRING,
+ visual, transform, 0, NULL, NAN,
+ RRDF_FIELD_SORT_FIXED,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
+ options, FACET_VALUE_UNSET);
+ }
foreach_key_in_facets_done(k);
}
buffer_json_object_close(wb); // columns
@@ -2744,6 +2747,9 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
+ if(k->options & FACET_KEY_OPTION_HIDDEN)
+ continue;
+
FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
if(unlikely(k->dynamic.cb)) {
@@ -2786,7 +2792,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
{
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
- if (!k->values.enabled)
+ if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
if(unlikely(!first_histogram_hash))
diff --git a/src/libnetdata/facets/facets.h b/src/libnetdata/facets/facets.h
index 5f14c4e938..1d2b89c2bf 100644
--- a/src/libnetdata/facets/facets.h
+++ b/src/libnetdata/facets/facets.h
@@ -37,6 +37,7 @@ typedef enum __attribute__((packed)) {
FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 10), // when registering the transformation, do it only at the view, not on all data
FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 11), // the presentation should have this filter expanded by default
FACET_KEY_OPTION_PRETTY_XML = (1 << 12), // instruct the UI to parse this as an XML document
+ FACET_KEY_OPTION_HIDDEN = (1 << 13), // do not include this field in the response
} FACET_KEY_OPTIONS;
typedef enum __attribute__((packed)) {
diff --git a/src/libnetdata/facets/logs_query_status.h b/src/libnetdata/facets/logs_query_status.h
index 4517130466..4fde249980 100644
--- a/src/libnetdata/facets/logs_query_status.h
+++ b/src/libnetdata/facets/logs_query_status.h
@@ -17,7 +17,6 @@
#define LQS_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"
#define LQS_PARAMETER_DATA_ONLY "data_only"
#define LQS_PARAMETER_SOURCE "__logs_sources" // this must never conflict with user fields
-#define LQS_PARAMETER_SOURCE_NAME "Logs Sources" // this is how it is shown to users
#define LQS_PARAMETER_INFO "info"
#define LQS_PARAMETER_SLICE "slice"
#define LQS_PARAMETER_DELTA "delta"
diff --git a/src/libnetdata/libnetdata.h b/src/libnetdata/libnetdata.h
index d40987f323..969e677cb7 100644
--- a/src/libnetdata/libnetdata.h
+++ b/src/libnetdata/libnetdata.h
@@ -459,14 +459,14 @@ extern const char *netdata_configured_host_prefix;
#include "datetime/rfc3339.h"
#include "datetime/rfc7231.h"
#include "completion/completion.h"
-#include "log/log.h"
+#include "libnetdata/log/nd_log.h"
#include "spawn_server/spawn_server.h"
#include "spawn_server/spawn_popen.h"
#include "simple_pattern/simple_pattern.h"
#include "socket/security.h"
#include "socket/socket.h"
#include "config/appconfig.h"
-#include "log/journal.h"
+#include "log/systemd-journal-helpers.h"
#include "buffered_reader/buffered_reader.h"
#include "procfile/procfile.h"
#include "string/string.h"
diff --git a/src/libnetdata/log/README.md b/src/libnetdata/log/README.md
index 44ba99f806..c7a42f28b0 100644
--- a/src/libnetdata/log/README.md
+++ b/src/libnetdata/log/README.md
@@ -17,14 +17,15 @@ For each log source, Netdata supports the following output methods:
- **off**, to disable this log source
- **journal**, to send the logs to systemd-journal.
+- **etw**, to send the logs to Event Tracing for Windows (ETW).
+- **wel**, to send the logs to the Windows Event Log (WEL).
- **syslog**, to send the logs to syslog.
- **system**, to send the output to `stderr` or `stdout` depending on the log source.
- **stdout**, to write the logs to Netdata's `stdout`.
- **stderr**, to write the logs to Netdata's `stderr`.
- **filename**, to send the logs to a file.
-For `daemon` and `collector` the default is `journal` when systemd-journal is available.
-To decide if systemd-journal is available, Netdata checks:
+On Linux, when systemd-journal is available, the default is `journal` for `daemon` and `collector` and `filename` for the rest. To decide if systemd-journal is available, Netdata checks:
1. `stderr` is connected to systemd-journald
2. `/run/systemd/journal/socket` exists
@@ -32,13 +33,16 @@ To decide if systemd-journal is available, Netdata checks:
If any of the above is detected, Netdata will select `journal` for `daemon` and `collector` sources.
-All other sources default to a file.
+On Windows, the default is `etw` and if that is not available it falls back to `wel`. The availability of `etw` is decided at compile time.
## Log formats
| Format | Description |
|---------|--------------------------------------------------------------------------------------------------------|
| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
+| etw | Event Tracing for Windows specific format. Structured logging in Event Viewer. |
+| wel | Windows Event Log specific format. Basic field-based logging in Event Viewer. |
+| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
| logfmt | logs data as a series of key/value pairs. The default when logging to any output other than `journal`. |
| json | logs data in JSON format. |
@@ -57,6 +61,9 @@ Each time Netdata logs, it assigns a priority to the log. It can be one of this
| info | the default log level about information the user should know. |
| debug | these are more verbose logs that can be ignored. |
+For `etw` these are mapped to `Verbose`, `Informational`, `Warning`, `Error` and `Critical`.
+For `wel` these are mapped to `Informational`, `Warning`, `Error`.
+
## Logs Configuration
In `netdata.conf`, there are the following settings:
@@ -108,66 +115,69 @@ Sending a `SIGHUP` to Netdata, will instruct it to re-open all its log files.
<details>
<summary>All fields exposed by Netdata</summary>
-| journal | logfmt | json | Description |
-|:--------------------------------------:|:------------------------------:|:------------------------------:|:---------------------------------------------------------------------------------------------------------:|
-| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `time` | the timestamp of the event |
-| `SYSLOG_IDENTIFIER` | `comm` | `comm` | the program logging the event |
-| `ND_LOG_SOURCE` | `source` | `source` | one of the [log sources](#log-sources) |
-| `PRIORITY`<br/>numeric | `level`<br/>text | `level`<br/>numeric | one of the [log levels](#log-levels) |
-| `ERRNO` | `errno` | `errno` | the numeric value of `errno` |
-| `INVOCATION_ID` | - | - | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
-| `CODE_LINE` | - | - | the line number of of the source code logging this event |
-| `CODE_FILE` | - | - | the filename of the source code logging this event |
-| `CODE_FUNCTION` | - | - | the function name of the source code logging this event |
-| `TID` | `tid` | `tid` | the thread id of the thread logging this event |
-| `THREAD_TAG` | `thread` | `thread` | the name of the thread logging this event |
-| `MESSAGE_ID` | `msg_id` | `msg_id` | see [message IDs](#message-ids) |
-| `ND_MODULE` | `module` | `module` | the Netdata module logging this event |
-| `ND_NIDL_NODE` | `node` | `node` | the hostname of the node the event is related to |
-| `ND_NIDL_INSTANCE` | `instance` | `instance` | the instance of the node the event is related to |
-| `ND_NIDL_CONTEXT` | `context` | `context` | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
-| `ND_NIDL_DIMENSION` | `dimension` | `dimension` | the dimension the event is related to |
-| `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport |
-| `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from |
-| `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from |
-| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` |
-| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` |
-| `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child |
-| `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport |
-| `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination |
-| `ND_DST_PORT` | `dst_port` | `dst_port` | when the event happened during an outbound request, this is the port the request destination |
-| `ND_DST_CAPABILITIES` | `dst_capabilities` | `dst_capabilities` | when the request goes to a parent, this is the communication capabilities of the parent |
-| `ND_REQUEST_METHOD` | `req_method` | `req_method` | when the event happened during an inbound request, this is the method the request was received |
-| `ND_RESPONSE_CODE` | `code` | `code` | when responding to a request, this this the response code |
-| `ND_CONNECTION_ID` | `conn` | `conn` | when there is a connection id for an inbound connection, this is the connection id |
-| `ND_TRANSACTION_ID` | `transaction` | `transaction` | the transaction id (UUID) of all API requests |
-| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `sent_bytes` | the bytes we sent to API responses |
-| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `size_bytes` | the uncompressed bytes of the API responses |
-| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `prep_ut` | the time needed to prepare a response |
-| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `sent_ut` | the time needed to send a response |
-| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `total_ut` | the total time needed to complete a response |
-| `ND_ALERT_ID` | `alert_id` | `alert_id` | the alert id this event is related to |
-| `ND_ALERT_EVENT_ID` | `alert_event_id` | `alert_event_id` | a sequential number of the alert transition (per host) |
-| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `alert_unique_id` | a sequential number of the alert transition (per alert) |
-| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `alert_transition_id` | the unique UUID of this alert transition |
-| `ND_ALERT_CONFIG` | `alert_config` | `alert_config` | the alert configuration hash (UUID) |
-| `ND_ALERT_NAME` | `alert` | `alert` | the alert name |
-| `ND_ALERT_CLASS` | `alert_class` | `alert_class` | the alert classification |
-| `ND_ALERT_COMPONENT` | `alert_component` | `alert_component` | the alert component |
-| `ND_ALERT_TYPE` | `alert_type` | `alert_type` | the alert type |
-| `ND_ALERT_EXEC` | `alert_exec` | `alert_exec` | the alert notification program |
-| `ND_ALERT_RECIPIENT` | `alert_recipient` | `alert_recipient` | the alert recipient(s) |
-| `ND_ALERT_VALUE` | `alert_value` | `alert_value` | the current alert value |
-| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
-| `ND_ALERT_STATUS` | `alert_status` | `alert_status` | the current alert status |
-| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
-| `ND_ALERT_UNITS` | `alert_units` | `alert_units` | the units of the alert |
-| `ND_ALERT_SUMMARY` | `alert_summary` | `alert_summary` | the summary text of the alert |
-| `ND_ALERT_INFO` | `alert_info` | `alert_info` | the info text of the alert |
-| `ND_ALERT_DURATION` | `alert_duration` | `alert_duration` | the duration the alert was in its previous state |
-| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `alert_notification_timestamp` | the timestamp the notification delivery is scheduled |
-| `ND_REQUEST` | `request` | `request` | the full request during which the event happened |
-| `MESSAGE` | `msg` | `msg` | the event message |
+| `journal` | `logfmt` and `json` | `etw` | `wel` | Description |
+|:--------------------------------------:|:------------------------------:|:-----------------------------:|:-----:|:----------------------------------------------------------------------------------------------------------|
+| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `Timestamp` | 1 | the timestamp of the event |
+| `SYSLOG_IDENTIFIER` | `comm` | `Program` | 2 | the program logging the event |
+| `ND_LOG_SOURCE` | `source` | `NetdataLogSource` | 3 | one of the [log sources](#log-sources) |
+| `PRIORITY`<br/>numeric | `level`<br/>text | `Level`<br/>text | 4 | one of the [log levels](#log-levels) |
+| `ERRNO` | `errno` | `UnixErrno` | 5 | the numeric value of `errno` |
+| `INVOCATION_ID` | - | `InvocationID` | 7 | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
+| `CODE_LINE` | - | `CodeLine` | 8 | the line number of of the source code logging this event |
+| `CODE_FILE` | - | `CodeFile` | 9 | the filename of the source code logging this event |
+| `CODE_FUNCTION` | - | `CodeFunction` | 10 | the function name of the source code logging this event |
+| `TID` | `tid` | `ThreadID` | 11 | the thread id of the thread logging this event |
+| `THREAD_TAG` | `thread` | `ThreadName` | 12 | the name of the thread logging this event |
+| `MESSAGE_ID` | `msg_id` | `MessageID` | 13 | see [message IDs](#message-ids) |
+| `ND_MODULE` | `module` | `Module` | 14 | the Netdata module logging this event |
+| `ND_NIDL_NODE` | `node` | `Node` | 15 | the hostname of the node the event is related to |
+| `ND_NIDL_INSTANCE` | `instance` | `Instance` | 16 | the instance of the node the event is related to |
+| `ND_NIDL_CONTEXT` | `context` | `Context` | 17 | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
+| `ND_NIDL_DIMENSION` | `dimension` | `Dimension` | 18 | the dimension the event is related to |
+| `ND_SRC_TRANSPORT` | `src_transport` | `SourceTransport` | 19 | when the event happened during a request, this is the request transport |
+| `ND_SRC_IP` | `src_ip` | `SourceIP` | 24 | when the event happened during an inbound request, this is the IP the request came from |
+| `ND_SRC_PORT` | `src_port` | `SourcePort` | 25 | when the event happened during an inbound request, this is the port the request came from |
+| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `SourceForwardedHost` | 26 | the contents of the HTTP header `X-Forwarded-Host` |
+| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `SourceForwardedFor` | 27 | the contents of the HTTP header `X-Forwarded-For` |
+| `ND_SRC_CAPABILITIES` | `src_capabilities` | `SourceCapabilities` | 28 | when the request came from a child, this is the communication capabilities of the child |
+| `ND_DST_TRANSPORT` | `dst_transport` | `DestinationTransport` | 29 | when the event happened during an outbound request, this is the outbound request transport |
+| `ND_DST_IP` | `dst_ip` | `DestinationIP` | 30 | when the event happened during an outbound request, this is the IP the request destination |
+| `ND_DST_PORT` | `dst_port` | `DestinationPort` | 31 | when the event happened during an outbound request, this is the port the request destination |
+| `ND_DST_CAPABILITIES` | `dst_capabilities` | `DestinationCapabilities` | 32 | when the request goes to a parent, this is the communication capabilities of the parent |
+| `ND_REQUEST_METHOD` | `req_method` | `RequestMethod` | 33 | when the event happened during an inbound request, this is the method the request was received |
+| `ND_RESPONSE_CODE` | `code` | `ResponseCode` | 34 | when responding to a request, this this the response code |
+| `ND_CONNECTION_ID` | `conn` | `ConnectionID` | 35 | when there is a connection id for an inbound connection, this is the connection id |
+| `ND_TRANSACTION_ID` | `transaction` | `TransactionID` | 36 | the transaction id (UUID) of all API requests |
+| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `ResponseSentBytes` | 37 | the bytes we sent to API responses |
+| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `ResponseSizeBytes` | 38 | the uncompressed bytes of the API responses |
+| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `ResponsePreparationTimeUsec` | 39 | the time needed to prepare a response |
+| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `ResponseSentTimeUsec` | 40 | the time needed to send a response |
+| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `ResponseTotalTimeUsec` | 41 | the total time needed to complete a response |
+| `ND_ALERT_ID` | `alert_id` | `AlertID` | 42 | the alert id this event is related to |
+| `ND_ALERT_EVENT_ID` | `alert_event_id` | `AlertEventID` | 44 | a sequential number of the alert transition (per host) |
+| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `AlertUniqueID` | 43 | a sequential number of the alert transition (per alert) |
+| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `AlertTransitionID` | 45 | the unique UUID of this alert transition |
+| `ND_ALERT_CONFIG` | `alert_config` | `AlertConfig` | 46 | the alert configuration hash (UUID) |
+| `ND_ALERT_NAME` | `alert` | `AlertName` | 47 | the alert name |
+| `ND_ALERT_CLASS` | `alert_class` | `AlertClass` | 48 | the alert classification |
+| `ND_ALERT_COMPONENT` | `alert_component` | `AlertComponent` | 49 | the alert component |
+| `ND_ALERT_TYPE` | `alert_type` | `AlertType` | 50 | the alert type |
+| `ND_ALERT_EXEC` | `alert_exec` | `AlertExec` | 51 | the alert notification program |
+| `ND_ALERT_RECIPIENT` | `alert_recipient` | `AlertRecipient` | 52 | the alert recipient(s) |
+| `ND_ALERT_VALUE` | `alert_value` | `AlertValue` | 54 | the current alert value |
+| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `AlertOldValue` | 55 | the previous alert value |
+| `ND_ALERT_STATUS` | `alert_status` | `AlertStatus` | 56 | the current alert status |
+| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `AlertOldStatus` | 57 | the previous alert status |
+| `ND_ALERT_UNITS` | `alert_units` | `AlertUnits` | 59 | the units of the alert |
+| `ND_ALERT_SUMMARY` | `alert_summary` | `AlertSummary` | 60 | the summary text of the alert |
+| `ND_ALERT_INFO` | `alert_info` | `AlertInfo` | 61 | the info text of the alert |
+| `ND_ALERT_DURATION` | `alert_duration` | `AlertDuration` | 53 | the duration the alert was in its previous state |
+| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `AlertNotificationTimeUsec` | 62 | the timestamp the notification delivery is scheduled |
+| `ND_REQUEST` | `request` | `Request` | 63 | the full request during which the event happened |
+| `MESSAGE` | `msg` | `Message` | 64 | the event message |
+
+For `wel` (Windows Event Logs), all logs have an array of 64 fields strings, and their index number provides their meaning.
+For `etw` (Event Tracing for Windows), Netdata logs in a structured way, and field names are available.
</details>
@@ -212,3 +222,117 @@ journalctl -u netdata --namespace=netdata
# All netdata logs, the newest entries are displayed first
journalctl -u netdata --namespace=netdata -r
```
+
+## Using Event Tracing for Windows (ETW)
+
+ETW requires the publisher `Netdata` to be registered. Our Windows installer does this automatically.
+
+Registering the publisher is done via a manifest (`%SystemRoot%\System32\wevt_netdata_manifest.xml`)
+and its messages resources DLL (`%SystemRoot%\System32\wevt_netdata.dll`).
+
+If needed, the publisher can be registered and unregistered manually using these commands:
+
+```bat
+REM register the Netdata publisher
+wevtutil im "%SystemRoot%\System32\wevt_netdata_manifest.xml" "/mf:%SystemRoot%\System32\wevt_netdata.dll" "/rf:%SystemRoot%\System32\wevt_netdata.dll"
+
+REM unregister the Netdata publisher
+wevtutil um "%SystemRoot%\System32\wevt_netdata_manifest.xml"
+```
+
+The structure of the logs are as follows:
+
+ - Publisher `Netdata`
+ - Channel `Netdata/Daemon`: general messages about the Netdata service
+ - Channel `Netdata/Collector`: general messages about Netdata external plugins
+ - Channel `Netdata/Health`: alert transitions and general messages generated by Netdata's health engine
+ - Channel `Netdata/Access`: all accesses to Netdata APIs
+ - Channel `Netdata/Aclk`: for cloud connectivity tracing (disabled by default)
+
+Retention can be configured per Channel via the Event Viewer. Netdata does not set a default, so the system default is used.
+
+> **IMPORTANT**<br/>
+> Event Tracing for Windows (ETW) does not allow logging the percentage character `%`.
+> The `%` followed by a number, is recursively used for fields expansion and ETW has not
+> provided any way to escape the character for preventing further expansion.<br/>
+> <br/>
+> To work around this limitation, Netdata replaces all `%` which are followed by a number, with `â„…`
+> (the Unicode character `care of`). Visually, they look similar, but when copying IPv6 addresses
+> or URLs from the logs, you have to be careful to manually replace `â„…` with `%` before using them.
+
+## Using Windows Event Logs (WEL)
+
+WEL has a different logs structure and unfortunately WEL and ETW need to use different names if they are to be used
+concurrently.
+
+For WEL, Netdata logs as follows:
+
+ - Channel `NetdataWEL` (unfortunately `Netdata` cannot be used, it conflicts with the ETW Publisher name)
+ - Publisher `NetdataDaemon`: general messages about the Netdata service
+ - Publisher `NetdataCollector`: general messages about Netdata external plugins
+ - Publisher `NetdataHealth`: alert transitions and general messages generated by Netdata's health engine
+ - Publisher `NetdataAccess`: all accesses to Netdata APIs
+ - Publisher `NetdataAclk`: for cloud connectivity tracing (disabled by default)
+
+Publishers must have unique names system-wide, so we had to prefix them with `Netdata`.
+
+Retention can be configured per Publisher via the Event Viewer or the Registry.
+Netdata sets by default 20MiB for all of them, except `NetdataAclk` (5MiB) and `NetdataAccess` (35MiB),
+for a total of 100MiB.
+
+For WEL some registry entries are needed. Netdata automatically takes care of them when it starts.
+
+WEL does not have the problem ETW has with the percent character `%`, so Netdata logs it as-is.
+
+## Differences between ETW and WEL
+
+There are key differences between ETW and WEL.
+
+### Publishers and Providers
+**Publishers** are collections of ETW Providers. A Publisher is implied by a manifest file,
+each of which is considered a Publisher, and each manifest file can define multiple **Providers** in it.
+Other than that there is no entity related to **Publishers** in the system.
+
+**Publishers** are not defined for WEL.
+
+**Providers** are the applications or modules logging. Provider names must be unique across the system,
+for ETW and WEL together.
+
+To define a **Provider**:
+
+- ETW requires a **Publisher** manifest coupled with resources DLLs and must be registered
+ via `wevtutil` (handled by the Netdata Windows installer automatically).
+- WEL requires some registry entries and a message resources DLL (handled by Netdata automatically on startup).
+
+The Provider appears as `Source` in the Event Viewer, for both WEL and ETW.
+
+### Channels
+- **Channels** for WEL are collections of WEL Providers, (each WEL Provider is a single Stream of logs).
+- **Channels** for ETW slice the logs of each Provider into multiple Streams.
+
+WEL Channels cannot have the same name as ETW Providers. This is why Netdata's ETW provider is
+called `Netdata`, and WEL channel is called `NetdataWEL`.
+
+Despite the fact that ETW **Publishers** and WEL **Channels** are both collections of Providers,
+they are not similar. In ETW a Publisher is a collection on the publisher's Providers, but in WEL
+a Channel may include independent WEL Providers (e.g. the "Applications" Channel). Additionally,
+WEL Channels cannot include ETW Providers.
+
+### Retention
+Retention is always defined per Stream.
+
+- Retention in ETW is defined per ETW Channel (ETW Provider Stream).
+- Retention in WEL is defined per WEL Provider (each WEL Provider is a single Stream).
+
+### Messages Formatting
+- ETW supports recursive fields expansion, and therefore `%N` in fields is expanded recursively
+ (or replaced with an error message if expansion fails). Netdata replaces `%N` with `â„…N` to stop
+ recursive expansion (since `%N` cannot be logged otherwise).
+- WEL performs a single field expansion, and therefore the `%` character in fields is never expanded.
+
+### Usability
+
+- ETW names all the fields and allows multiple datatypes per field, enabling log consumers to know
+ what each field means and its datatype.
+- WEL uses a simple string table for fields, and consumers need to map these string fields based on
+ their index.
diff --git a/src/libnetdata/log/log.c b/src/libnetdata/log/log.c
deleted file mode 100644
index b7efa26464..0000000000
--- a/src/libnetdata/log/log.c
+++ /dev/null
@@ -1,2555 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the
-// source code that makes the calls, allowing our loggers to log the lines of source code that actually log
-#define SD_JOURNAL_SUPPRESS_LOCATION
-
-#include "../libnetdata.h"
-
-#if defined(OS_WINDOWS)
-#include <windows.h>
-#endif
-
-#ifdef __FreeBSD__
-#include <sys/endian.h>
-#endif
-
-#ifdef __APPLE__
-#include <machine/endian.h>
-#endif
-
-#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
-#include <execinfo.h>
-#endif
-
-#ifdef HAVE_SYSTEMD
-#include <systemd/sd-journal.h>
-#endif
-
-const char *program_name = "";
-uint64_t debug_flags = 0;
-int aclklog_enabled = 0;
-
-// ----------------------------------------------------------------------------
-
-struct nd_log_source;
-static bool nd_log_limit_reached(struct nd_log_source *source);
-
-// ----------------------------------------------------------------------------
-
-void errno_clear(void) {
- errno = 0;
-
-#if defined(OS_WINDOWS)
- SetLastError(ERROR_SUCCESS);
-#endif
-}
-
-// ----------------------------------------------------------------------------
-// logging method
-
-typedef enum __attribute__((__packed__)) {
- NDLM_DISABLED = 0,
- NDLM_DEVNULL,
- NDLM_DEFAULT,
- NDLM_JOURNAL,
- NDLM_SYSLOG,
- NDLM_STDOUT,
- NDLM_STDERR,
- NDLM_FILE,
-} ND_LOG_METHOD;
-
-static struct {
- ND_LOG_METHOD method;
- const char *name;
-} nd_log_methods[] = {
- { .method = NDLM_DISABLED, .name = "none" },
- { .method = NDLM_DEVNULL, .name = "/dev/null" },
- { .method = NDLM_DEFAULT, .name = "default" },
- { .method = NDLM_JOURNAL, .name = "journal" },
- { .method = NDLM_SYSLOG, .name = "syslog" },
- { .method = NDLM_STDOUT, .name = "stdout" },
- { .method = NDLM_STDERR, .name = "stderr" },
- { .method = NDLM_FILE, .name = "file" },
-};
-
-static ND_LOG_METHOD nd_log_method2id(const char *method) {
- if(!method || !*method)
- return NDLM_DEFAULT;
-
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_methods[i].name, method) == 0)
- return nd_log_methods[i].method;
- }
-
- return NDLM_FILE;
-}
-
-static const char *nd_log_id2method(ND_LOG_METHOD method) {
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(method == nd_log_methods[i].method)
- return nd_log_methods[i].name;
- }
-
- return "unknown";
-}
-
-#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR)
-
-const char *nd_log_method_for_external_plugins(const char *s) {
- if(s && *s) {
- ND_LOG_METHOD method = nd_log_method2id(s);
- if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
- return nd_log_id2method(method);
- }
-
- return nd_log_id2method(NDLM_STDERR);
-}
-
-// ----------------------------------------------------------------------------
-// workaround strerror_r()
-
-#if defined(STRERROR_R_CHAR_P)
-// GLIBC version of strerror_r
-static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
-#elif defined(HAVE_STRERROR_R)
-// POSIX version of strerror_r
-static const char *strerror_result(int a, const char *b) { (void)a; return b; }
-#elif defined(HAVE_C__GENERIC)
-
-// what a trick!
-// http://stackoverflow.com/questions/479207/function-overloading-in-c
-static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
-static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
-
-#define strerror_result(a, b) _Generic((a), \
- int: strerror_result_int, \
- char *: strerror_result_string \
- )(a, b)
-
-#else
-#error "cannot detect the format of function strerror_r()"
-#endif
-
-static const char *errno2str(int errnum, char *buf, size_t size) {
- return strerror_result(strerror_r(errnum, buf, size), buf);
-}
-
-// ----------------------------------------------------------------------------
-// facilities
-//
-// sys/syslog.h (Linux)
-// sys/sys/syslog.h (FreeBSD)
-// bsd/sys/syslog.h (darwin-xnu)
-
-static struct {
- int facility;
- const char *name;
-} nd_log_facilities[] = {
- { LOG_AUTH, "auth" },
- { LOG_AUTHPRIV, "authpriv" },
- { LOG_CRON, "cron" },
- { LOG_DAEMON, "daemon" },
- { LOG_FTP, "ftp" },
- { LOG_KERN, "kern" },
- { LOG_LPR, "lpr" },
- { LOG_MAIL, "mail" },
- { LOG_NEWS, "news" },
- { LOG_SYSLOG, "syslog" },
- { LOG_USER, "user" },
- { LOG_UUCP, "uucp" },
- { LOG_LOCAL0, "local0" },
- { LOG_LOCAL1, "local1" },
- { LOG_LOCAL2, "local2" },
- { LOG_LOCAL3, "local3" },
- { LOG_LOCAL4, "local4" },
- { LOG_LOCAL5, "local5" },
- { LOG_LOCAL6, "local6" },
- { LOG_LOCAL7, "local7" },
-
-#ifdef __FreeBSD__
- { LOG_CONSOLE, "console" },
- { LOG_NTP, "ntp" },
-
- // FreeBSD does not consider 'security' as deprecated.
- { LOG_SECURITY, "security" },
-#else
- // For all other O/S 'security' is mapped to 'auth'.
- { LOG_AUTH, "security" },
-#endif
-
-#ifdef __APPLE__
- { LOG_INSTALL, "install" },
- { LOG_NETINFO, "netinfo" },
- { LOG_RAS, "ras" },
- { LOG_REMOTEAUTH, "remoteauth" },
- { LOG_LAUNCHD, "launchd" },
-
-#endif
-};
-
-static int nd_log_facility2id(const char *facility) {
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_facilities[i].name, facility) == 0)
- return nd_log_facilities[i].facility;
- }
-
- return LOG_DAEMON;
-}
-
-static const char *nd_log_id2facility(int facility) {
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(nd_log_facilities[i].facility == facility)
- return nd_log_facilities[i].name;
- }
-
- return "daemon";
-}
-
-// ----------------------------------------------------------------------------
-// priorities
-
-static struct {
- ND_LOG_FIELD_PRIORITY priority;
- const char *name;
-} nd_log_priorities[] = {
- { .priority = NDLP_EMERG, .name = "emergency" },
- { .priority = NDLP_EMERG, .name = "emerg" },
- { .priority = NDLP_ALERT, .name = "alert" },
- { .priority = NDLP_CRIT, .name = "critical" },
- { .priority = NDLP_CRIT, .name = "crit" },
- { .priority = NDLP_ERR, .name = "error" },
- { .priority = NDLP_ERR, .name = "err" },
- { .priority = NDLP_WARNING, .name = "warning" },
- { .priority = NDLP_WARNING, .name = "warn" },
- { .priority = NDLP_NOTICE, .name = "notice" },
- { .priority = NDLP_INFO, .name = NDLP_INFO_STR },
- { .priority = NDLP_DEBUG, .name = "debug" },
-};
-
-int nd_log_priority2id(const char *priority) {
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_priorities[i].name, priority) == 0)
- return nd_log_priorities[i].priority;
- }
-
- return NDLP_INFO;
-}
-
-const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(priority == nd_log_priorities[i].priority)
- return nd_log_priorities[i].name;
- }
-
- return NDLP_INFO_STR;
-}
-
-// ----------------------------------------------------------------------------
-// log sources
-
-const char *nd_log_sources[] = {
- [NDLS_UNSET] = "UNSET",
- [NDLS_ACCESS] = "access",
- [NDLS_ACLK] = "aclk",
- [NDLS_COLLECTORS] = "collector",
- [NDLS_DAEMON] = "daemon",
- [NDLS_HEALTH] = "health",
- [NDLS_DEBUG] = "debug",
-};
-
-size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_sources[i], source) == 0)
- return i;
- }
-
- return def;
-}
-
-
-static const char *nd_log_id2source(ND_LOG_SOURCES source) {
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
- if(source < entries)
- return nd_log_sources[source];
-
- return nd_log_sources[NDLS_COLLECTORS];
-}
-
-// ----------------------------------------------------------------------------
-// log output formats
-
-typedef enum __attribute__((__packed__)) {
- NDLF_JOURNAL,
- NDLF_LOGFMT,
- NDLF_JSON,
-} ND_LOG_FORMAT;
-
-static struct {
- ND_LOG_FORMAT format;
- const char *name;
-} nd_log_formats[] = {
- { .format = NDLF_JOURNAL, .name = "journal" },
- { .format = NDLF_LOGFMT, .name = "logfmt" },
- { .format = NDLF_JSON, .name = "json" },
-};
-
-static ND_LOG_FORMAT nd_log_format2id(const char *format) {
- if(!format || !*format)
- return NDLF_LOGFMT;
-
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_formats[i].name, format) == 0)
- return nd_log_formats[i].format;
- }
-
- return NDLF_LOGFMT;
-}
-
-static const char *nd_log_id2format(ND_LOG_FORMAT format) {
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(format == nd_log_formats[i].format)
- return nd_log_formats[i].name;
- }
-
- return "logfmt";
-}
-
-// ----------------------------------------------------------------------------
-// format dates
-
-void log_date(char *buffer, size_t len, time_t now) {
- if(unlikely(!buffer || !len))
- return;
-
- time_t t = now;
- struct tm *tmp, tmbuf;
-
- tmp = localtime_r(&t, &tmbuf);
-
- if (unlikely(!tmp)) {
- buffer[0] = '\0';
- return;
- }
-
- if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
- buffer[0] = '\0';
-
- buffer[len - 1] = '\0';
-}
-
-// ----------------------------------------------------------------------------
-
-struct nd_log_limit {
- usec_t started_monotonic_ut;
- uint32_t counter;
- uint32_t prevented;
-
- uint32_t throttle_period;
- uint32_t logs_per_period;
- uint32_t logs_per_period_backup;
-};
-
-#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
-#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
-
-struct nd_log_source {
- SPINLOCK spinlock;
- ND_LOG_METHOD method;
- ND_LOG_FORMAT format;
- const char *filename;
- int fd;
- FILE *fp;
-
- ND_LOG_FIELD_PRIORITY min_priority;
- const char *pending_msg;
- struct nd_log_limit limits;
-};
-
-static struct {
- nd_uuid_t invocation_id;
-
- ND_LOG_SOURCES overwrite_process_source;
-
- struct nd_log_source sources[_NDLS_MAX];
-
- struct {
- bool initialized;
- } journal;
-
- struct {
- bool initialized;
- int fd;
- char filename[FILENAME_MAX + 1];
- } journal_direct;
-
- struct {
- bool initialized;
- int facility;
- } syslog;
-
- struct {
- SPINLOCK spinlock;
- bool initialized;
- } std_output;
-
- struct {
- SPINLOCK spinlock;
- bool initialized;
- } std_error;
-
-} nd_log = {
- .overwrite_process_source = 0,
- .journal = {
- .initialized = false,
- },
- .journal_direct = {
- .initialized = false,
- .fd = -1,
- },
- .syslog = {
- .initialized = false,
- .facility = LOG_DAEMON,
- },
- .std_output = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .initialized = false,
- },
- .std_error = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .initialized = false,
- },
- .sources = {
- [NDLS_UNSET] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DISABLED,
- .format = NDLF_JOURNAL,
- .filename = NULL,
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_EMERG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_ACCESS] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/access.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_ACLK] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_FILE,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/aclk.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_COLLECTORS] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/collector.log",
- .fd = STDERR_FILENO,
- .fp = NULL,
- .min_priority = NDLP_INFO,
- .limits = ND_LOG_LIMITS_DEFAULT,
- },
- [NDLS_DEBUG] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DISABLED,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/debug.log",
- .fd = STDOUT_FILENO,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_DAEMON] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .filename = LOG_DIR "/daemon.log",
- .format = NDLF_LOGFMT,
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_INFO,
- .limits = ND_LOG_LIMITS_DEFAULT,
- },
- [NDLS_HEALTH] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/health.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- },
-};
-
-__attribute__((constructor)) void initialize_invocation_id(void) {
- // check for a NETDATA_INVOCATION_ID
- if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
- // not found, check for systemd set INVOCATION_ID
- if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
- // not found, generate a new one
- uuid_generate_random(nd_log.invocation_id);
- }
- }
-
- char uuid[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
- nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);
-}
-
-int nd_log_health_fd(void) {
- if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
- return nd_log.sources[NDLS_HEALTH].fd;
-
- return STDERR_FILENO;
-}
-
-int nd_log_collectors_fd(void) {
- if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
- return nd_log.sources[NDLS_COLLECTORS].fd;
-
- return STDERR_FILENO;
-}
-
-void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
- char buf[FILENAME_MAX + 100];
- if(setting && *setting)
- strncpyz(buf, setting, sizeof(buf) - 1);
- else
- buf[0] = '\0';
-
- struct nd_log_source *ls = &nd_log.sources[source];
- char *output = strrchr(buf, '@');
-
- if(!output)
- // all of it is the output
- output = buf;
- else {
- // we found an '@', the next char is the output
- *output = '\0';
- output++;
-
- // parse the other params
- char *remaining = buf;
- while(remaining) {
- char *value = strsep_skip_consecutive_separators(&remaining, ",");
- if (!value || !*value) continue;
-
- char *name = strsep_skip_consecutive_separators(&value, "=");
- if (!name || !*name) continue;
-
- if(strcmp(name, "logfmt") == 0)
- ls->format = NDLF_LOGFMT;
- else if(strcmp(name, "json") == 0)
- ls->format = NDLF_JSON;
- else if(strcmp(name, "journal") == 0)
- ls->format = NDLF_JOURNAL;
- else if(strcmp(name, "level") == 0 && value && *value)
- ls->min_priority = nd_log_priority2id(value);
- else if(strcmp(name, "protection") == 0 && value && *value) {
- if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
- ls->limits = ND_LOG_LIMITS_UNLIMITED;
- ls->limits.counter = 0;
- ls->limits.prevented = 0;
- }
- else {
- ls->limits = ND_LOG_LIMITS_DEFAULT;
-
- char *slash = strchr(value, '/');
- if(slash) {
- *slash = '\0';
- slash++;
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
-
- int period;
- if(!duration_parse_seconds(slash, &period)) {
- nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash);
- period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
- }
-
- ls->limits.throttle_period = period;
- }
- else {
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
- ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
- }
- }
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Error while parsing configuration of log source '%s'. "
- "In config '%s', '%s' is not understood.",
- nd_log_id2source(source), setting, name);
- }
- }
-
- if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
- ls->method = NDLM_DISABLED;
- ls->filename = "/dev/null";
- }
- else if(strcmp(output, "journal") == 0) {
- ls->method = NDLM_JOURNAL;
- ls->filename = NULL;
- }
- else if(strcmp(output, "syslog") == 0) {
- ls->method = NDLM_SYSLOG;
- ls->filename = NULL;
- }
- else if(strcmp(output, "/dev/null") == 0) {
- ls->method = NDLM_DEVNULL;
- ls->filename = "/dev/null";
- }
- else if(strcmp(output, "system") == 0) {
- if(ls->fd == STDERR_FILENO) {
- ls->method = NDLM_STDERR;
- ls->filename = NULL;
- ls->fd = STDERR_FILENO;
- }
- else {
- ls->method = NDLM_STDOUT;
- ls->filename = NULL;
- ls->fd = STDOUT_FILENO;
- }
- }
- else if(strcmp(output, "stderr") == 0) {
- ls->method = NDLM_STDERR;
- ls->filename = NULL;
- ls->fd = STDERR_FILENO;
- }
- else if(strcmp(output, "stdout") == 0) {
- ls->method = NDLM_STDOUT;
- ls->filename = NULL;
- ls->fd = STDOUT_FILENO;
- }
- else {
- ls->method = NDLM_FILE;
- ls->filename = strdupz(output);
- }
-
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
- ls->min_priority = NDLP_DEBUG;
-#endif
-
- if(source == NDLS_COLLECTORS) {
- // set the method for the collector processes we will spawn
-
- ND_LOG_METHOD method;
- ND_LOG_FORMAT format = ls->format;
- ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
-
- if(ls->method == NDLM_SYSLOG || ls->method == NDLM_JOURNAL)
- method = ls->method;
- else
- method = NDLM_STDERR;
-
- nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
- nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
- nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
- }
-}
-
-void nd_log_set_priority_level(const char *setting) {
- if(!setting || !*setting)
- setting = "info";
-
- ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
-
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
- priority = NDLP_DEBUG;
-#endif
-
- for (size_t i = 0; i < _NDLS_MAX; i++) {
- if (i != NDLS_DEBUG)
- nd_log.sources[i].min_priority = priority;
- }
-
- // the right one
- nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
-}
-
-void nd_log_set_facility(const char *facility) {
- if(!facility || !*facility)
- facility = "daemon";
-
- nd_log.syslog.facility = nd_log_facility2id(facility);
- nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
-}
-
-void nd_log_set_flood_protection(size_t logs, time_t period) {
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
-
- nd_log.sources[NDLS_DAEMON].limits.throttle_period =
- nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
-
- char buf[100];
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
- nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
- nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
-}
-
-static bool nd_log_journal_systemd_init(void) {
-#ifdef HAVE_SYSTEMD
- nd_log.journal.initialized = true;
-#else
- nd_log.journal.initialized = false;
-#endif
-
- return nd_log.journal.initialized;
-}
-
-static void nd_log_journal_direct_set_env(void) {
- if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
- nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
-}
-
-static bool nd_log_journal_direct_init(const char *path) {
- if(nd_log.journal_direct.initialized) {
- nd_log_journal_direct_set_env();
- return true;
- }
-
- int fd;
- char filename[FILENAME_MAX + 1];
- if(!is_path_unix_socket(path)) {
-
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), NULL, "netdata");
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), NULL, NULL);
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
- return false;
- }
- }
- }
- }
- else {
- snprintfz(filename, sizeof(filename), "%s", path);
- fd = journal_direct_fd(filename);
- }
-
- if(fd < 0)
- return false;
-
- nd_log.journal_direct.fd = fd;
- nd_log.journal_direct.initialized = true;
-
- strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
- nd_log_journal_direct_set_env();
-
- return true;
-}
-
-static void nd_log_syslog_init() {
- if(nd_log.syslog.initialized)
- return;
-
- openlog(program_name, LOG_PID, nd_log.syslog.facility);
- nd_log.syslog.initialized = true;
-}
-
-void nd_log_initialize_for_external_plugins(const char *name) {
- // if we don't run under Netdata, log to stderr,
- // otherwise, use the logging method Netdata wants us to use.
- nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
- nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
-
- nd_log.overwrite_process_source = NDLS_COLLECTORS;
- program_name = name;
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].method = STDERR_FILENO;
- nd_log.sources[i].fd = -1;
- nd_log.sources[i].fp = NULL;
- }
-
- nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
- nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
-
- time_t period = 1200;
- size_t logs = 200;
- const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
- if(s && *s >= '0' && *s <= '9') {
- period = str2l(s);
- if(period < 0) period = 0;
- }
-
- s = getenv("NETDATA_ERRORS_PER_PERIOD");
- if(s && *s >= '0' && *s <= '9')
- logs = str2u(s);
-
- nd_log_set_flood_protection(logs, period);
-
- if(!netdata_configured_host_prefix) {
- s = getenv("NETDATA_HOST_PREFIX");
- if(s && *s)
- netdata_configured_host_prefix = (char *)s;
- }
-
- ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
- ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
-
- if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
- if(is_stderr_connected_to_journal()) {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
- method = NDLM_JOURNAL;
- }
- else {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
- method = NDLM_STDERR;
- }
- }
-
- switch(method) {
- case NDLM_JOURNAL:
- if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
- !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
- method = NDLM_STDERR;
- }
- break;
-
- case NDLM_SYSLOG:
- nd_log_syslog_init();
- break;
-
- default:
- method = NDLM_STDERR;
- break;
- }
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].method = method;
- nd_log.sources[i].format = format;
- nd_log.sources[i].fd = -1;
- nd_log.sources[i].fp = NULL;
- }
-
-// nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
-}
-
-static bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
- if(new_fd == -1 || e->fd == -1 ||
- (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
- (e->fd == STDERR_FILENO && nd_log.std_error.initialized))
- return false;
-
- if(new_fd != e->fd) {
- int t = dup2(new_fd, e->fd);
-
- bool ret = true;
- if (t == -1) {
- netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
- ret = false;
- }
- else
- close(new_fd);
-
- if(e->fd == STDOUT_FILENO)
- nd_log.std_output.initialized = true;
- else if(e->fd == STDERR_FILENO)
- nd_log.std_error.initialized = true;
-
- return ret;
- }
-
- return false;
-}
-
-static void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
- if(e->method == NDLM_DEFAULT)
- nd_log_set_user_settings(source, e->filename);
-
- if((e->method == NDLM_FILE && !e->filename) ||
- (e->method == NDLM_DEVNULL && e->fd == -1))
- e->method = NDLM_DISABLED;
-
- if(e->fp)
- fflush(e->fp);
-
- switch(e->method) {
- case NDLM_SYSLOG:
- nd_log_syslog_init();
- break;
-
- case NDLM_JOURNAL:
- nd_log_journal_direct_init(NULL);
- nd_log_journal_systemd_init();
- break;
-
- case NDLM_STDOUT:
- e->fp = stdout;
- e->fd = STDOUT_FILENO;
- break;
-
- case NDLM_DISABLED:
- break;
-
- case NDLM_DEFAULT:
- case NDLM_STDERR:
- e->method = NDLM_STDERR;
- e->fp = stderr;
- e->fd = STDERR_FILENO;
- break;
-
- case NDLM_DEVNULL:
- case NDLM_FILE: {
- int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
- if(fd == -1) {
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
- e->fd = STDERR_FILENO;
- e->method = NDLM_STDERR;
- netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
- }
- else
- netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
- }
- else {
- if (!nd_log_replace_existing_fd(e, fd)) {
- if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
- if(e->fd == STDOUT_FILENO)
- e->method = NDLM_STDOUT;
- else if(e->fd == STDERR_FILENO)
- e->method = NDLM_STDERR;
-
- // we have dup2() fd, so we can close the one we opened
- if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
- close(fd);
- }
- else
- e->fd = fd;
- }
- }
-
- // at this point we have e->fd set properly
-
- if(e->fd == STDOUT_FILENO)
- e->fp = stdout;
- else if(e->fd == STDERR_FILENO)
- e->fp = stderr;
-
- if(!e->fp) {
- e->fp = fdopen(e->fd, "a");
- if (!e->fp) {
- netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
-
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
- close(e->fd);
-
- e->fp = stderr;
- e->fd = STDERR_FILENO;
- }
- }
- else {
- if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
- netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
- }
- }
- break;
- }
-}
-
-static void nd_log_stdin_init(int fd, const char *filename) {
- int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
- if(f == -1)
- return;
-
- if(f != fd) {
- dup2(f, fd);
- close(f);
- }
-}
-
-void nd_log_initialize(void) {
- nd_log_stdin_init(STDIN_FILENO, "/dev/null");
-
- for(size_t i = 0 ; i < _NDLS_MAX ; i++)
- nd_log_open(&nd_log.sources[i], i);
-}
-
-void nd_log_reopen_log_files(bool log) {
- if(log)
- netdata_log_info("Reopening all log files.");
-
- nd_log.std_output.initialized = false;
- nd_log.std_error.initialized = false;
- nd_log_initialize();
-
- if(log)
- netdata_log_info("Log files re-opened.");
-}
-
-void nd_log_reopen_log_files_for_spawn_server(void) {
- if(nd_log.syslog.initialized) {
- closelog();
- nd_log.syslog.initialized = false;
- nd_log_syslog_init();
- }
-
- if(nd_log.journal_direct.initialized) {
- close(nd_log.journal_direct.fd);
- nd_log.journal_direct.fd = -1;
- nd_log.journal_direct.initialized = false;
- nd_log_journal_direct_init(NULL);
- }
-
- nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
- nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
- nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
- nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
- nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
- nd_log_reopen_log_files(false);
-}
-
-void chown_open_file(int fd, uid_t uid, gid_t gid) {
- if(fd == -1) return;
-
- struct stat buf;
-
- if(fstat(fd, &buf) == -1) {
- netdata_log_error("Cannot fstat() fd %d", fd);
- return;
- }
-
- if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
- if(fchown(fd, uid, gid) == -1)
- netdata_log_error("Cannot fchown() fd %d.", fd);
- }
-}
-
-void nd_log_chown_log_files(uid_t uid, gid_t gid) {
- for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
- if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
- chown_open_file(nd_log.sources[i].fd, uid, gid);
- }
-}
-
-// ----------------------------------------------------------------------------
-// annotators
-struct log_field;
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-
-#if defined(OS_WINDOWS)
-static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-#endif
-
-// ----------------------------------------------------------------------------
-
-typedef void (*annotator_t)(BUFFER *wb, const char *key, struct log_field *lf);
-
-struct log_field {
- const char *journal;
- const char *logfmt;
- annotator_t logfmt_annotator;
- struct log_stack_entry entry;
-};
-
-#define THREAD_LOG_STACK_MAX 50
-
-static __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
-static __thread size_t thread_log_stack_next = 0;
-
-static __thread struct log_field thread_log_fields[_NDF_MAX] = {
- // THE ORDER DEFINES THE ORDER FIELDS WILL APPEAR IN logfmt
-
- [NDF_STOP] = { // processing will not stop on this - so it is ok to be first
- .journal = NULL,
- .logfmt = NULL,
- .logfmt_annotator = NULL,
- },
- [NDF_TIMESTAMP_REALTIME_USEC] = {
- .journal = NULL,
- .logfmt = "time",
- .logfmt_annotator = timestamp_usec_annotator,
- },
- [NDF_SYSLOG_IDENTIFIER] = {
- .journal = "SYSLOG_IDENTIFIER", // standard journald field
- .logfmt = "comm",
- },
- [NDF_LOG_SOURCE] = {
- .journal = "ND_LOG_SOURCE",
- .logfmt = "source",
- },
- [NDF_PRIORITY] = {
- .journal = "PRIORITY", // standard journald field
- .logfmt = "level",
- .logfmt_annotator = priority_annotator,
- },
- [NDF_ERRNO] = {
- .journal = "ERRNO", // standard journald field
- .logfmt = "errno",
- .logfmt_annotator = errno_annotator,
- },
-#if defined(OS_WINDOWS)
- [NDF_WINERROR] = {
- .journal = "WINERROR",
- .logfmt = "winerror",
- .logfmt_annotator = winerror_annotator,
- },
-#endif
- [NDF_INVOCATION_ID] = {
- .journal = "INVOCATION_ID", // standard journald field
- .logfmt = NULL,
- },
- [NDF_LINE] = {
- .journal = "CODE_LINE", // standard journald field
- .logfmt = NULL,
- },
- [NDF_FILE] = {
- .journal = "CODE_FILE", // standard journald field
- .logfmt = NULL,
- },
- [NDF_FUNC] = {
- .journal = "CODE_FUNC", // standard journald field
- .logfmt = NULL,
- },
- [NDF_TID] = {
- .journal = "TID", // standard journald field
- .logfmt = "tid",
- },
- [NDF_THREAD_TAG] = {
- .journal = "THREAD_TAG",
- .logfmt = "thread",
- },
- [NDF_MESSAGE_ID] = {
- .journal = "MESSAGE_ID",
- .logfmt = "msg_id",
- },
- [NDF_MODULE] = {
- .journal = "ND_MODULE",
- .logfmt = "module",
- },
- [NDF_NIDL_NODE] = {
- .journal = "ND_NIDL_NODE",
- .logfmt = "node",
- },
- [NDF_NIDL_INSTANCE] = {
- .journal = "ND_NIDL_INSTANCE",
- .logfmt = "instance",
- },
- [NDF_NIDL_CONTEXT] = {
- .journal = "ND_NIDL_CONTEXT",
- .logfmt = "context",
- },
- [NDF_NIDL_DIMENSION] = {
- .journal = "ND_NIDL_DIMENSION",
- .logfmt = "dimension",
- },
- [NDF_SRC_TRANSPORT] = {
- .journal = "ND_SRC_TRANSPORT",
- .logfmt = "src_transport",
- },
- [NDF_ACCOUNT_ID] = {
- .journal = "ND_ACCOUNT_ID",
- .logfmt = "account",
- },
- [NDF_USER_NAME] = {
- .journal = "ND_USER_NAME",
- .logfmt = "user",
- },
- [NDF_USER_ROLE] = {
- .journal = "ND_USER_ROLE",
- .logfmt = "role",
- },
- [NDF_USER_ACCESS] = {
- .journal = "ND_USER_PERMISSIONS",
- .logfmt = "permissions",
- },
- [NDF_SRC_IP] = {
- .journal = "ND_SRC_IP",
- .logfmt = "src_ip",
- },
- [NDF_SRC_FORWARDED_HOST] = {
- .journal = "ND_SRC_FORWARDED_HOST",
- .logfmt = "src_forwarded_host",
- },
- [NDF_SRC_FORWARDED_FOR] = {
- .journal = "ND_SRC_FORWARDED_FOR",
- .logfmt = "src_forwarded_for",
- },
- [NDF_SRC_PORT] = {
- .journal = "ND_SRC_PORT",
- .logfmt = "src_port",
- },
- [NDF_SRC_CAPABILITIES] = {
- .journal = "ND_SRC_CAPABILITIES",
- .logfmt = "src_capabilities",
- },
- [NDF_DST_TRANSPORT] = {
- .journal = "ND_DST_TRANSPORT",
- .logfmt = "dst_transport",
- },
- [NDF_DST_IP] = {
- .journal = "ND_DST_IP",
- .logfmt = "dst_ip",
- },
- [NDF_DST_PORT] = {
- .journal = "ND_DST_PORT",
- .logfmt = "dst_port",
- },
- [NDF_DST_CAPABILITIES] = {
- .journal = "ND_DST_CAPABILITIES",
- .logfmt = "dst_capabilities",
- },
- [NDF_REQUEST_METHOD] = {
- .journal = "ND_REQUEST_METHOD",
- .logfmt = "req_method",
- },
- [NDF_RESPONSE_CODE] = {
- .journal = "ND_RESPONSE_CODE",
- .logfmt = "code",
- },
- [NDF_CONNECTION_ID] = {
- .journal = "ND_CONNECTION_ID",
- .logfmt = "conn",
- },
- [NDF_TRANSACTION_ID] = {
- .journal = "ND_TRANSACTION_ID",
- .logfmt = "transaction",
- },
- [NDF_RESPONSE_SENT_BYTES] = {
- .journal = "ND_RESPONSE_SENT_BYTES",
- .logfmt = "sent_bytes",
- },
- [NDF_RESPONSE_SIZE_BYTES] = {
- .journal = "ND_RESPONSE_SIZE_BYTES",
- .logfmt = "size_bytes",
- },
- [NDF_RESPONSE_PREPARATION_TIME_USEC] = {
- .journal = "ND_RESPONSE_PREP_TIME_USEC",
- .logfmt = "prep_ut",
- },
- [NDF_RESPONSE_SENT_TIME_USEC] = {
- .journal = "ND_RESPONSE_SENT_TIME_USEC",
- .logfmt = "sent_ut",
- },
- [NDF_RESPONSE_TOTAL_TIME_USEC] = {
- .journal = "ND_RESPONSE_TOTAL_TIME_USEC",
- .logfmt = "total_ut",
- },
- [NDF_ALERT_ID] = {
- .journal = "ND_ALERT_ID",
- .logfmt = "alert_id",
- },
- [NDF_ALERT_UNIQUE_ID] = {
- .journal = "ND_ALERT_UNIQUE_ID",
- .logfmt = "alert_unique_id",
- },
- [NDF_ALERT_TRANSITION_ID] = {
- .journal = "ND_ALERT_TRANSITION_ID",
- .logfmt = "alert_transition_id",
- },
- [NDF_ALERT_EVENT_ID] = {
- .journal = "ND_ALERT_EVENT_ID",
- .logfmt = "alert_event_id",
- },
- [NDF_ALERT_CONFIG_HASH] = {
- .journal = "ND_ALERT_CONFIG",
- .logfmt = "alert_config",
- },
- [NDF_ALERT_NAME] = {
- .journal = "ND_ALERT_NAME",
- .logfmt = "alert",
- },
- [NDF_ALERT_CLASS] = {
- .journal = "ND_ALERT_CLASS",
- .logfmt = "alert_class",
- },
- [NDF_ALERT_COMPONENT] = {
- .journal = "ND_ALERT_COMPONENT",
- .logfmt = "alert_component",
- },
- [NDF_ALERT_TYPE] = {
- .journal = "ND_ALERT_TYPE",
- .logfmt = "alert_type",
- },
- [NDF_ALERT_EXEC] = {
- .journal = "ND_ALERT_EXEC",
- .logfmt = "alert_exec",
- },
- [NDF_ALERT_RECIPIENT] = {
- .journal = "ND_ALERT_RECIPIENT",
- .logfmt = "alert_recipient",
- },
- [NDF_ALERT_VALUE] = {
- .journal = "ND_ALERT_VALUE",
- .logfmt = "alert_value",
- },
- [NDF_ALERT_VALUE_OLD] = {
- .journal = "ND_ALERT_VALUE_OLD",
- .logfmt = "alert_value_old",
- },
- [NDF_ALERT_STATUS] = {
- .journal = "ND_ALERT_STATUS",
- .logfmt = "alert_status",
- },
- [NDF_ALERT_STATUS_OLD] = {
- .journal = "ND_ALERT_STATUS_OLD",
- .logfmt = "alert_value_old",
- },
- [NDF_ALERT_UNITS] = {
- .journal = "ND_ALERT_UNITS",
- .logfmt = "alert_units",
- },
- [NDF_ALERT_SUMMARY] = {
- .journal = "ND_ALERT_SUMMARY",
- .logfmt = "alert_summary",
- },
- [NDF_ALERT_INFO] = {
- .journal = "ND_ALERT_INFO",
- .logfmt = "alert_info",
- },
- [NDF_ALERT_DURATION] = {
- .journal = "ND_ALERT_DURATION",
- .logfmt = "alert_duration",
- },
- [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
- .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
- .logfmt = "alert_notification_timestamp",
- .logfmt_annotator = timestamp_usec_annotator,
- },
-
- // put new items here
- // leave the request URL and the message last
-
- [NDF_REQUEST] = {
- .journal = "ND_REQUEST",
- .logfmt = "request",
- },
- [NDF_MESSAGE] = {
- .journal = "MESSAGE",
- .logfmt = "msg",
- },
-};
-
-#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
-
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len) {
- for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
- if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
- return i;
- }
-
- return NDF_STOP;
-}
-
-void log_stack_pop(void *ptr) {
- if(!ptr) return;
-
- struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
-
- if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
- fatal("You cannot pop in the middle of the stack, or an item not in the stack");
- return;
- }
-
- thread_log_stack_next--;
-}
-
-void log_stack_push(struct log_stack_entry *lgs) {
- if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
- thread_log_stack_base[thread_log_stack_next++] = lgs;
-}
-
-// ----------------------------------------------------------------------------
-// json formatter
-
-static void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].logfmt)
- continue;
-
- const char *key = fields[i].logfmt;
-
- const char *s = NULL;
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- s = fields[i].entry.txt;
- break;
- case NDFT_STR:
- s = string2str(fields[i].entry.str);
- break;
- case NDFT_BFR:
- s = buffer_tostring(fields[i].entry.bfr);
- break;
- case NDFT_U64:
- buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
- break;
- case NDFT_I64:
- buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
- break;
- case NDFT_UUID:
- if(!uuid_is_null(*fields[i].entry.uuid)) {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_json_member_add_string(wb, key, u);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- }
- break;
- default:
- s = "UNHANDLED";
- break;
- }
-
- if(s && *s)
- buffer_json_member_add_string(wb, key, s);
- }
-
- buffer_json_finalize(wb);
-}
-
-// ----------------------------------------------------------------------------
-// logfmt formatter
-
-
-static int64_t log_field_to_int64(struct log_field *lf) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
- const char *s = NULL;
-
- switch(lf->entry.type) {
- case NDFT_UUID:
- case NDFT_UNSET:
- return 0;
-
- case NDFT_TXT:
- s = lf->entry.txt;
- break;
-
- case NDFT_STR:
- s = string2str(lf->entry.str);
- break;
-
- case NDFT_BFR:
- s = buffer_tostring(lf->entry.bfr);
- break;
-
- case NDFT_CALLBACK:
- tmp = buffer_create(0, NULL);
-
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- break;
-
- case NDFT_U64:
- return (int64_t)lf->entry.u64;
-
- case NDFT_I64:
- return (int64_t)lf->entry.i64;
-
- case NDFT_DBL:
- return (int64_t)lf->entry.dbl;
- }
-
- if(s && *s)
- return str2ll(s, NULL);
-
- return 0;
-}
-
-static uint64_t log_field_to_uint64(struct log_field *lf) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
- const char *s = NULL;
-
- switch(lf->entry.type) {
- case NDFT_UUID:
- case NDFT_UNSET:
- return 0;
-
- case NDFT_TXT:
- s = lf->entry.txt;
- break;
-
- case NDFT_STR:
- s = string2str(lf->entry.str);
- break;
-
- case NDFT_BFR:
- s = buffer_tostring(lf->entry.bfr);
- break;
-
- case NDFT_CALLBACK:
- tmp = buffer_create(0, NULL);
-
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- break;
-
- case NDFT_U64:
- return lf->entry.u64;
-
- case NDFT_I64:
- return lf->entry.i64;
-
- case NDFT_DBL:
- return (uint64_t) lf->entry.dbl;
- }
-
- if(s && *s)
- return str2uint64_t(s, NULL);
-
- return 0;
-}
-
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- usec_t ut = log_field_to_uint64(lf);
-
- if(!ut)
- return;
-
- char datetime[RFC3339_MAX_LENGTH];
- rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_json_strcat(wb, datetime);
-}
-
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- int64_t errnum = log_field_to_int64(lf);
-
- if(errnum == 0)
- return;
-
- char buf[1024];
- const char *s = errno2str((int)errnum, buf, sizeof(buf));
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=\"", 2);
- buffer_print_int64(wb, errnum);
- buffer_fast_strcat(wb, ", ", 2);
- buffer_json_strcat(wb, s);
- buffer_fast_strcat(wb, "\"", 1);
-}
-
-#if defined(OS_WINDOWS)
-static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- DWORD errnum = log_field_to_uint64(lf);
-
- if (errnum == 0)
- return;
-
- char buf[1024];
- wchar_t wbuf[1024];
- DWORD size = FormatMessageW(
- FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
- NULL,
- errnum,
- MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
- wbuf,
- (DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1),
- NULL
- );
-
- if (size > 0) {
- // Remove \r\n at the end
- while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n'))
- wbuf[--size] = L'\0';
-
- // Convert wide string to UTF-8
- int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, buf, sizeof(buf), NULL, NULL);
- if (utf8_size == 0)
- snprintf(buf, sizeof(buf) - 1, "unknown error code");
- buf[sizeof(buf) - 1] = '\0';
- }
- else
- snprintf(buf, sizeof(buf) - 1, "unknown error code");
-
- if (buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=\"", 2);
- buffer_print_int64(wb, errnum);
- buffer_fast_strcat(wb, ", ", 2);
- buffer_json_strcat(wb, buf);
- buffer_fast_strcat(wb, "\"", 1);
-}
-#endif
-
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- uint64_t pri = log_field_to_uint64(lf);
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_strcat(wb, nd_log_id2priority(pri));
-}
-
-static bool needs_quotes_for_logfmt(const char *s)
-{
- static bool safe_for_logfmt[256] = {
- [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
- ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
- ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
- ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
- ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
- ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
- ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
- ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
- ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
- ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
- ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
- ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
- ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
- ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
- };
-
- if(!*s)
- return true;
-
- while(*s) {
- if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s])
- return true;
-
- s++;
- }
-
- return false;
-}
-
-static void string_to_logfmt(BUFFER *wb, const char *s)
-{
- bool spaces = needs_quotes_for_logfmt(s);
-
- if(spaces)
- buffer_fast_strcat(wb, "\"", 1);
-
- buffer_json_strcat(wb, s);
-
- if(spaces)
- buffer_fast_strcat(wb, "\"", 1);
-}
-
-static void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max)
-{
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].logfmt)
- continue;
-
- const char *key = fields[i].logfmt;
-
- if(fields[i].logfmt_annotator)
- fields[i].logfmt_annotator(wb, key, &fields[i]);
- else {
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- if(*fields[i].entry.txt) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, fields[i].entry.txt);
- }
- break;
- case NDFT_STR:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, string2str(fields[i].entry.str));
- break;
- case NDFT_BFR:
- if(buffer_strlen(fields[i].entry.bfr)) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
- }
- break;
- case NDFT_U64:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_uint64(wb, fields[i].entry.u64);
- break;
- case NDFT_I64:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_int64(wb, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
- break;
- case NDFT_UUID:
- if(!uuid_is_null(*fields[i].entry.uuid)) {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, buffer_tostring(tmp));
- }
- }
- break;
- default:
- buffer_strcat(wb, "UNHANDLED");
- break;
- }
- }
- }
-}
-
-// ----------------------------------------------------------------------------
-// journal logger
-
-bool nd_log_journal_socket_available(void) {
- if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
- char filename[FILENAME_MAX + 1];
-
- snprintfz(filename, sizeof(filename), "%s%s",
- netdata_configured_host_prefix, "/run/systemd/journal/socket");
-
- if(is_path_unix_socket(filename))
- return true;
- }
-
- return is_path_unix_socket("/run/systemd/journal/socket");
-}
-
-static bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
-#ifdef HAVE_SYSTEMD
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- struct iovec iov[fields_max];
- int iov_count = 0;
-
- memset(iov, 0, sizeof(iov));
-
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].journal)
- continue;
-
- const char *key = fields[i].journal;
- char *value = NULL;
- int rc = 0;
- switch (fields[i].entry.type) {
- case NDFT_TXT:
- if(*fields[i].entry.txt)
- rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt);
- break;
- case NDFT_STR:
- rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
- break;
- case NDFT_BFR:
- if(buffer_strlen(fields[i].entry.bfr))
- rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
- break;
- case NDFT_U64:
- rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
- break;
- case NDFT_I64:
- rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
- break;
- case NDFT_UUID:
- if(!uuid_is_null(*fields[i].entry.uuid)) {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- rc = asprintf(&value, "%s=%s", key, u);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
- }
- break;
- default:
- rc = asprintf(&value, "%s=%s", key, "UNHANDLED");
- break;
- }
-
- if (rc != -1 && value) {
- iov[iov_count].iov_base = value;
- iov[iov_count].iov_len = strlen(value);
- iov_count++;
- }
- }
-
- int r = sd_journal_sendv(iov, iov_count);
-
- // Clean up allocated memory
- for (int i = 0; i < iov_count; i++) {
- if (iov[i].iov_base != NULL) {
- free(iov[i].iov_base);
- }
- }
-
- return r == 0;
-#else
- return false;
-#endif
-}
-
-static bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
- if(!nd_log.journal_direct.initialized)
- return false;
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *wb = buffer_create(4096, NULL);
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].journal)
- continue;
-
- const char *key = fields[i].journal;
-
- const char *s = NULL;
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- s = fields[i].entry.txt;
- break;
- case NDFT_STR:
- s = string2str(fields[i].entry.str);
- break;
- case NDFT_BFR:
- s = buffer_tostring(fields[i].entry.bfr);
- break;
- case NDFT_U64:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_uint64(wb, fields[i].entry.u64);
- buffer_putc(wb, '\n');
- break;
- case NDFT_I64:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_int64(wb, fields[i].entry.i64);
- buffer_putc(wb, '\n');
- break;
- case NDFT_DBL:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
- buffer_putc(wb, '\n');
- break;
- case NDFT_UUID:
- if(!uuid_is_null(*fields[i].entry.uuid)) {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
- buffer_putc(wb, '\n');
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- }
- break;
- default:
- s = "UNHANDLED";
- break;
- }
-
- if(s && *s) {
- buffer_strcat(wb, key);
- if(!strchr(s, '\n')) {
- buffer_putc(wb, '=');
- buffer_strcat(wb, s);
- buffer_putc(wb, '\n');
- }
- else {
- buffer_putc(wb, '\n');
- size_t size = strlen(s);
- uint64_t le_size = htole64(size);
- buffer_memcat(wb, &le_size, sizeof(le_size));
- buffer_memcat(wb, s, size);
- buffer_putc(wb, '\n');
- }
- }
- }
-
- return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
-}
-
-// ----------------------------------------------------------------------------
-// syslog logger - uses logfmt
-
-static bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) {
- CLEAN_BUFFER *wb = buffer_create(1024, NULL);
-
- nd_logger_logfmt(wb, fields, fields_max);
- syslog(priority, "%s", buffer_tostring(wb));
-
- return true;
-}
-
-// ----------------------------------------------------------------------------
-// file logger - uses logfmt
-
-static bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
- BUFFER *wb = buffer_create(1024, NULL);
-
- if(format == NDLF_JSON)
- nd_logger_json(wb, fields, fields_max);
- else
- nd_logger_logfmt(wb, fields, fields_max);
-
- int r = fprintf(fp, "%s\n", buffer_tostring(wb));
- fflush(fp);
-
- buffer_free(wb);
- return r > 0;
-}
-
-// ----------------------------------------------------------------------------
-// logger router
-
-static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
- *spinlock = NULL;
- ND_LOG_METHOD output = nd_log.sources[source].method;
-
- switch(output) {
- case NDLM_JOURNAL:
- if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
- output = NDLM_FILE;
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- }
- else {
- *fpp = NULL;
- *spinlock = NULL;
- }
- break;
-
- case NDLM_SYSLOG:
- if(unlikely(!nd_log.syslog.initialized)) {
- output = NDLM_FILE;
- *spinlock = &nd_log.std_error.spinlock;
- *fpp = stderr;
- }
- else {
- *spinlock = NULL;
- *fpp = NULL;
- }
- break;
-
- case NDLM_FILE:
- if(!nd_log.sources[source].fp) {
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- }
- else {
- *fpp = nd_log.sources[source].fp;
- *spinlock = &nd_log.sources[source].spinlock;
- }
- break;
-
- case NDLM_STDOUT:
- output = NDLM_FILE;
- *fpp = stdout;
- *spinlock = &nd_log.std_output.spinlock;
- break;
-
- default:
- case NDLM_DEFAULT:
- case NDLM_STDERR:
- output = NDLM_FILE;
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- break;
-
- case NDLM_DISABLED:
- case NDLM_DEVNULL:
- output = NDLM_DISABLED;
- *fpp = NULL;
- *spinlock = NULL;
- break;
- }
-
- return output;
-}
-
-// ----------------------------------------------------------------------------
-// high level logger
-
-static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
- ND_LOG_METHOD output, struct nd_log_source *source,
- struct log_field *fields, size_t fields_max) {
- if(spinlock)
- spinlock_lock(spinlock);
-
- // check the limits
- if(limit && nd_log_limit_reached(source))
- goto cleanup;
-
- if(output == NDLM_JOURNAL) {
- if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
- // we can't log to journal, let's log to stderr
- if(spinlock)
- spinlock_unlock(spinlock);
-
- output = NDLM_FILE;
- spinlock = &nd_log.std_error.spinlock;
- fp = stderr;
-
- if(spinlock)
- spinlock_lock(spinlock);
- }
- }
-
- if(output == NDLM_SYSLOG)
- nd_logger_syslog(priority, source->format, fields, fields_max);
-
- if(output == NDLM_FILE)
- nd_logger_file(fp, source->format, fields, fields_max);
-
-
-cleanup:
- if(spinlock)
- spinlock_unlock(spinlock);
-}
-
-static void nd_logger_unset_all_thread_fields(void) {
- size_t fields_max = THREAD_FIELDS_MAX;
- for(size_t i = 0; i < fields_max ; i++)
- thread_log_fields[i].entry.set = false;
-}
-
-static void nd_logger_merge_log_stack_to_thread_fields(void) {
- for(size_t c = 0; c < thread_log_stack_next ;c++) {
- struct log_stack_entry *lgs = thread_log_stack_base[c];
-
- for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
- if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
- continue;
-
- struct log_stack_entry *e = &lgs[i];
- ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
-
- // do not add empty / unset fields
- if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
- (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
- (type == NDFT_STR && !e->str) ||
- (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
- (type == NDFT_CALLBACK && !e->cb.formatter) ||
- type == NDFT_UNSET)
- continue;
-
- thread_log_fields[lgs[i].id].entry = *e;
- }
- }
-}
-
-static void nd_logger(const char *file, const char *function, const unsigned long line,
- ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,
- int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {
-
- SPINLOCK *spinlock;
- FILE *fp;
- ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
- return;
-
- // mark all fields as unset
- nd_logger_unset_all_thread_fields();
-
- // flatten the log stack into the fields
- nd_logger_merge_log_stack_to_thread_fields();
-
- // set the common fields that are automatically set by the logging subsystem
-
- if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
- thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
-
- if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
- thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
- else {
- ND_LOG_SOURCES src = source;
-
- if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
- src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
- else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
- src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
-
- if(src != source && src < _NDLS_MAX) {
- source = src;
- output = nd_logger_select_output(source, &fp, &spinlock);
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
- return;
- }
- }
-
- if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
-
- if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
- thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
- thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
- thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
- }
-
- if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
- thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
- }
-
- if(likely(!thread_log_fields[NDF_TID].entry.set))
- thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached());
-
- if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
- const char *thread_tag = nd_thread_tag();
- thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
-
- // TODO: fix the ND_MODULE in logging by setting proper module name in threads
-// if(!thread_log_fields[NDF_MODULE].entry.set)
-// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
- }
-
- if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
-
- if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
- thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
-
-#if defined(OS_WINDOWS)
- if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)
- thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);
-#endif
-
- CLEAN_BUFFER *wb = NULL;
- if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
- wb = buffer_create(1024, NULL);
- buffer_vsprintf(wb, fmt, ap);
- thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
- }
-
- nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
- thread_log_fields, THREAD_FIELDS_MAX);
-
- if(nd_log.sources[source].pending_msg) {
- // log a pending message
-
- nd_logger_unset_all_thread_fields();
-
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_U64,
- .u64 = now_realtime_usec(),
- };
-
- thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = nd_log_id2source(source),
- };
-
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = program_name,
- };
-
- thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = nd_log.sources[source].pending_msg,
- };
-
- nd_logger_log_fields(spinlock, fp, false, priority, output,
- &nd_log.sources[source],
- thread_log_fields, THREAD_FIELDS_MAX);
-
- freez((void *)nd_log.sources[source].pending_msg);
- nd_log.sources[source].pending_msg = NULL;
- }
-
- errno_clear();
-}
-
-static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
- if(source >= _NDLS_MAX)
- source = NDLS_DAEMON;
-
- if(nd_log.overwrite_process_source)
- source = nd_log.overwrite_process_source;
-
- return source;
-}
-
-// ----------------------------------------------------------------------------
-// public API for loggers
-
-void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )
-{
- int saved_errno = errno;
-
- size_t saved_winerror = 0;
-#if defined(OS_WINDOWS)
- saved_winerror = GetLastError();
-#endif
-
- source = nd_log_validate_source(source);
-
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
- return;
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, priority,
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
- saved_errno, saved_winerror, fmt, args);
- va_end(args);
-}
-
-void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
- int saved_errno = errno;
-
- size_t saved_winerror = 0;
-#if defined(OS_WINDOWS)
- saved_winerror = GetLastError();
-#endif
-
- source = nd_log_validate_source(source);
-
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
- return;
-
- if(erl->sleep_ut)
- sleep_usec(erl->sleep_ut);
-
- spinlock_lock(&erl->spinlock);
-
- erl->count++;
- time_t now = now_boottime_sec();
- if(now - erl->last_logged < erl->log_every) {
- spinlock_unlock(&erl->spinlock);
- return;
- }
-
- spinlock_unlock(&erl->spinlock);
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, priority,
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
- saved_errno, saved_winerror, fmt, args);
- va_end(args);
- erl->last_logged = now;
- erl->count = 0;
-}
-
-void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
- int saved_errno = errno;
-
- size_t saved_winerror = 0;
-#if defined(OS_WINDOWS)
- saved_winerror = GetLastError();
-#endif
-
- ND_LOG_SOURCES source = NDLS_DAEMON;
- source = nd_log_validate_source(source);
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);
- va_end(args);
-
- char date[LOG_DATE_LENGTH];
- log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
-
- char action_data[70+1];
- snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
-
- const char *thread_tag = nd_thread_tag();
- const char *tag_to_send = thread_tag;
-
- // anonymize thread names
- if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
- tag_to_send = THREAD_TAG_STREAM_RECEIVER;
- if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
- tag_to_send = THREAD_TAG_STREAM_SENDER;
-
- char action_result[60+1];
- snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
-
-#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
- int fd = nd_log.sources[NDLS_DAEMON].fd;
- if(fd == -1)
- fd = STDERR_FILENO;
-
- int nptrs;
- void *buffer[10000];
-
- nptrs = backtrace(buffer, sizeof(buffer));
- if(nptrs)
- backtrace_symbols_fd(buffer, nptrs, fd);
-#endif
-
-#ifdef NETDATA_INTERNAL_CHECKS
- abort();
-#endif
-
- netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);
-}
-
-// ----------------------------------------------------------------------------
-// log limits
-
-void nd_log_limits_reset(void) {
- usec_t now_ut = now_monotonic_usec();
-
- spinlock_lock(&nd_log.std_output.spinlock);
- spinlock_lock(&nd_log.std_error.spinlock);
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- spinlock_lock(&nd_log.sources[i].spinlock);
- nd_log.sources[i].limits.prevented = 0;
- nd_log.sources[i].limits.counter = 0;
- nd_log.sources[i].limits.started_monotonic_ut = now_ut;
- nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
- spinlock_unlock(&nd_log.sources[i].spinlock);
- }
-
- spinlock_unlock(&nd_log.std_output.spinlock);
- spinlock_unlock(&nd_log.std_error.spinlock);
-}
-
-void nd_log_limits_unlimited(void) {
- nd_log_limits_reset();
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].limits.logs_per_period = 0;
- }
-}
-
-static bool nd_log_limit_reached(struct nd_log_source *source) {
- if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
- return false;
-
- usec_t now_ut = now_monotonic_usec();
- if(!source->limits.started_monotonic_ut)
- source->limits.started_monotonic_ut = now_ut;
-
- source->limits.counter++;
-
- if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
- if(source->limits.prevented) {
- BUFFER *wb = buffer_create(1024, NULL);
- buffer_sprintf(wb,
- "LOG FLOOD PROTECTION: resuming logging "
- "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
- source->limits.prevented,
- source->limits.throttle_period);
-
- if(source->pending_msg)
- freez((void *)source->pending_msg);
-
- source->pending_msg = strdupz(buffer_tostring(wb));
-
- buffer_free(wb);
- }
-
- // restart the period accounting
- source->limits.started_monotonic_ut = now_ut;
- source->limits.counter = 1;
- source->limits.prevented = 0;
-
- // log this error
- return false;
- }
-
- if(source->limits.counter > source->limits.logs_per_period) {
- if(!source->limits.prevented) {
- BUFFER *wb = buffer_create(1024, NULL);
- buffer_sprintf(wb,
- "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
- "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
- source->limits.counter,
- (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
- source->limits.logs_per_period,
- source->limits.throttle_period,
- program_name,
- (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC)
- );
-
- if(source->pending_msg)
- freez((void *)source->pending_msg);
-
- source->pending_msg = strdupz(buffer_tostring(wb));
-
- buffer_free(wb);
- }
-
- source->limits.prevented++;
-
- // prevent logging this error
-#ifdef NETDATA_INTERNAL_CHECKS
- return false;
-#else
- return true;
-#endif
- }
-
- return false;
-}
diff --git a/src/libnetdata/log/nd_log-annotators.c b/src/libnetdata/log/nd_log-annotators.c
new file mode 100644
index 0000000000..92e9bf3100
--- /dev/null
+++ b/src/libnetdata/log/nd_log-annotators.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+const char *timestamp_usec_annotator(struct log_field *lf) {
+ usec_t ut = log_field_to_uint64(lf);
+
+ if(!ut)
+ return NULL;
+
+ static __thread char datetime[RFC3339_MAX_LENGTH];
+ rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
+ return datetime;
+}
+
+const char *errno_annotator(struct log_field *lf) {
+ int64_t errnum = log_field_to_int64(lf);
+
+ if(errnum == 0)
+ return NULL;
+
+ static __thread char buf[256];
+ size_t len = print_uint64(buf, errnum);
+ buf[len++] = ',';
+ buf[len++] = ' ';
+
+ char *msg_to = &buf[len];
+ size_t msg_size = sizeof(buf) - len;
+
+ const char *s = errno2str((int)errnum, msg_to, msg_size);
+ if(s != msg_to)
+ strncpyz(msg_to, s, msg_size - 1);
+
+ return buf;
+}
+
+#if defined(OS_WINDOWS)
+const char *winerror_annotator(struct log_field *lf) {
+ DWORD errnum = log_field_to_uint64(lf);
+
+ if (errnum == 0)
+ return NULL;
+
+ static __thread char buf[256];
+ size_t len = print_uint64(buf, errnum);
+ buf[len++] = ',';
+ buf[len++] = ' ';
+
+ char *msg_to = &buf[len];
+ size_t msg_size = sizeof(buf) - len;
+
+ wchar_t wbuf[1024];
+ DWORD size = FormatMessageW(
+ FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL,
+ errnum,
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ wbuf,
+ (DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1),
+ NULL
+ );
+
+ if (size > 0) {
+ // Remove \r\n at the end
+ while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n'))
+ wbuf[--size] = L'\0';
+
+ // Convert wide string to UTF-8
+ int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, msg_to, (int)msg_size, NULL, NULL);
+ if (utf8_size == 0)
+ snprintf(msg_to, msg_size - 1, "unknown error code");
+ msg_to[msg_size - 1] = '\0';
+ }
+ else
+ snprintf(msg_to, msg_size - 1, "unknown error code");
+
+ return buf;
+}
+#endif
+
+const char *priority_annotator(struct log_field *lf) {
+ uint64_t pri = log_field_to_uint64(lf);
+ return nd_log_id2priority(pri);
+}
diff --git a/src/libnetdata/log/nd_log-common.h b/src/libnetdata/log/nd_log-common.h
new file mode 100644
index 0000000000..d06bbbd16e
--- /dev/null
+++ b/src/libnetdata/log/nd_log-common.h
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ND_LOG_COMMON_H
+#define NETDATA_ND_LOG_COMMON_H
+
+#include <syslog.h>
+
+typedef enum __attribute__((__packed__)) {
+ NDLS_UNSET = 0, // internal use only
+ NDLS_ACCESS, // access.log
+ NDLS_ACLK, // aclk.log
+ NDLS_COLLECTORS, // collector.log
+ NDLS_DAEMON, // error.log
+ NDLS_HEALTH, // health.log
+ NDLS_DEBUG, // debug.log
+
+ // terminator
+ _NDLS_MAX,
+} ND_LOG_SOURCES;
+
+typedef enum __attribute__((__packed__)) {
+ NDLP_EMERG = LOG_EMERG, // from syslog.h
+ NDLP_ALERT = LOG_ALERT, // from syslog.h
+ NDLP_CRIT = LOG_CRIT, // from syslog.h
+ NDLP_ERR = LOG_ERR, // from syslog.h
+ NDLP_WARNING = LOG_WARNING, // from syslog.h
+ NDLP_NOTICE = LOG_NOTICE, // from syslog.h
+ NDLP_INFO = LOG_INFO, // from syslog.h
+ NDLP_DEBUG = LOG_DEBUG, // from syslog.h
+
+ // terminator
+ _NDLP_MAX,
+} ND_LOG_FIELD_PRIORITY;
+
+typedef enum __attribute__((__packed__)) {
+ // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
+ // so that it easy to audit for missing fields
+
+ // NEVER RENUMBER THIS LIST
+ // The Windows Events Log has them at fixed positions
+
+ NDF_STOP = 0,
+ NDF_TIMESTAMP_REALTIME_USEC = 1, // the timestamp of the log message - added automatically
+ NDF_SYSLOG_IDENTIFIER = 2, // the syslog identifier of the application - added automatically
+ NDF_LOG_SOURCE = 3, // DAEMON, COLLECTORS, HEALTH, MSGID_ACCESS, ACLK - set at the log call
+ NDF_PRIORITY = 4, // the syslog priority (severity) - set at the log call
+ NDF_ERRNO = 5, // the ERRNO at the time of the log call - added automatically
+ NDF_WINERROR = 6, // Windows GetLastError()
+ NDF_INVOCATION_ID = 7, // the INVOCATION_ID of Netdata - added automatically
+ NDF_LINE = 8, // the source code file line number - added automatically
+ NDF_FILE = 9, // the source code filename - added automatically
+ NDF_FUNC = 10, // the source code function - added automatically
+ NDF_TID = 11, // the thread ID of the thread logging - added automatically
+ NDF_THREAD_TAG = 12, // the thread tag of the thread logging - added automatically
+ NDF_MESSAGE_ID = 13, // for specific events
+ NDF_MODULE = 14, // for internal plugin module, all other get the NDF_THREAD_TAG
+
+ NDF_NIDL_NODE = 15, // the node / rrdhost currently being worked
+ NDF_NIDL_INSTANCE = 16, // the instance / rrdset currently being worked
+ NDF_NIDL_CONTEXT = 17, // the context of the instance currently being worked
+ NDF_NIDL_DIMENSION = 18, // the dimension / rrddim currently being worked
+
+ // web server, aclk and stream receiver
+ NDF_SRC_TRANSPORT = 19, // the transport we received the request, one of: http, https, pluginsd
+
+ // Netdata Cloud Related
+ NDF_ACCOUNT_ID = 20,
+ NDF_USER_NAME = 21,
+ NDF_USER_ROLE = 22,
+ NDF_USER_ACCESS = 23,
+
+ // web server and stream receiver
+ NDF_SRC_IP = 24, // the streaming / web server source IP
+ NDF_SRC_PORT = 25, // the streaming / web server source Port
+ NDF_SRC_FORWARDED_HOST = 26,
+ NDF_SRC_FORWARDED_FOR = 27,
+ NDF_SRC_CAPABILITIES = 28, // the stream receiver capabilities
+
+ // stream sender (established links)
+ NDF_DST_TRANSPORT = 29, // the transport we send the request, one of: http, https
+ NDF_DST_IP = 30, // the destination streaming IP
+ NDF_DST_PORT = 31, // the destination streaming Port
+ NDF_DST_CAPABILITIES = 32, // the destination streaming capabilities
+
+ // web server, aclk and stream receiver
+ NDF_REQUEST_METHOD = 33, // for http like requests, the http request method
+ NDF_RESPONSE_CODE = 34, // for http like requests, the http response code, otherwise a status string
+
+ // web server (all), aclk (queries)
+ NDF_CONNECTION_ID = 35, // the web server connection ID
+ NDF_TRANSACTION_ID = 36, // the web server and API transaction ID
+ NDF_RESPONSE_SENT_BYTES = 37, // for http like requests, the response bytes
+ NDF_RESPONSE_SIZE_BYTES = 38, // for http like requests, the uncompressed response size
+ NDF_RESPONSE_PREPARATION_TIME_USEC = 39, // for http like requests, the preparation time
+ NDF_RESPONSE_SENT_TIME_USEC = 40, // for http like requests, the time to send the response back
+ NDF_RESPONSE_TOTAL_TIME_USEC = 41, // for http like requests, the total time to complete the response
+
+ // health alerts
+ NDF_ALERT_ID = 42,
+ NDF_ALERT_UNIQUE_ID = 43,
+ NDF_ALERT_EVENT_ID = 44,
+ NDF_ALERT_TRANSITION_ID = 45,
+ NDF_ALERT_CONFIG_HASH = 46,
+ NDF_ALERT_NAME = 47,
+ NDF_ALERT_CLASS = 48,
+ NDF_ALERT_COMPONENT = 49,
+ NDF_ALERT_TYPE = 50,
+ NDF_ALERT_EXEC = 51,
+ NDF_ALERT_RECIPIENT = 52,
+ NDF_ALERT_DURATION = 53,
+ NDF_ALERT_VALUE = 54,
+ NDF_ALERT_VALUE_OLD = 55,
+ NDF_ALERT_STATUS = 56,
+ NDF_ALERT_STATUS_OLD = 57,
+ NDF_ALERT_SOURCE = 58,
+ NDF_ALERT_UNITS = 59,
+ NDF_ALERT_SUMMARY = 60,
+ NDF_ALERT_INFO = 61,
+ NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62,
+ // NDF_ALERT_FLAGS,
+
+ // put new items here
+ // leave the request URL and the message last
+
+ NDF_REQUEST = 63, // the request we are currently working on
+ NDF_MESSAGE = 64, // the log message, if any
+
+ // terminator
+ _NDF_MAX,
+} ND_LOG_FIELD_ID;
+
+typedef enum __attribute__((__packed__)) {
+ NDFT_UNSET = 0,
+ NDFT_TXT,
+ NDFT_STR,
+ NDFT_BFR,
+ NDFT_U64,
+ NDFT_I64,
+ NDFT_DBL,
+ NDFT_UUID,
+ NDFT_CALLBACK,
+
+ // terminator
+ _NDFT_MAX,
+} ND_LOG_STACK_FIELD_TYPE;
+
+#endif //NETDATA_ND_LOG_COMMON_H
diff --git a/src/libnetdata/log/nd_log-config.c b/src/libnetdata/log/nd_log-config.c
new file mode 100644
index 0000000000..c8e17402e7
--- /dev/null
+++ b/src/libnetdata/log/nd_log-config.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
+ char buf[FILENAME_MAX + 100];
+ if(setting && *setting)
+ strncpyz(buf, setting, sizeof(buf) - 1);
+ else
+ buf[0] = '\0';
+
+ struct nd_log_source *ls = &nd_log.sources[source];
+ char *output = strrchr(buf, '@');
+
+ if(!output)
+ // all of it is the output
+ output = buf;
+ else {
+ // we found an '@', the next char is the output
+ *output = '\0';
+ output++;
+
+ // parse the other params
+ char *remaining = buf;
+ while(remaining) {
+ char *value = strsep_skip_consecutive_separators(&remaining, ",");
+ if (!value || !*value) continue;
+
+ char *name = strsep_skip_consecutive_separators(&value, "=");
+ if (!name || !*name) continue;
+
+ if(strcmp(name, "logfmt") == 0)
+ ls->format = NDLF_LOGFMT;
+ else if(strcmp(name, "json") == 0)
+ ls->format = NDLF_JSON;
+ else if(strcmp(name, "journal") == 0)
+ ls->format = NDLF_JOURNAL;
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ else if(strcmp(name, ETW_NAME) == 0)
+ ls->format = NDLF_ETW;
+#endif
+#if defined(HAVE_WEL)
+ else if(strcmp(name, WEL_NAME) == 0)
+ ls->format = NDLF_WEL;
+#endif
+#endif
+ else if(strcmp(name, "level") == 0 && value && *value)
+ ls->min_priority = nd_log_priority2id(value);
+ else if(strcmp(name, "protection") == 0 && value && *value) {
+ if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
+ ls->limits = ND_LOG_LIMITS_UNLIMITED;
+ ls->limits.counter = 0;
+ ls->limits.prevented = 0;
+ }
+ else {
+ ls->limits = ND_LOG_LIMITS_DEFAULT;
+
+ char *slash = strchr(value, '/');
+ if(slash) {
+ *slash = '\0';
+ slash++;
+ ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
+
+ int period;
+ if(!duration_parse_seconds(slash, &period)) {
+ nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash);
+ period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
+ }
+
+ ls->limits.throttle_period = period;
+ }
+ else {
+ ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
+ ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
+ }
+ }
+ }
+ else
+ nd_log(NDLS_DAEMON, NDLP_ERR,
+ "Error while parsing configuration of log source '%s'. "
+ "In config '%s', '%s' is not understood.",
+ nd_log_id2source(source), setting, name);
+ }
+ }
+
+ if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
+ ls->method = NDLM_DISABLED;
+ ls->filename = "/dev/null";
+ }
+ else if(strcmp(output, "journal") == 0) {
+ ls->method = NDLM_JOURNAL;
+ ls->filename = NULL;
+ }
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ else if(strcmp(output, ETW_NAME) == 0) {
+ ls->method = NDLM_ETW;
+ ls->filename = NULL;
+ }
+#endif
+#if defined(HAVE_WEL)
+ else if(strcmp(output, WEL_NAME) == 0) {
+ ls->method = NDLM_WEL;
+ ls->filename = NULL;
+ }
+#endif
+#endif
+ else if(strcmp(output, "syslog") == 0) {
+ ls->method = NDLM_SYSLOG;
+ ls->filename = NULL;
+ }
+ else if(strcmp(output, "/dev/null") == 0) {
+ ls->method = NDLM_DEVNULL;
+ ls->filename = "/dev/null";
+ }
+ else if(strcmp(output, "system") == 0) {
+ if(ls->fd == STDERR_FILENO) {
+ ls->method = NDLM_STDERR;
+ ls->filename = NULL;
+ ls->fd = STDERR_FILENO;
+ }
+ else {
+ ls->method = NDLM_STDOUT;
+ ls->filename = NULL;
+ ls->fd = STDOUT_FILENO;
+ }
+ }
+ else if(strcmp(output, "stderr") == 0) {
+ ls->method = NDLM_STDERR;
+ ls->filename = NULL;
+ ls->fd = STDERR_FILENO;
+ }
+ else if(strcmp(output, "stdout") == 0) {
+ ls->method = NDLM_STDOUT;
+ ls->filename = NULL;
+ ls->fd = STDOUT_FILENO;
+ }
+ else {
+ ls->method = NDLM_FILE;
+ ls->filename = strdupz(output);
+ }
+
+#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
+ ls->min_priority = NDLP_DEBUG;
+#endif
+
+ if(source == NDLS_COLLECTORS) {
+ // set the method for the collector processes we will spawn
+
+ ND_LOG_METHOD method = NDLM_STDERR;
+ ND_LOG_FORMAT format = NDLF_LOGFMT;
+ ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
+
+ if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ls->method)) {
+ method = ls->method;
+ format = ls->format;
+ }
+
+ nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
+ nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
+ nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
+ }
+}
+
+void nd_log_set_priority_level(const char *setting) {
+ if(!setting || !*setting)
+ setting = "info";
+
+ ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
+
+#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
+ priority = NDLP_DEBUG;
+#endif
+
+ for (size_t i = 0; i < _NDLS_MAX; i++) {
+ if (i != NDLS_DEBUG)
+ nd_log.sources[i].min_priority = priority;
+ }
+
+ // the right one
+ nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
+}
+
+void nd_log_set_facility(const char *facility) {
+ if(!facility || !*facility)
+ facility = "daemon";
+
+ nd_log.syslog.facility = nd_log_facility2id(facility);
+ nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
+}
+
+void nd_log_set_flood_protection(size_t logs, time_t period) {
+ nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
+ nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
+ nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
+ nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
+
+ nd_log.sources[NDLS_DAEMON].limits.throttle_period =
+ nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
+
+ char buf[100];
+ snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
+ nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
+ snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
+ nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
+}
diff --git a/src/libnetdata/log/nd_log-field-formatters.c b/src/libnetdata/log/nd_log-field-formatters.c
new file mode 100644
index 0000000000..e1b3c0d08d
--- /dev/null
+++ b/src/libnetdata/log/nd_log-field-formatters.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+int64_t log_field_to_int64(struct log_field *lf) {
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ CLEAN_BUFFER *tmp = NULL;
+ const char *s = NULL;
+
+ switch(lf->entry.type) {
+ default:
+ case NDFT_UUID:
+ case NDFT_UNSET:
+ return 0;
+
+ case NDFT_TXT:
+ s = lf->entry.txt;
+ break;
+
+ case NDFT_STR:
+ s = string2str(lf->entry.str);
+ break;
+
+ case NDFT_BFR:
+ s = buffer_tostring(lf->entry.bfr);
+ break;
+
+ case NDFT_CALLBACK:
+ tmp = buffer_create(0, NULL);
+
+ if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
+ s = buffer_tostring(tmp);
+ else
+ s = NULL;
+ break;
+
+ case NDFT_U64:
+ return (int64_t)lf->entry.u64;
+
+ case NDFT_I64:
+ return (int64_t)lf->entry.i64;
+
+ case NDFT_DBL:
+ return (int64_t)lf->entry.dbl;
+ }
+
+ if(s && *s)
+ return str2ll(s, NULL);
+
+ return 0;
+}
+
+uint64_t log_field_to_uint64(struct log_field *lf) {
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ CLEAN_BUFFER *tmp = NULL;
+ const char *s = NULL;
+
+ switch(lf->entry.type) {
+ default:
+ case NDFT_UUID:
+ case NDFT_UNSET:
+ return 0;
+
+ case NDFT_TXT:
+ s = lf->entry.txt;
+ break;
+
+ case NDFT_STR:
+ s = string2str(lf->entry.str);
+ break;
+
+ case NDFT_BFR:
+ s = buffer_tostring(lf->entry.bfr);
+ break;
+
+ case NDFT_CALLBACK:
+ tmp = buffer_create(0, NULL);
+
+ if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
+ s = buffer_tostring(tmp);
+ else
+ s = NULL;
+ break;
+
+ case NDFT_U64:
+ return lf->entry.u64;
+
+ case NDFT_I64:
+ return lf->entry.i64;
+
+ case NDFT_DBL:
+ return (uint64_t) lf->entry.dbl;
+ }
+
+ if(s && *s)
+ return str2uint64_t(s, NULL);
+
+ return 0;
+}
diff --git a/src/libnetdata/log/nd_log-format-json.c b/src/libnetdata/log/nd_log-format-json.c
new file mode 100644
index 0000000000..c25bf19c5d
--- /dev/null
+++ b/src/libnetdata/log/nd_log-format-json.c
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
+ CLEAN_BUFFER *tmp = NULL;
+
+ for (size_t i = 0; i < fields_max; i++) {
+ if (!fields[i].entry.set || !fields[i].logfmt)
+ continue;
+
+ const char *key = fields[i].logfmt;
+
+ const char *s = NULL;
+ switch(fields[i].entry.type) {
+ case NDFT_TXT:
+ s = fields[i].entry.txt;
+ break;
+ case NDFT_STR:
+ s = string2str(fields[i].entry.str);
+ break;
+ case NDFT_BFR:
+ s = buffer_tostring(fields[i].entry.bfr);
+ break;
+ case NDFT_U64:
+ buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
+ break;
+ case NDFT_I64:
+ buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
+ break;
+ case NDFT_DBL:
+ buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
+ break;
+ case NDFT_UUID:
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
+ char u[UUID_COMPACT_STR_LEN];
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
+ buffer_json_member_add_string(wb, key, u);
+ }
+ break;
+ case NDFT_CALLBACK: {
+ if(!tmp)
+ tmp = buffer_create(1024, NULL);
+ else
+ buffer_flush(tmp);
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
+ s = buffer_tostring(tmp);
+ else
+ s = NULL;
+ }
+ break;
+ default:
+ s = "UNHANDLED";
+ break;
+ }
+
+ if(s && *s)
+ buffer_json_member_add_string(wb, key, s);
+ }
+
+ buffer_json_finalize(wb);
+}
diff --git a/src/libnetdata/log/nd_log-format-logfmt.c b/src/libnetdata/log/nd_log-format-logfmt.c
new file mode 100644
index 0000000000..d65211dfcb
--- /dev/null
+++ b/src/libnetdata/log/nd_log-format-logfmt.c
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+static bool needs_quotes_for_logfmt(const char *s)
+{
+ static bool safe_for_logfmt[256] = {
+ [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
+ ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
+ ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
+ ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
+ ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
+ ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
+ ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
+ ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
+ ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
+ ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
+ ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
+ ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
+ ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
+ ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
+ };
+
+ if(!*s)
+ return true;
+
+ while(*s) {
+ if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s])
+ return true;
+
+ s++;
+ }
+
+ return false;
+}
+
+static void string_to_logfmt(BUFFER *wb, const char *s)
+{
+ bool spaces = needs_quotes_for_logfmt(s);
+
+ if(spaces)
+ buffer_fast_strcat(wb, "\"", 1);
+
+ buffer_json_strcat(wb, s);
+
+ if(spaces)
+ buffer_fast_strcat(wb, "\"", 1);
+}
+
+void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) {
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ CLEAN_BUFFER *tmp = NULL;
+
+ for (size_t i = 0; i < fields_max; i++) {
+ if (!fields[i].entry.set || !fields[i].logfmt)
+ continue;
+
+ const char *key = fields[i].logfmt;
+
+ if(fields[i].annotator) {
+ const char *s = fields[i].annotator(&fields[i]);
+ if(!s) continue;
+
+ if(buffer_strlen(wb))
+ buffer_fast_strcat(wb, " ", 1);
+
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ string_to_logfmt(wb, s);
+ }
+ else {
+ if(buffer_strlen(wb))
+ buffer_fast_strcat(wb, " ", 1);
+
+ switch(fields[i].entry.type) {
+ case NDFT_TXT:
+ if(*fields[i].entry.txt) {
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ string_to_logfmt(wb, fields[i].entry.txt);
+ }
+ break;
+ case NDFT_STR:
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ string_to_logfmt(wb, string2str(fields[i].entry.str));
+ break;
+ case NDFT_BFR:
+ if(buffer_strlen(fields[i].entry.bfr)) {
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
+ }
+ break;
+ case NDFT_U64:
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ buffer_print_uint64(wb, fields[i].entry.u64);
+ break;
+ case NDFT_I64:
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ buffer_print_int64(wb, fields[i].entry.i64);
+ break;
+ case NDFT_DBL:
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ buffer_print_netdata_double(wb, fields[i].entry.dbl);
+ break;
+ case NDFT_UUID:
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
+ char u[UUID_COMPACT_STR_LEN];
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
+ }
+ break;
+ case NDFT_CALLBACK: {
+ if(!tmp)
+ tmp = buffer_create(1024, NULL);
+ else
+ buffer_flush(tmp);
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
+ buffer_strcat(wb, key);
+ buffer_fast_strcat(wb, "=", 1);
+ string_to_logfmt(wb, buffer_tostring(tmp));
+ }
+ }
+ break;
+ default:
+ buffer_strcat(wb, "UNHANDLED");
+ break;
+ }
+ }
+ }
+}
diff --git a/src/libnetdata/log/nd_log-init.c b/src/libnetdata/log/nd_log-init.c
new file mode 100644
index 0000000000..f1527b7444
--- /dev/null
+++ b/src/libnetdata/log/nd_log-init.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+// --------------------------------------------------------------------------------------------------------------------
+
+__attribute__((constructor)) void initialize_invocation_id(void) {
+ // check for a NETDATA_INVOCATION_ID
+ if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
+ // not found, check for systemd set INVOCATION_ID
+ if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
+ // not found, generate a new one
+ uuid_generate_random(nd_log.invocation_id);
+ }
+ }
+
+ char uuid[UUID_COMPACT_STR_LEN];
+ uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
+ nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void nd_log_initialize_for_external_plugins(const char *name) {
+ // if we don't run under Netdata, log to stderr,
+ // otherwise, use the logging method Netdata wants us to use.
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ nd_setenv("NETDATA_LOG_METHOD", ETW_NAME, 0);
+ nd_setenv("NETDATA_LOG_FORMAT", ETW_NAME, 0);
+#elif defined(HAVE_WEL)
+ nd_setenv("NETDATA_LOG_METHOD", WEL_NAME, 0);
+ nd_setenv("NETDATA_LOG_FORMAT", WEL_NAME, 0);
+#else
+ nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
+ nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
+#endif
+#else
+ nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
+ nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
+#endif
+
+ nd_log.overwrite_process_source = NDLS_COLLECTORS;
+ program_name = name;
+
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
+ nd_log.sources[i].method = STDERR_FILENO;
+ nd_log.sources[i].fd = -1;
+ nd_log.sources[i].fp = NULL;
+ }
+
+ nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
+ nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
+
+ time_t period = 1200;
+ size_t logs = 200;
+ const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
+ if(s && *s >= '0' && *s <= '9') {
+ period = str2l(s);
+ if(period < 0) period = 0;
+ }
+
+ s = getenv("NETDATA_ERRORS_PER_PERIOD");
+ if(s && *s >= '0' && *s <= '9')
+ logs = str2u(s);
+
+ nd_log_set_flood_protection(logs, period);
+
+ if(!netdata_configured_host_prefix) {
+ s = getenv("NETDATA_HOST_PREFIX");
+ if(s && *s)
+ netdata_configured_host_prefix = (char *)s;
+ }
+
+ ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
+ ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
+
+ if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
+ if(is_stderr_connected_to_journal()) {
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
+ method = NDLM_JOURNAL;
+ }
+ else {
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
+ method = NDLM_STDERR;
+ }
+ }
+
+ switch(method) {
+ case NDLM_JOURNAL:
+ if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
+ !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
+ method = NDLM_STDERR;
+ }
+ break;
+
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ case NDLM_ETW:
+ if(!nd_log_init_etw()) {
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Events Tracing for Windows (ETW). Using stderr.");
+ method = NDLM_STDERR;
+ }
+ break;
+#endif
+#if defined(HAVE_WEL)
+ case NDLM_WEL:
+ if(!nd_log_init_wel()) {
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Windows Event Log (WEL). Using stderr.");
+ method = NDLM_STDERR;
+ }
+ break;
+#endif
+#endif
+
+ case NDLM_SYSLOG:
+ nd_log_init_syslog();
+ break;
+
+ default:
+ method = NDLM_STDERR;
+ break;
+ }
+
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
+ nd_log.sources[i].method = method;
+ nd_log.sources[i].format = format;
+ nd_log.sources[i].fd = -1;
+ nd_log.sources[i].fp = NULL;
+ }
+
+ // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
+ if(e->method == NDLM_DEFAULT)
+ nd_log_set_user_settings(source, e->filename);
+
+ if((e->method == NDLM_FILE && !e->filename) ||
+ (e->method == NDLM_DEVNULL && e->fd == -1))
+ e->method = NDLM_DISABLED;
+
+ if(e->fp)
+ fflush(e->fp);
+
+ switch(e->method) {
+ case NDLM_SYSLOG:
+ nd_log_init_syslog();
+ break;
+
+ case NDLM_JOURNAL:
+ nd_log_journal_direct_init(NULL);
+ nd_log_journal_systemd_init();
+ break;
+
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ case NDLM_ETW:
+ nd_log_init_etw();
+ break;
+#endif
+#if defined(HAVE_WEL)
+ case NDLM_WEL:
+ nd_log_init_wel();
+ break;
+#endif
+#endif
+
+ case NDLM_STDOUT:
+ e->fp = stdout;
+ e->fd = STDOUT_FILENO;
+ break;
+
+ case NDLM_DISABLED:
+ break;
+
+ case NDLM_DEFAULT:
+ case NDLM_STDERR:
+ e->method = NDLM_STDERR;
+ e->fp = stderr;
+ e->fd = STDERR_FILENO;
+ break;
+
+ case NDLM_DEVNULL:
+ case NDLM_FILE: {
+ int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
+ if(fd == -1) {
+ if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
+ e->fd = STDERR_FILENO;
+ e->method = NDLM_STDERR;
+ netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
+ }
+ else
+ netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
+ }
+ else {
+ if (!nd_log_replace_existing_fd(e, fd)) {
+ if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
+ if(e->fd == STDOUT_FILENO)
+ e->method = NDLM_STDOUT;
+ else if(e->fd == STDERR_FILENO)
+ e->method = NDLM_STDERR;
+
+ // we have dup2() fd, so we can close the one we opened
+ if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
+ close(fd);
+ }
+ else
+ e->fd = fd;
+ }
+ }
+
+ // at this point we have e->fd set properly
+
+ if(e->fd == STDOUT_FILENO)
+ e->fp = stdout;
+ else if(e->fd == STDERR_FILENO)
+ e->fp = stderr;
+
+ if(!e->fp) {
+ e->fp = fdopen(e->fd, "a");
+ if (!e->fp) {
+ netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
+
+ if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
+ close(e->fd);
+
+ e->fp = stderr;
+ e->fd = STDERR_FILENO;
+ }
+ }
+ else {
+ if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
+ netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
+ }
+ }
+ break;
+ }
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void nd_log_stdin_init(int fd, const char *filename) {
+ int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
+ if(f == -1)
+ return;
+
+ if(f != fd) {
+ dup2(f, fd);
+ close(f);
+ }
+}
+
+void nd_log_initialize(void) {
+ nd_log_stdin_init(STDIN_FILENO, "/dev/null");
+
+ for(size_t i = 0 ; i < _NDLS_MAX ; i++)
+ nd_log_open(&nd_log.sources[i], i);
+}
+
+void nd_log_reopen_log_files(bool log) {
+ if(log)
+ netdata_log_info("Reopening all log files.");
+
+ nd_log.std_output.initialized = false;
+ nd_log.std_error.initialized = false;
+ nd_log.journal_direct.initialized = false;
+ nd_log.journal.initialized = false;
+ nd_log_initialize();
+
+ if(log)
+ netdata_log_info("Log files re-opened.");
+}
+
+void nd_log_reopen_log_files_for_spawn_server(void) {
+ gettid_uncached();
+
+ if(nd_log.syslog.initialized) {
+ closelog();
+ nd_log.syslog.initialized = false;
+ nd_log_init_syslog();
+ }
+
+ if(nd_log.journal_direct.initialized) {
+ close(nd_log.journal_direct.fd);
+ nd_log.journal_direct.fd = -1;
+ nd_log.journal_direct.initialized = false;
+ nd_log_journal_direct_init(NULL);
+ }
+
+ nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
+ nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
+ nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
+ nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
+ nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
+ nd_log_reopen_log_files(false);
+}
+
diff --git a/src/libnetdata/log/nd_log-internals.c b/src/libnetdata/log/nd_log-internals.c
new file mode 100644
index 0000000000..cb26b816e9
--- /dev/null
+++ b/src/libnetdata/log/nd_log-internals.c
@@ -0,0 +1,821 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+// --------------------------------------------------------------------------------------------------------------------
+// workaround strerror_r()
+
+#if defined(STRERROR_R_CHAR_P)
+// GLIBC version of strerror_r
+static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
+#elif defined(HAVE_STRERROR_R)
+// POSIX version of strerror_r
+static const char *strerror_result(int a, const char *b) { (void)a; return b; }
+#elif defined(HAVE_C__GENERIC)
+
+// what a trick!
+// http://stackoverflow.com/questions/479207/function-overloading-in-c
+static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
+static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
+
+#define strerror_result(a, b) _Generic((a), \
+ int: strerror_result_int, \
+ char *: strerror_result_string \
+ )(a, b)
+
+#else
+#error "cannot detect the format of function strerror_r()"
+#endif
+
+const char *errno2str(int errnum, char *buf, size_t size) {
+ return strerror_result(strerror_r(errnum, buf, size), buf);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// logging method
+
+static struct {
+ ND_LOG_METHOD method;
+ const char *name;
+} nd_log_methods[] = {
+ { .method = NDLM_DISABLED, .name = "none" },
+ { .method = NDLM_DEVNULL, .name = "/dev/null" },
+ { .method = NDLM_DEFAULT, .name = "default" },
+ { .method = NDLM_JOURNAL, .name = "journal" },
+ { .method = NDLM_SYSLOG, .name = "syslog" },
+ { .method = NDLM_STDOUT, .name = "stdout" },
+ { .method = NDLM_STDERR, .name = "stderr" },
+ { .method = NDLM_FILE, .name = "file" },
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ { .method = NDLM_ETW, .name = ETW_NAME },
+#endif
+#if defined(HAVE_WEL)
+ { .method = NDLM_WEL, .name = WEL_NAME },
+#endif
+#endif
+};
+
+ND_LOG_METHOD nd_log_method2id(const char *method) {
+ if(!method || !*method)
+ return NDLM_DEFAULT;
+
+ size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(strcmp(nd_log_methods[i].name, method) == 0)
+ return nd_log_methods[i].method;
+ }
+
+ return NDLM_FILE;
+}
+
+const char *nd_log_id2method(ND_LOG_METHOD method) {
+ size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(method == nd_log_methods[i].method)
+ return nd_log_methods[i].name;
+ }
+
+ return "unknown";
+}
+
+const char *nd_log_method_for_external_plugins(const char *s) {
+ if(s && *s) {
+ ND_LOG_METHOD method = nd_log_method2id(s);
+ if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
+ return nd_log_id2method(method);
+ }
+
+ return nd_log_id2method(NDLM_STDERR);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// facilities
+//
+// sys/syslog.h (Linux)
+// sys/sys/syslog.h (FreeBSD)
+// bsd/sys/syslog.h (darwin-xnu)
+
+static struct {
+ int facility;
+ const char *name;
+} nd_log_facilities[] = {
+ { LOG_AUTH, "auth" },
+ { LOG_AUTHPRIV, "authpriv" },
+ { LOG_CRON, "cron" },
+ { LOG_DAEMON, "daemon" },
+ { LOG_FTP, "ftp" },
+ { LOG_KERN, "kern" },
+ { LOG_LPR, "lpr" },
+ { LOG_MAIL, "mail" },
+ { LOG_NEWS, "news" },
+ { LOG_SYSLOG, "syslog" },
+ { LOG_USER, "user" },
+ { LOG_UUCP, "uucp" },
+ { LOG_LOCAL0, "local0" },
+ { LOG_LOCAL1, "local1" },
+ { LOG_LOCAL2, "local2" },
+ { LOG_LOCAL3, "local3" },
+ { LOG_LOCAL4, "local4" },
+ { LOG_LOCAL5, "local5" },
+ { LOG_LOCAL6, "local6" },
+ { LOG_LOCAL7, "local7" },
+
+#ifdef __FreeBSD__
+ { LOG_CONSOLE, "console" },
+ { LOG_NTP, "ntp" },
+
+ // FreeBSD does not consider 'security' as deprecated.
+ { LOG_SECURITY, "security" },
+#else
+ // For all other O/S 'security' is mapped to 'auth'.
+ { LOG_AUTH, "security" },
+#endif
+
+#ifdef __APPLE__
+ { LOG_INSTALL, "install" },
+ { LOG_NETINFO, "netinfo" },
+ { LOG_RAS, "ras" },
+ { LOG_REMOTEAUTH, "remoteauth" },
+ { LOG_LAUNCHD, "launchd" },
+
+#endif
+};
+
+int nd_log_facility2id(const char *facility) {
+ size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(strcmp(nd_log_facilities[i].name, facility) == 0)
+ return nd_log_facilities[i].facility;
+ }
+
+ return LOG_DAEMON;
+}
+
+const char *nd_log_id2facility(int facility) {
+ size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(nd_log_facilities[i].facility == facility)
+ return nd_log_facilities[i].name;
+ }
+
+ return "daemon";
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// priorities
+
+static struct {
+ ND_LOG_FIELD_PRIORITY priority;
+ const char *name;
+} nd_log_priorities[] = {
+ { .priority = NDLP_EMERG, .name = "emergency" },
+ { .priority = NDLP_EMERG, .name = "emerg" },
+ { .priority = NDLP_ALERT, .name = "alert" },
+ { .priority = NDLP_CRIT, .name = "critical" },
+ { .priority = NDLP_CRIT, .name = "crit" },
+ { .priority = NDLP_ERR, .name = "error" },
+ { .priority = NDLP_ERR, .name = "err" },
+ { .priority = NDLP_WARNING, .name = "warning" },
+ { .priority = NDLP_WARNING, .name = "warn" },
+ { .priority = NDLP_NOTICE, .name = "notice" },
+ { .priority = NDLP_INFO, .name = NDLP_INFO_STR },
+ { .priority = NDLP_DEBUG, .name = "debug" },
+};
+
+int nd_log_priority2id(const char *priority) {
+ size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(strcmp(nd_log_priorities[i].name, priority) == 0)
+ return nd_log_priorities[i].priority;
+ }
+
+ return NDLP_INFO;
+}
+
+const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
+ size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(priority == nd_log_priorities[i].priority)
+ return nd_log_priorities[i].name;
+ }
+
+ return NDLP_INFO_STR;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// log sources
+
+const char *nd_log_sources[] = {
+ [NDLS_UNSET] = "UNSET",
+ [NDLS_ACCESS] = "access",
+ [NDLS_ACLK] = "aclk",
+ [NDLS_COLLECTORS] = "collector",
+ [NDLS_DAEMON] = "daemon",
+ [NDLS_HEALTH] = "health",
+ [NDLS_DEBUG] = "debug",
+};
+
+size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
+ size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(strcmp(nd_log_sources[i], source) == 0)
+ return i;
+ }
+
+ return def;
+}
+
+
+const char *nd_log_id2source(ND_LOG_SOURCES source) {
+ size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
+ if(source < entries)
+ return nd_log_sources[source];
+
+ return nd_log_sources[NDLS_COLLECTORS];
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// log output formats
+
+static struct {
+ ND_LOG_FORMAT format;
+ const char *name;
+} nd_log_formats[] = {
+ { .format = NDLF_JOURNAL, .name = "journal" },
+ { .format = NDLF_LOGFMT, .name = "logfmt" },
+ { .format = NDLF_JSON, .name = "json" },
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ { .format = NDLF_ETW, .name = ETW_NAME },
+#endif
+#if defined(HAVE_WEL)
+ { .format = NDLF_WEL, .name = WEL_NAME },
+#endif
+#endif
+};
+
+ND_LOG_FORMAT nd_log_format2id(const char *format) {
+ if(!format || !*format)
+ return NDLF_LOGFMT;
+
+ size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(strcmp(nd_log_formats[i].name, format) == 0)
+ return nd_log_formats[i].format;
+ }
+
+ return NDLF_LOGFMT;
+}
+
+const char *nd_log_id2format(ND_LOG_FORMAT format) {
+ size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
+ for(size_t i = 0; i < entries ;i++) {
+ if(format == nd_log_formats[i].format)
+ return nd_log_formats[i].name;
+ }
+
+ return "logfmt";
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+struct nd_log nd_log = {
+ .overwrite_process_source = 0,
+ .journal = {
+ .initialized = false,
+ },
+ .journal_direct = {
+ .initialized = false,
+ .fd = -1,
+ },
+ .syslog = {
+ .initialized = false,
+ .facility = LOG_DAEMON,
+ },
+#if defined(OS_WINDOWS)
+ .eventlog = {
+ .initialized = false,
+ },
+#endif
+ .std_output = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .initialized = false,
+ },
+ .std_error = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .initialized = false,
+ },
+ .sources = {
+ [NDLS_UNSET] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DISABLED,
+ .format = NDLF_JOURNAL,
+ .filename = NULL,
+ .fd = -1,
+ .fp = NULL,
+ .min_priority = NDLP_EMERG,
+ .limits = ND_LOG_LIMITS_UNLIMITED,
+ },
+ [NDLS_ACCESS] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DEFAULT,
+ .format = NDLF_LOGFMT,
+ .filename = LOG_DIR "/access.log",
+ .fd = -1,
+ .fp = NULL,
+ .min_priority = NDLP_DEBUG,
+ .limits = ND_LOG_LIMITS_UNLIMITED,
+ },
+ [NDLS_ACLK] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_FILE,
+ .format = NDLF_LOGFMT,
+ .filename = LOG_DIR "/aclk.log",
+ .fd = -1,
+ .fp = NULL,
+ .min_priority = NDLP_DEBUG,
+ .limits = ND_LOG_LIMITS_UNLIMITED,
+ },
+ [NDLS_COLLECTORS] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DEFAULT,
+ .format = NDLF_LOGFMT,
+ .filename = LOG_DIR "/collector.log",
+ .fd = STDERR_FILENO,
+ .fp = NULL,
+ .min_priority = NDLP_INFO,
+ .limits = ND_LOG_LIMITS_DEFAULT,
+ },
+ [NDLS_DEBUG] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DISABLED,
+ .format = NDLF_LOGFMT,
+ .filename = LOG_DIR "/debug.log",
+ .fd = STDOUT_FILENO,
+ .fp = NULL,
+ .min_priority = NDLP_DEBUG,
+ .limits = ND_LOG_LIMITS_UNLIMITED,
+ },
+ [NDLS_DAEMON] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DEFAULT,
+ .filename = LOG_DIR "/daemon.log",
+ .format = NDLF_LOGFMT,
+ .fd = -1,
+ .fp = NULL,
+ .min_priority = NDLP_INFO,
+ .limits = ND_LOG_LIMITS_DEFAULT,
+ },
+ [NDLS_HEALTH] = {
+ .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+ .method = NDLM_DEFAULT,
+ .format = NDLF_LOGFMT,
+ .filename = LOG_DIR "/health.log",
+ .fd = -1,
+ .fp = NULL,
+ .min_priority = NDLP_DEBUG,
+ .limits = ND_LOG_LIMITS_UNLIMITED,
+ },
+ },
+};
+
+// --------------------------------------------------------------------------------------------------------------------
+
+__thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
+__thread size_t thread_log_stack_next = 0;
+__thread struct log_field thread_log_fields[_NDF_MAX] = {
+ // THE ORDER HERE IS IRRELEVANT (but keep them sorted by their number)
+
+ [NDF_STOP] = { // processing will not stop on this - so it is ok to be first
+ .journal = NULL,
+ .logfmt = NULL,
+ .eventlog = NULL,
+ .annotator = NULL,
+ },
+ [NDF_TIMESTAMP_REALTIME_USEC] = {
+ .journal = NULL,
+ .eventlog = "Timestamp",
+ .logfmt = "time",
+ .annotator = timestamp_usec_annotator,
+ },
+ [NDF_SYSLOG_IDENTIFIER] = {
+ .journal = "SYSLOG_IDENTIFIER", // standard journald field
+ .eventlog = "Program",
+ .logfmt = "comm",
+ },
+ [NDF_LOG_SOURCE] = {
+ .journal = "ND_LOG_SOURCE",
+ .eventlog = "NetdataLogSource",
+ .logfmt = "source",
+ },
+ [NDF_PRIORITY] = {
+ .journal = "PRIORITY", // standard journald field
+ .eventlog = "Level",
+ .logfmt = "level",
+ .annotator = priority_annotator,
+ },
+ [NDF_ERRNO] = {
+ .journal = "ERRNO", // standard journald field
+ .eventlog = "UnixErrno",
+ .logfmt = "errno",
+ .annotator = errno_annotator,
+ },
+ [NDF_WINERROR] = {
+#if defined(OS_WINDOWS)
+ .journal = "WINERROR",
+ .eventlog = "WindowsLastError",
+ .logfmt = "winerror",
+ .annotator = winerror_annotator,
+#endif
+ },
+ [NDF_INVOCATION_ID] = {
+ .journal = "INVOCATION_ID", // standard journald field
+ .eventlog = "InvocationID",
+ .logfmt = NULL,
+ },
+ [NDF_LINE] = {
+ .journal = "CODE_LINE", // standard journald field
+ .eventlog = "CodeLine",
+ .logfmt = NULL,
+ },
+ [NDF_FILE] = {
+ .journal = "CODE_FILE", // standard journald field
+ .eventlog = "CodeFile",
+ .logfmt = NULL,
+ },
+ [NDF_FUNC] = {
+ .journal = "CODE_FUNC", // standard journald field
+ .eventlog = "CodeFunction",
+ .logfmt = NULL,
+ },
+ [NDF_TID] = {
+ .journal = "TID", // standard journald field
+ .eventlog = "ThreadID",
+ .logfmt = "tid",
+ },
+ [NDF_THREAD_TAG] = {
+ .journal = "THREAD_TAG",
+ .eventlog = "ThreadName",
+ .logfmt = "thread",
+ },
+ [NDF_MESSAGE_ID] = {
+ .journal = "MESSAGE_ID",
+ .eventlog = "MessageID",
+ .logfmt = "msg_id",
+ },
+ [NDF_MODULE] = {
+ .journal = "ND_MODULE",
+ .eventlog = "Module",
+ .logfmt = "module",
+ },
+ [NDF_NIDL_NODE] = {
+ .journal = "ND_NIDL_NODE",
+ .eventlog = "Node",
+ .logfmt = "node",
+ },
+ [NDF_NIDL_INSTANCE] = {
+ .journal = "ND_NIDL_INSTANCE",
+ .eventlog = "Instance",
+ .logfmt = "instance",
+ },
+ [NDF_NIDL_CONTEXT] = {
+ .journal = "ND_NIDL_CONTEXT",
+ .eventlog = "Context",
+ .logfmt = "context",
+ },
+ [NDF_NIDL_DIMENSION] = {
+ .journal = "ND_NIDL_DIMENSION",
+ .eventlog = "Dimension",
+ .logfmt = "dimension",
+ },
+ [NDF_SRC_TRANSPORT] = {
+ .journal = "ND_SRC_TRANSPORT",
+ .eventlog = "SourceTransport",
+ .logfmt = "src_transport",
+ },
+ [NDF_ACCOUNT_ID] = {
+ .journal = "ND_ACCOUNT_ID",
+ .eventlog = "AccountID",
+ .logfmt = "account",
+ },
+ [NDF_USER_NAME] = {
+ .journal = "ND_USER_NAME",
+ .eventlog = "UserName",
+ .logfmt = "user",
+ },
+ [NDF_USER_ROLE] = {
+ .journal = "ND_USER_ROLE",
+ .eventlog = "UserRole",
+ .logfmt = "role",
+ },
+ [NDF_USER_ACCESS] = {
+ .journal = "ND_USER_PERMISSIONS",
+ .eventlog = "UserPermissions",
+ .logfmt = "permissions",
+ },
+ [NDF_SRC_IP] = {
+ .journal = "ND_SRC_IP",
+ .eventlog = "SourceIP",
+ .logfmt = "src_ip",
+ },
+ [NDF_SRC_FORWARDED_HOST] = {
+ .journal = "ND_SRC_FORWARDED_HOST",
+ .eventlog = "SourceForwardedHost",
+ .logfmt = "src_forwarded_host",
+ },
+ [NDF_SRC_FORWARDED_FOR] = {
+ .journal = "ND_SRC_FORWARDED_FOR",
+ .eventlog = "SourceForwardedFor",
+ .logfmt = "src_forwarded_for",
+ },
+ [NDF_SRC_PORT] = {
+ .journal = "ND_SRC_PORT",
+ .eventlog = "SourcePort",
+ .logfmt = "src_port",
+ },
+ [NDF_SRC_CAPABILITIES] = {
+ .journal = "ND_SRC_CAPABILITIES",
+ .eventlog = "SourceCapabilities",
+ .logfmt = "src_capabilities",
+ },
+ [NDF_DST_TRANSPORT] = {
+ .journal = "ND_DST_TRANSPORT",
+ .eventlog = "DestinationTransport",
+ .logfmt = "dst_transport",
+ },
+ [NDF_DST_IP] = {
+ .journal = "ND_DST_IP",
+ .eventlog = "DestinationIP",
+ .logfmt = "dst_ip",
+ },
+ [NDF_DST_PORT] = {
+ .journal = "ND_DST_PORT",
+ .eventlog = "DestinationPort",
+ .logfmt = "dst_port",
+ },
+ [NDF_DST_CAPABILITIES] = {
+ .journal = "ND_DST_CAPABILITIES",
+ .eventlog = "DestinationCapabilities",
+ .logfmt = "dst_capabilities",
+ },
+ [NDF_REQUEST_METHOD] = {
+ .journal = "ND_REQUEST_METHOD",
+ .eventlog = "RequestMethod",
+ .logfmt = "req_method",
+ },
+ [NDF_RESPONSE_CODE] = {
+ .journal = "ND_RESPONSE_CODE",
+ .eventlog = "ResponseCode",
+ .logfmt = "code",
+ },
+ [NDF_CONNECTION_ID] = {
+ .journal = "ND_CONNECTION_ID",
+ .eventlog = "ConnectionID",
+ .logfmt = "conn",
+ },
+ [NDF_TRANSACTION_ID] = {
+ .journal = "ND_TRANSACTION_ID",
+ .eventlog = "TransactionID",
+ .logfmt = "transaction",
+ },
+ [NDF_RESPONSE_SENT_BYTES] = {
+ .journal = "ND_RESPONSE_SENT_BYTES",
+ .eventlog = "ResponseSentBytes",
+ .logfmt = "sent_bytes",
+ },
+ [NDF_RESPONSE_SIZE_BYTES] = {
+ .journal = "ND_RESPONSE_SIZE_BYTES",
+ .eventlog = "ResponseSizeBytes",
+ .logfmt = "size_bytes",
+ },
+ [NDF_RESPONSE_PREPARATION_TIME_USEC] = {
+ .journal = "ND_RESPONSE_PREP_TIME_USEC",
+ .eventlog = "ResponsePreparationTimeUsec",
+ .logfmt = "prep_ut",
+ },
+ [NDF_RESPONSE_SENT_TIME_USEC] = {
+ .journal = "ND_RESPONSE_SENT_TIME_USEC",
+ .eventlog = "ResponseSentTimeUsec",
+ .logfmt = "sent_ut",
+ },
+ [NDF_RESPONSE_TOTAL_TIME_USEC] = {
+ .journal = "ND_RESPONSE_TOTAL_TIME_USEC",
+ .eventlog = "ResponseTotalTimeUsec",
+ .logfmt = "total_ut",
+ },
+ [NDF_ALERT_ID] = {
+ .journal = "ND_ALERT_ID",
+ .eventlog = "AlertID",
+ .logfmt = "alert_id",
+ },
+ [NDF_ALERT_UNIQUE_ID] = {
+ .journal = "ND_ALERT_UNIQUE_ID",
+ .eventlog = "AlertUniqueID",
+ .logfmt = "alert_unique_id",
+ },
+ [NDF_ALERT_TRANSITION_ID] = {
+ .journal = "ND_ALERT_TRANSITION_ID",
+ .eventlog = "AlertTransitionID",
+ .logfmt = "alert_transition_id",
+ },
+ [NDF_ALERT_EVENT_ID] = {
+ .journal = "ND_ALERT_EVENT_ID",
+ .eventlog = "AlertEventID",
+ .logfmt = "alert_event_id",
+ },
+ [NDF_ALERT_CONFIG_HASH] = {
+ .journal = "ND_ALERT_CONFIG",
+ .eventlog = "AlertConfig",
+ .logfmt = "alert_config",
+ },
+ [NDF_ALERT_NAME] = {
+ .journal = "ND_ALERT_NAME",
+ .eventlog = "AlertName",
+ .logfmt = "alert",
+ },
+ [NDF_ALERT_CLASS] = {
+ .journal = "ND_ALERT_CLASS",
+ .eventlog = "AlertClass",
+ .logfmt = "alert_class",
+ },
+ [NDF_ALERT_COMPONENT] = {
+ .journal = "ND_ALERT_COMPONENT",
+ .eventlog = "AlertComponent",
+ .logfmt = "alert_component",
+ },
+ [NDF_ALERT_TYPE] = {
+ .journal = "ND_ALERT_TYPE",
+ .eventlog = "AlertType",
+ .logfmt = "alert_type",
+ },
+ [NDF_ALERT_EXEC] = {
+ .journal = "ND_ALERT_EXEC",
+ .eventlog = "AlertExec",
+ .logfmt = "alert_exec",
+ },
+ [NDF_ALERT_RECIPIENT] = {
+ .journal = "ND_ALERT_RECIPIENT",
+ .eventlog = "AlertRecipient",
+ .logfmt = "alert_recipient",
+ },
+ [NDF_ALERT_VALUE] = {
+ .journal = "ND_ALERT_VALUE",
+ .eventlog = "AlertValue",
+ .logfmt = "alert_value",
+ },
+ [NDF_ALERT_VALUE_OLD] = {
+ .journal = "ND_ALERT_VALUE_OLD",
+ .eventlog = "AlertOldValue",
+ .logfmt = "alert_value_old",
+ },
+ [NDF_ALERT_STATUS] = {
+ .journal = "ND_ALERT_STATUS",
+ .eventlog = "AlertStatus",
+ .logfmt = "alert_status",
+ },
+ [NDF_ALERT_STATUS_OLD] = {
+ .journal = "ND_ALERT_STATUS_OLD",
+ .eventlog = "AlertOldStatus",
+ .logfmt = "alert_value_old",
+ },
+ [NDF_ALERT_UNITS] = {
+ .journal = "ND_ALERT_UNITS",
+ .eventlog = "AlertUnits",
+ .logfmt = "alert_units",
+ },
+ [NDF_ALERT_SUMMARY] = {
+ .journal = "ND_ALERT_SUMMARY",
+ .eventlog = "AlertSummary",
+ .logfmt = "alert_summary",
+ },
+ [NDF_ALERT_INFO] = {
+ .journal = "ND_ALERT_INFO",
+ .eventlog = "AlertInfo",
+ .logfmt = "alert_info",
+ },
+ [NDF_ALERT_DURATION] = {
+ .journal = "ND_ALERT_DURATION",
+ .eventlog = "AlertDuration",
+ .logfmt = "alert_duration",
+ },
+ [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
+ .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
+ .eventlog = "AlertNotificationTime",
+ .logfmt = "alert_notification_timestamp",
+ .annotator = timestamp_usec_annotator,
+ },
+
+ // put new items here
+ // leave the request URL and the message last
+
+ [NDF_REQUEST] = {
+ .journal = "ND_REQUEST",
+ .eventlog = "Request",
+ .logfmt = "request",
+ },
+ [NDF_MESSAGE] = {
+ .journal = "MESSAGE",
+ .eventlog = "Message",
+ .logfmt = "msg",
+ },
+};
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void log_stack_pop(void *ptr) {
+ if(!ptr) return;
+
+ struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
+
+ if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
+ fatal("You cannot pop in the middle of the stack, or an item not in the stack");
+ return;
+ }
+
+ thread_log_stack_next--;
+}
+
+void log_stack_push(struct log_stack_entry *lgs) {
+ if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
+ thread_log_stack_base[thread_log_stack_next++] = lgs;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len) {
+ for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
+ if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
+ return i;
+ }
+
+ return NDF_STOP;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+int nd_log_health_fd(void) {
+ if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
+ return nd_log.sources[NDLS_HEALTH].fd;
+
+ return STDERR_FILENO;
+}
+
+int nd_log_collectors_fd(void) {
+ if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
+ return nd_log.sources[NDLS_COLLECTORS].fd;
+
+ return STDERR_FILENO;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void log_date(char *buffer, size_t len, time_t now) {
+ if(unlikely(!buffer || !len))
+ return;
+
+ time_t t = now;
+ struct tm *tmp, tmbuf;
+
+ tmp = localtime_r(&t, &tmbuf);
+
+ if (unlikely(!tmp)) {
+ buffer[0] = '\0';
+ return;
+ }
+
+ if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
+ buffer[0] = '\0';
+
+ buffer[len - 1] = '\0';
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
+ if(new_fd == -1 || e->fd == -1 ||
+ (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
+ (e->fd == STDERR_FILENO && nd_log.std_error.initialized))
+ return false;
+
+ if(new_fd != e->fd) {
+ int t = dup2(new_fd, e->fd);
+
+ bool ret = true;
+ if (t == -1) {
+ netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
+ ret = false;
+ }
+ else
+ close(new_fd);
+
+ if(e->fd == STDOUT_FILENO)
+ nd_log.std_output.initialized = true;
+ else if(e->fd == STDERR_FILENO)
+ nd_log.std_error.initialized = true;
+
+ return ret;
+ }
+
+ return false;
+}
diff --git a/src/libnetdata/log/nd_log-internals.h b/src/libnetdata/log/nd_log-internals.h
new file mode 100644
index 0000000000..03ac40405f
--- /dev/null
+++ b/src/libnetdata/log/nd_log-internals.h
@@ -0,0 +1,251 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ND_LOG_INTERNALS_H
+#define NETDATA_ND_LOG_INTERNALS_H
+
+#include "../libnetdata.h"
+
+#if defined(OS_WINDOWS)
+#include <windows.h>
+#endif
+
+#ifdef __FreeBSD__
+#include <sys/endian.h>
+#endif
+
+#ifdef __APPLE__
+#include <machine/endian.h>
+#endif
+
+#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
+#include <execinfo.h>
+#endif
+
+#ifdef HAVE_SYSTEMD
+#include <systemd/sd-journal.h>
+#endif
+
+const char *errno2str(int errnum, char *buf, size_t size);
+
+// --------------------------------------------------------------------------------------------------------------------
+// ND_LOG_METHOD
+
+typedef enum __attribute__((__packed__)) {
+ NDLM_DISABLED = 0,
+ NDLM_DEVNULL,
+ NDLM_DEFAULT,
+ NDLM_JOURNAL,
+ NDLM_SYSLOG,
+ NDLM_STDOUT,
+ NDLM_STDERR,
+ NDLM_FILE,
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ NDLM_ETW,
+#endif
+#if defined(HAVE_WEL)
+ NDLM_WEL,
+#endif
+#endif
+} ND_LOG_METHOD;
+
+// all the log methods are finally mapped to these
+#if defined(HAVE_ETW)
+#define ETW_CONDITION(ndlo) ((ndlo) == NDLM_ETW)
+#else
+#define ETW_CONDITION(ndlo) (false)
+#endif
+
+#if defined(HAVE_WEL)
+#define WEL_CONDITION(ndlo) ((ndlo) == NDLM_WEL)
+#else
+#define WEL_CONDITION(ndlo) (false)
+#endif
+
+#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
+#define IS_FINAL_LOG_METHOD(ndlo) ((ndlo) == NDLM_FILE || (ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
+
+ND_LOG_METHOD nd_log_method2id(const char *method);
+const char *nd_log_id2method(ND_LOG_METHOD method);
+
+// --------------------------------------------------------------------------------------------------------------------
+// ND_LOG_FORMAT
+
+typedef enum __attribute__((__packed__)) {
+ NDLF_JOURNAL,
+ NDLF_LOGFMT,
+ NDLF_JSON,
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ NDLF_ETW, // Event Tracing for Windows
+#endif
+#if defined(HAVE_WEL)
+ NDLF_WEL, // Windows Event Log
+#endif
+#endif
+} ND_LOG_FORMAT;
+
+#define ETW_NAME "etw"
+#define WEL_NAME "wel"
+
+const char *nd_log_id2format(ND_LOG_FORMAT format);
+ND_LOG_FORMAT nd_log_format2id(const char *format);
+
+size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def);
+const char *nd_log_id2source(ND_LOG_SOURCES source);
+
+const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
+int nd_log_priority2id(const char *priority);
+
+const char *nd_log_id2facility(int facility);
+int nd_log_facility2id(const char *facility);
+
+#include "nd_log_limit.h"
+
+struct nd_log_source {
+ SPINLOCK spinlock;
+ ND_LOG_METHOD method;
+ ND_LOG_FORMAT format;
+ const char *filename;
+ int fd;
+ FILE *fp;
+
+ ND_LOG_FIELD_PRIORITY min_priority;
+ const char *pending_msg;
+ struct nd_log_limit limits;
+
+#if defined(OS_WINDOWS)
+ ND_LOG_SOURCES source;
+ HANDLE hEventLog;
+ USHORT channelID;
+ UCHAR Opcode;
+ USHORT Task;
+ ULONGLONG Keyword;
+#endif
+};
+
+struct nd_log {
+ nd_uuid_t invocation_id;
+
+ ND_LOG_SOURCES overwrite_process_source;
+
+ struct nd_log_source sources[_NDLS_MAX];
+
+ struct {
+ bool initialized;
+ } journal;
+
+ struct {
+ bool initialized;
+ int fd;
+ char filename[FILENAME_MAX + 1];
+ } journal_direct;
+
+ struct {
+ bool initialized;
+ int facility;
+ } syslog;
+
+ struct {
+ bool etw; // when set use etw, otherwise wel
+ bool initialized;
+ } eventlog;
+
+ struct {
+ SPINLOCK spinlock;
+ bool initialized;
+ } std_output;
+
+ struct {
+ SPINLOCK spinlock;
+ bool initialized;
+ } std_error;
+
+};
+
+// --------------------------------------------------------------------------------------------------------------------
+
+struct log_field;
+typedef const char *(*annotator_t)(struct log_field *lf);
+
+struct log_field {
+ const char *journal;
+ const char *logfmt;
+ const char *eventlog;
+ annotator_t annotator;
+ struct log_stack_entry entry;
+};
+
+#define THREAD_LOG_STACK_MAX 50
+#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
+
+extern __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
+extern __thread size_t thread_log_stack_next;
+extern __thread struct log_field thread_log_fields[_NDF_MAX];
+
+// --------------------------------------------------------------------------------------------------------------------
+
+extern struct nd_log nd_log;
+bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd);
+void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source);
+void nd_log_stdin_init(int fd, const char *filename);
+
+// --------------------------------------------------------------------------------------------------------------------
+// annotators
+
+struct log_field;
+const char *errno_annotator(struct log_field *lf);
+const char *priority_annotator(struct log_field *lf);
+const char *timestamp_usec_annotator(struct log_field *lf);
+
+#if defined(OS_WINDOWS)
+const char *winerror_annotator(struct log_field *lf);
+#endif
+
+// --------------------------------------------------------------------------------------------------------------------
+// field formatters
+
+uint64_t log_field_to_uint64(struct log_field *lf);
+int64_t log_field_to_int64(struct log_field *lf);
+
+// --------------------------------------------------------------------------------------------------------------------
+// common text formatters
+
+void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max);
+void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max);
+
+// --------------------------------------------------------------------------------------------------------------------
+// output to syslog
+
+void nd_log_init_syslog(void);
+void nd_log_reset_syslog(void);
+bool nd_logger_syslog(int priority, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
+
+// --------------------------------------------------------------------------------------------------------------------
+// output to systemd-journal
+
+bool nd_log_journal_systemd_init(void);
+bool nd_log_journal_direct_init(const char *path);
+bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max);
+bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max);
+
+// --------------------------------------------------------------------------------------------------------------------
+// output to file
+
+bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
+
+// --------------------------------------------------------------------------------------------------------------------
+// output to windows events log
+
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+bool nd_log_init_etw(void);
+bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
+#endif
+#if defined(HAVE_WEL)
+bool nd_log_init_wel(void);
+bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
+#endif
+#endif
+
+#endif //NETDATA_ND_LOG_INTERNALS_H
diff --git a/src/libnetdata/log/nd_log-to-file.c b/src/libnetdata/log/nd_log-to-file.c
new file mode 100644
index 0000000000..2de76536b1
--- /dev/null
+++ b/src/libnetdata/log/nd_log-to-file.c
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+void chown_open_file(int fd, uid_t uid, gid_t gid) {
+ if(fd == -1) return;
+
+ struct stat buf;
+
+ if(fstat(fd, &buf) == -1) {
+ netdata_log_error("Cannot fstat() fd %d", fd);
+ return;
+ }
+
+ if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
+ if(fchown(fd, uid, gid) == -1)
+ netdata_log_error("Cannot fchown() fd %d.", fd);
+ }
+}
+
+void nd_log_chown_log_files(uid_t uid, gid_t gid) {
+ for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
+ if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
+ chown_open_file(nd_log.sources[i].fd, uid, gid);
+ }
+}
+
+bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
+ BUFFER *wb = buffer_create(1024, NULL);
+
+ if(format == NDLF_JSON)
+ nd_logger_json(wb, fields, fields_max);
+ else
+ nd_logger_logfmt(wb, fields, fields_max);
+
+ int r = fprintf(fp, "%s\n", buffer_tostring(wb));
+ fflush(fp);
+
+ buffer_free(wb);
+ return r > 0;
+}
diff --git a/src/libnetdata/log/nd_log-to-syslog.c b/src/libnetdata/log/nd_log-to-syslog.c
new file mode 100644
index 0000000000..2903bf5917
--- /dev/null
+++ b/src/libnetdata/log/nd_log-to-syslog.c
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+void nd_log_init_syslog(void) {
+ if(nd_log.syslog.initialized)
+ return;
+
+ openlog(program_name, LOG_PID, nd_log.syslog.facility);
+ nd_log.syslog.initialized = true;
+}
+
+bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) {
+ CLEAN_BUFFER *wb = buffer_create(1024, NULL);
+
+ nd_logger_logfmt(wb, fields, fields_max);
+ syslog(priority, "%s", buffer_tostring(wb));
+
+ return true;
+}
diff --git a/src/libnetdata/log/nd_log-to-systemd-journal.c b/src/libnetdata/log/nd_log-to-systemd-journal.c
new file mode 100644
index 0000000000..b574e693cf
--- /dev/null
+++ b/src/libnetdata/log/nd_log-to-systemd-journal.c
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+bool nd_log_journal_systemd_init(void) {
+#ifdef HAVE_SYSTEMD
+ nd_log.journal.initialized = true;
+#else
+ nd_log.journal.initialized = false;
+#endif
+
+ return nd_log.journal.initialized;
+}
+
+bool nd_log_journal_socket_available(void) {
+ if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
+ char filename[FILENAME_MAX + 1];
+
+ snprintfz(filename, sizeof(filename), "%s%s",
+ netdata_configured_host_prefix, "/run/systemd/journal/socket");
+
+ if(is_path_unix_socket(filename))
+ return true;
+ }
+
+ return is_path_unix_socket("/run/systemd/journal/socket");
+}
+
+static void nd_log_journal_direct_set_env(void) {
+ if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
+ nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
+}
+
+bool nd_log_journal_direct_init(const char *path) {
+ if(nd_log.journal_direct.initialized) {
+ nd_log_journal_direct_set_env();
+ return true;
+ }
+
+ int fd;
+ char filename[FILENAME_MAX + 1];
+ if(!is_path_unix_socket(path)) {
+
+ journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
+
+ journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
+
+ journal_construct_path(filename, sizeof(filename), NULL, "netdata");
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
+
+ journal_construct_path(filename, sizeof(filename), NULL, NULL);
+ if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
+ return false;
+ }
+ }
+ }
+ }
+ else {
+ snprintfz(filename, sizeof(filename), "%s", path);
+ fd = journal_direct_fd(filename);
+ }
+
+ if(fd < 0)
+ return false;
+
+ nd_log.journal_direct.fd = fd;
+ nd_log.journal_direct.initialized = true;
+
+ strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
+ nd_log_journal_direct_set_env();
+
+ return true;
+}
+
+bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
+#ifdef HAVE_SYSTEMD
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ struct iovec iov[fields_max];
+ int iov_count = 0;
+
+ memset(iov, 0, sizeof(iov));
+
+ CLEAN_BUFFER *tmp = NULL;
+
+ for (size_t i = 0; i < fields_max; i++) {
+ if (!fields[i].entry.set || !fields[i].journal)
+ continue;
+
+ const char *key = fields[i].journal;
+ char *value = NULL;
+ int rc = 0;
+ switch (fields[i].entry.type) {
+ case NDFT_TXT:
+ if(*fields[i].entry.txt)
+ rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt);
+ break;
+ case NDFT_STR:
+ rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
+ break;
+ case NDFT_BFR:
+ if(buffer_strlen(fields[i].entry.bfr))
+ rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
+ break;
+ case NDFT_U64:
+ rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
+ break;
+ case NDFT_I64:
+ rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
+ break;
+ case NDFT_DBL:
+ rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
+ break;
+ case NDFT_UUID:
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
+ char u[UUID_COMPACT_STR_LEN];
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
+ rc = asprintf(&value, "%s=%s", key, u);
+ }
+ break;
+ case NDFT_CALLBACK: {
+ if(!tmp)
+ tmp = buffer_create(1024, NULL);
+ else
+ buffer_flush(tmp);
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
+ rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
+ }
+ break;
+ default:
+ rc = asprintf(&value, "%s=%s", key, "UNHANDLED");
+ break;
+ }
+
+ if (rc != -1 && value) {
+ iov[iov_count].iov_base = value;
+ iov[iov_count].iov_len = strlen(value);
+ iov_count++;
+ }
+ }
+
+ int r = sd_journal_sendv(iov, iov_count);
+
+ // Clean up allocated memory
+ for (int i = 0; i < iov_count; i++) {
+ if (iov[i].iov_base != NULL) {
+ free(iov[i].iov_base);
+ }
+ }
+
+ return r == 0;
+#else
+ return false;
+#endif
+}
+
+bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
+ if(!nd_log.journal_direct.initialized)
+ return false;
+
+ // --- FIELD_PARSER_VERSIONS ---
+ //
+ // IMPORTANT:
+ // THERE ARE 6 VERSIONS OF THIS CODE
+ //
+ // 1. journal (direct socket API),
+ // 2. journal (libsystemd API),
+ // 3. logfmt,
+ // 4. json,
+ // 5. convert to uint64
+ // 6. convert to int64
+ //
+ // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
+
+ CLEAN_BUFFER *wb = buffer_create(4096, NULL);
+ CLEAN_BUFFER *tmp = NULL;
+
+ for (size_t i = 0; i < fields_max; i++) {
+ if (!fields[i].entry.set || !fields[i].journal)
+ continue;
+
+ const char *key = fields[i].journal;
+
+ const char *s = NULL;
+ switch(fields[i].entry.type) {
+ case NDFT_TXT:
+ s = fields[i].entry.txt;
+ break;
+ case NDFT_STR:
+ s = string2str(fields[i].entry.str);
+ break;
+ case NDFT_BFR:
+ s = buffer_tostring(fields[i].entry.bfr);
+ break;
+ case NDFT_U64:
+ buffer_strcat(wb, key);
+ buffer_putc(wb, '=');
+ buffer_print_uint64(wb, fields[i].entry.u64);
+ buffer_putc(wb, '\n');
+ break;
+ case NDFT_I64:
+ buffer_strcat(wb, key);
+ buffer_putc(wb, '=');
+ buffer_print_int64(wb, fields[i].entry.i64);
+ buffer_putc(wb, '\n');
+ break;
+ case NDFT_DBL:
+ buffer_strcat(wb, key);
+ buffer_putc(wb, '=');
+ buffer_print_netdata_double(wb, fields[i].entry.dbl);
+ buffer_putc(wb, '\n');
+ break;
+ case NDFT_UUID:
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
+ char u[UUID_COMPACT_STR_LEN];
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
+ buffer_strcat(wb, key);
+ buffer_putc(wb, '=');
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
+ buffer_putc(wb, '\n');
+ }
+ break;
+ case NDFT_CALLBACK: {
+ if(!tmp)
+ tmp = buffer_create(1024, NULL);
+ else
+ buffer_flush(tmp);
+ if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
+ s = buffer_tostring(tmp);
+ else
+ s = NULL;
+ }
+ break;
+ default:
+ s = "UNHANDLED";
+ break;
+ }
+
+ if(s && *s) {
+ buffer_strcat(wb, key);
+ if(!strchr(s, '\n')) {
+ buffer_putc(wb, '=');
+ buffer_strcat(wb, s);
+ buffer_putc(wb, '\n');
+ }
+ else {
+ buffer_putc(wb, '\n');
+ size_t size = strlen(s);
+ uint64_t le_size = htole64(size);
+ buffer_memcat(wb, &le_size, sizeof(le_size));
+ buffer_memcat(wb, s, size);
+ buffer_putc(wb, '\n');
+ }
+ }
+ }
+
+ return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
+}
diff --git a/src/libnetdata/log/nd_log-to-windows-common.h b/src/libnetdata/log/nd_log-to-windows-common.h
new file mode 100644
index 0000000000..2b2833ed1d
--- /dev/null
+++ b/src/libnetdata/log/nd_log-to-windows-common.h
@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
+#define NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
+
+// Helper macro to create wide string literals
+#define WIDEN2(x) L ## x
+#define WIDEN(x) WIDEN2(x)
+
+#define NETDATA_ETW_PROVIDER_GUID_STR "{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
+#define NETDATA_ETW_PROVIDER_GUID_STR_W WIDEN(NETDATA_ETW_PROVIDER_GUID)
+
+#define NETDATA_CHANNEL_NAME "Netdata"
+#define NETDATA_CHANNEL_NAME_W WIDEN(NETDATA_CHANNEL_NAME)
+
+#define NETDATA_WEL_CHANNEL_NAME "NetdataWEL"
+#define NETDATA_WEL_CHANNEL_NAME_W WIDEN(NETDATA_WEL_CHANNEL_NAME)
+
+#define NETDATA_ETW_CHANNEL_NAME "Netdata"
+#define NETDATA_ETW_CHANNEL_NAME_W WIDEN(NETDATA_ETW_CHANNEL_NAME)
+
+#define NETDATA_ETW_PROVIDER_NAME "Netdata"
+#define NETDATA_ETW_PROVIDER_NAME_W WIDEN(NETDATA_ETW_PROVIDER_NAME)
+
+#define NETDATA_WEL_PROVIDER_PREFIX "Netdata"
+#define NETDATA_WEL_PROVIDER_PREFIX_W WIDEN(NETDATA_WEL_PROVIDER_PREFIX)
+
+#define NETDATA_WEL_PROVIDER_ACCESS NETDATA_WEL_PROVIDER_PREFIX "Access"
+#define NETDATA_WEL_PROVIDER_ACCESS_W WIDEN(NETDATA_WEL_PROVIDER_ACCESS)
+
+#define NETDATA_WEL_PROVIDER_ACLK NETDATA_WEL_PROVIDER_PREFIX "Aclk"
+#define NETDATA_WEL_PROVIDER_ACLK_W WIDEN(NETDATA_WEL_PROVIDER_ACLK)
+
+#define NETDATA_WEL_PROVIDER_COLLECTORS NETDATA_WEL_PROVIDER_PREFIX "Collectors"
+#define NETDATA_WEL_PROVIDER_COLLECTORS_W WIDEN(NETDATA_WEL_PROVIDER_COLLECTORS)
+
+#define NETDATA_WEL_PROVIDER_DAEMON NETDATA_WEL_PROVIDER_PREFIX "Daemon"
+#define NETDATA_WEL_PROVIDER_DAEMON_W WIDEN(NETDATA_WEL_PROVIDER_DAEMON)
+
+#define NETDATA_WEL_PROVIDER_HEALTH NETDATA_WEL_PROVIDER_PREFIX "Health"
+#define NETDATA_WEL_PROVIDER_HEALTH_W WIDEN(NETDATA_WEL_PROVIDER_HEALTH)
+
+
+#define NETDATA_ETW_SUBCHANNEL_ACCESS "Access"
+#define NETDATA_ETW_SUBCHANNEL_ACCESS_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACCESS)
+
+#define NETDATA_ETW_SUBCHANNEL_ACLK "Aclk"
+#define NETDATA_ETW_SUBCHANNEL_ACLK_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACLK)
+
+#define NETDATA_ETW_SUBCHANNEL_COLLECTORS "Collectors"
+#define NETDATA_ETW_SUBCHANNEL_COLLECTORS_W WIDEN(NETDATA_ETW_SUBCHANNEL_COLLECTORS)
+
+#define NETDATA_ETW_SUBCHANNEL_DAEMON "Daemon"
+#define NETDATA_ETW_SUBCHANNEL_DAEMON_W WIDEN(NETDATA_ETW_SUBCHANNEL_DAEMON)
+
+#define NETDATA_ETW_SUBCHANNEL_HEALTH "Health"
+#define NETDATA_ETW_SUBCHANNEL_HEALTH_W WIDEN(NETDATA_ETW_SUBCHANNEL_HEALTH)
+
+// Define shift values
+#define EVENT_ID_SEV_SHIFT 30
+#define EVENT_ID_C_SHIFT 29
+#define EVENT_ID_R_SHIFT 28
+#define EVENT_ID_FACILITY_SHIFT 16
+#define EVENT_ID_CODE_SHIFT 0
+
+#define EVENT_ID_PRIORITY_SHIFT 0 // Shift 0 bits
+#define EVENT_ID_SOURCE_SHIFT 4 // Shift 4 bits
+
+// Define masks
+#define EVENT_ID_SEV_MASK 0xC0000000 // Bits 31-30
+#define EVENT_ID_C_MASK 0x20000000 // Bit 29
+#define EVENT_ID_R_MASK 0x10000000 // Bit 28
+#define EVENT_ID_FACILITY_MASK 0x0FFF0000 // Bits 27-16
+#define EVENT_ID_CODE_MASK 0x0000FFFF // Bits 15-0
+
+#define EVENT_ID_PRIORITY_MASK 0x000F // Bits 0-3
+#define EVENT_ID_SOURCE_MASK 0x00F0 // Bits 4-7
+
+typedef enum __attribute__((packed)) {
+ MSGID_MESSAGE_ONLY = 1,
+ MSGID_MESSAGE_ERRNO,
+ MSGID_REQUEST_ONLY,
+ MSGID_ALERT_TRANSITION,
+ MSGID_ACCESS,
+ MSGID_ACCESS_FORWARDER,
+ MSGID_ACCESS_USER,
+ MSGID_ACCESS_FORWARDER_USER,
+ MSGID_ACCESS_MESSAGE,
+ MSGID_ACCESS_MESSAGE_REQUEST,
+ MSGID_ACCESS_MESSAGE_USER,
+
+ // terminator
+ _MSGID_MAX,
+} MESSAGE_ID;
+
+static inline uint32_t get_event_type_from_priority(ND_LOG_FIELD_PRIORITY priority) {
+ switch (priority) {
+ case NDLP_EMERG:
+ case NDLP_ALERT:
+ case NDLP_CRIT:
+ case NDLP_ERR:
+ return EVENTLOG_ERROR_TYPE;
+
+ case NDLP_WARNING:
+ return EVENTLOG_WARNING_TYPE;
+
+ case NDLP_NOTICE:
+ case NDLP_INFO:
+ case NDLP_DEBUG:
+ default:
+ return EVENTLOG_INFORMATION_TYPE;
+ }
+}
+
+static inline uint8_t get_severity_from_priority(ND_LOG_FIELD_PRIORITY priority) {
+ switch (priority) {
+ case NDLP_EMERG:
+ case NDLP_ALERT:
+ case NDLP_CRIT:
+ case NDLP_ERR:
+ return STATUS_SEVERITY_ERROR;
+
+ case NDLP_WARNING:
+ return STATUS_SEVERITY_WARNING;
+
+ case NDLP_NOTICE:
+ case NDLP_INFO:
+ case NDLP_DEBUG:
+ default:
+ return STATUS_SEVERITY_INFORMATIONAL;
+ }
+}
+
+static inline uint8_t get_level_from_priority(ND_LOG_FIELD_PRIORITY priority) {
+ switch (priority) {
+ // return 0 = log an event regardless of any filtering applied
+
+ case NDLP_EMERG:
+ case NDLP_ALERT:
+ case NDLP_CRIT:
+ return 1;
+
+ case NDLP_ERR:
+ return 2;
+
+ case NDLP_WARNING:
+ return 3;
+
+ case NDLP_NOTICE:
+ case NDLP_INFO:
+ return 4;
+
+ case NDLP_DEBUG:
+ default:
+ return 5;
+ }
+}
+
+static inline const char *get_level_from_priority_str(ND_LOG_FIELD_PRIORITY priority) {
+ switch (priority) {
+ // return "win:LogAlways" to log an event regardless of any filtering applied
+
+ case NDLP_EMERG:
+ case NDLP_ALERT:
+ case NDLP_CRIT:
+ return "win:Critical";
+
+ case NDLP_ERR:
+ return "win:Error";
+
+ case NDLP_WARNING:
+ return "win:Warning";
+
+ case NDLP_NOTICE:
+ case NDLP_INFO:
+ return "win:Informational";
+
+ case NDLP_DEBUG:
+ default:
+ return "win:Verbose";
+ }
+}
+
+static inline uint16_t construct_event_code(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
+ return (source << 12 | priority << 8 | messageID << 0);
+}
+
+#endif //NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
diff --git a/src/libnetdata/log/nd_log-to-windows-events.c b/src/libnetdata/log/nd_log-to-windows-events.c
new file mode 100644
index 0000000000..61d87ad1b7
--- /dev/null
+++ b/src/libnetdata/log/nd_log-to-windows-events.c
@@ -0,0 +1,560 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log-internals.h"
+
+#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
+#include <windows.h>
+#include <winevt.h>
+#include <evntprov.h>
+#include <wchar.h>
+#include <guiddef.h>
+#include <wctype.h>
+
+// --------------------------------------------------------------------------------------------------------------------
+// construct an event id
+
+// load message resources generated header
+#include "wevt_netdata.h"
+
+// include the common definitions with the message resources and manifest generator
+#include "nd_log-to-windows-common.h"
+
+#if defined(HAVE_ETW)
+// we need the manifest, only in ETW mode
+
+// eliminate compiler warnings and load manifest generated header
+#undef EXTERN_C
+#define EXTERN_C
+#undef __declspec
+#define __declspec(x)
+#include "wevt_netdata_manifest.h"
+
+static REGHANDLE regHandle;
+#endif
+
+// Function to construct EventID
+static DWORD complete_event_id(DWORD facility, DWORD severity, DWORD event_code) {
+ DWORD event_id = 0;
+
+ // Set Severity
+ event_id |= ((DWORD)(severity) << EVENT_ID_SEV_SHIFT) & EVENT_ID_SEV_MASK;
+
+ // Set Customer Code Flag (C)
+ event_id |= (0x0 << EVENT_ID_C_SHIFT) & EVENT_ID_C_MASK;
+
+ // Set Reserved Bit (R) - typically 0
+ event_id |= (0x0 << EVENT_ID_R_SHIFT) & EVENT_ID_R_MASK;
+
+ // Set Facility
+ event_id |= ((DWORD)(facility) << EVENT_ID_FACILITY_SHIFT) & EVENT_ID_FACILITY_MASK;
+
+ // Set Code
+ event_id |= ((DWORD)(event_code) << EVENT_ID_CODE_SHIFT) & EVENT_ID_CODE_MASK;
+
+ return event_id;
+}
+
+DWORD construct_event_id(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
+ DWORD event_code = construct_event_code(source, priority, messageID);
+ return complete_event_id(FACILITY_NETDATA, get_severity_from_priority(priority), event_code);
+}
+
+static bool check_event_id(ND_LOG_SOURCES source __maybe_unused, ND_LOG_FIELD_PRIORITY priority __maybe_unused, MESSAGE_ID messageID __maybe_unused, DWORD event_code __maybe_unused) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ DWORD generated = construct_event_id(source, priority, messageID);
+ if(generated != event_code) {
+
+ // this is just used for a break point, to see the values in hex
+ char current[UINT64_HEX_MAX_LENGTH];
+ print_uint64_hex(current, generated);
+
+ char wanted[UINT64_HEX_MAX_LENGTH];
+ print_uint64_hex(wanted, event_code);
+
+ const char *got = current;
+ const char *good = wanted;
+ internal_fatal(true, "EventIDs mismatch, expected %s, got %s", good, got);
+ }
+#endif
+
+ return true;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// initialization
+
+// Define provider names per source (only when not using ETW)
+static const wchar_t *wel_provider_per_source[_NDLS_MAX] = {
+ [NDLS_UNSET] = NULL, // not used, linked to NDLS_DAEMON
+ [NDLS_ACCESS] = NETDATA_WEL_PROVIDER_ACCESS_W, //
+ [NDLS_ACLK] = NETDATA_WEL_PROVIDER_ACLK_W, //
+ [NDLS_COLLECTORS] = NETDATA_WEL_PROVIDER_COLLECTORS_W,//
+ [NDLS_DAEMON] = NETDATA_WEL_PROVIDER_DAEMON_W, //
+ [NDLS_HEALTH] = NETDATA_WEL_PROVIDER_HEALTH_W, //
+ [NDLS_DEBUG] = NULL, // used, linked to NDLS_DAEMON
+};
+
+bool wel_replace_program_with_wevt_netdata_dll(wchar_t *str, size_t size) {
+ const wchar_t *replacement = L"\\wevt_netdata.dll";
+
+ // Find the last occurrence of '\\' to isolate the filename
+ wchar_t *lastBackslash = wcsrchr(str, L'\\');
+
+ if (lastBackslash != NULL) {
+ // Calculate new length after replacement
+ size_t newLen = (lastBackslash - str) + wcslen(replacement);
+
+ // Ensure new length does not exceed buffer size
+ if (newLen >= size)
+ return false; // Not enough space in the buffer
+
+ // Terminate the string at the last backslash
+ *lastBackslash = L'\0';
+
+ // Append the replacement filename
+ wcsncat(str, replacement, size - wcslen(str) - 1);
+
+ // Check if the new file exists
+ if (GetFileAttributesW(str) != INVALID_FILE_ATTRIBUTES)
+ return true; // The file exists
+ else
+ return false; // The file does not exist
+ }
+
+ return false; // No backslash found (likely invalid input)
+}
+
+static bool wel_add_to_registry(const wchar_t *channel, const wchar_t *provider, DWORD defaultMaxSize) {
+ // Build the registry path: SYSTEM\CurrentControlSet\Services\EventLog\<LogName>\<SourceName>
+ wchar_t key[MAX_PATH];
+ if(!provider)
+ swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls", channel);
+ else
+ swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls\\%ls", channel, provider);
+
+ HKEY hRegKey;
+ DWORD disposition;
+ LONG result = RegCreateKeyExW(HKEY_LOCAL_MACHINE, key,
+ 0, NULL, REG_OPTION_NON_VOLATILE, KEY_SET_VALUE, NULL, &hRegKey, &disposition);
+
+ if (result != ERROR_SUCCESS)
+ return false; // Could not create the registry key
+
+ // Check if MaxSize is already set
+ DWORD maxSize = 0;
+ DWORD size = sizeof(maxSize);
+ if (RegQueryValueExW(hRegKey, L"MaxSize", NULL, NULL, (LPBYTE)&maxSize, &size) != ERROR_SUCCESS) {
+ // MaxSize is not set, set it to the default value
+ RegSetValueExW(hRegKey, L"MaxSize", 0, REG_DWORD, (const BYTE*)&defaultMaxSize, sizeof(defaultMaxSize));
+ }
+
+ wchar_t modulePath[MAX_PATH];
+ if (GetModuleFileNameW(NULL, modulePath, MAX_PATH) == 0) {
+ RegCloseKey(hRegKey);
+ return false;
+ }
+
+ if(wel_replace_program_with_wevt_netdata_dll(modulePath, _countof(modulePath))) {
+ RegSetValueExW(hRegKey, L"EventMessageFile", 0, REG_EXPAND_SZ,
+ (LPBYTE)modulePath, (wcslen(modulePath) + 1) * sizeof(wchar_t));
+
+ DWORD types_supported = EVENTLOG_SUCCESS | EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE;
+ RegSetValueExW(hRegKey, L"TypesSupported", 0, REG_DWORD, (LPBYTE)&types_supported, sizeof(DWORD));
+ }
+
+ RegCloseKey(hRegKey);
+ return true;
+}
+
+#if defined(HAVE_ETW)
+static void etw_set_source_meta(struct nd_log_source *source, USHORT channelID, const EVENT_DESCRIPTOR *ed) {
+ // It turns out that the keyword varies per only per channel!
+ // so, to log with the right keyword, Task, Opcode we copy the ids from the header
+ // the messages compiler (mc.exe) generated from the manifest.
+
+ source->channelID = channelID;
+ source->Opcode = ed->Opcode;
+ source->Task = ed->Task;
+ source->Keyword = ed->Keyword;
+}
+
+static bool etw_register_provider(void) {
+ // Register the ETW provider
+ if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, NULL, NULL, ®Handle) != ERROR_SUCCESS)
+ return false;
+
+ etw_set_source_meta(&nd_log.sources[NDLS_DAEMON], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_COLLECTORS], CHANNEL_COLLECTORS, &ED_COLLECTORS_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_ACCESS], CHANNEL_ACCESS, &ED_ACCESS_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_HEALTH], CHANNEL_HEALTH, &ED_HEALTH_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_ACLK], CHANNEL_ACLK, &ED_ACLK_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_UNSET], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
+ etw_set_source_meta(&nd_log.sources[NDLS_DEBUG], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
+
+ return true;
+}
+#endif
+
+bool nd_log_init_windows(void) {
+ if(nd_log.eventlog.initialized)
+ return true;
+
+ // validate we have the right keys
+ if(
+ !check_event_id(NDLS_COLLECTORS, NDLP_INFO, MSGID_MESSAGE_ONLY, MC_COLLECTORS_INFO_MESSAGE_ONLY) ||
+ !check_event_id(NDLS_DAEMON, NDLP_ERR, MSGID_MESSAGE_ONLY, MC_DAEMON_ERR_MESSAGE_ONLY) ||
+ !check_event_id(NDLS_ACCESS, NDLP_WARNING, MSGID_ACCESS_USER, MC_ACCESS_WARN_ACCESS_USER) ||
+ !check_event_id(NDLS_HEALTH, NDLP_CRIT, MSGID_ALERT_TRANSITION, MC_HEALTH_CRIT_ALERT_TRANSITION) ||
+ !check_event_id(NDLS_DEBUG, NDLP_ALERT, MSGID_ACCESS_FORWARDER_USER, MC_DEBUG_ALERT_ACCESS_FORWARDER_USER))
+ return false;
+
+#if defined(HAVE_ETW)
+ if(nd_log.eventlog.etw && !etw_register_provider())
+ return false;
+#endif
+
+// if(!nd_log.eventlog.etw && !wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, NULL, 50 * 1024 * 1024))
+// return false;
+
+ // Loop through each source and add it to the registry
+ for(size_t i = 0; i < _NDLS_MAX; i++) {
+ nd_log.sources[i].source = i;
+
+ const wchar_t *sub_channel = wel_provider_per_source[i];
+
+ if(!sub_channel)
+ // we will map these to NDLS_DAEMON
+ continue;
+
+ DWORD defaultMaxSize = 0;
+ switch (i) {
+ case NDLS_ACLK:
+ defaultMaxSize = 5 * 1024 * 1024;
+ break;
+
+ case NDLS_HEALTH:
+ defaultMaxSize = 35 * 1024 * 1024;
+ break;
+
+ default:
+ case NDLS_ACCESS:
+ case NDLS_COLLECTORS:
+ case NDLS_DAEMON:
+ defaultMaxSize = 20 * 1024 * 1024;
+ break;
+ }
+
+ if(!nd_log.eventlog.etw) {
+ if(!wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, sub_channel, defaultMaxSize))
+ return false;
+
+ // when not using a manifest, each source is a provider
+ nd_log.sources[i].hEventLog = RegisterEventSourceW(NULL, sub_channel);
+ if (!nd_log.sources[i].hEventLog)
+ return false;
+ }
+ }
+
+ if(!nd_log.eventlog.etw) {
+ // Map the unset ones to NDLS_DAEMON
+ for (size_t i = 0; i < _NDLS_MAX; i++) {
+ if (!nd_log.sources[i].hEventLog)
+ nd_log.sources[i].hEventLog = nd_log.sources[NDLS_DAEMON].hEventLog;
+ }
+ }
+
+ nd_log.eventlog.initialized = true;
+ return true;
+}
+
+bool nd_log_init_etw(void) {
+ nd_log.eventlog.etw = true;
+ return nd_log_init_windows();
+}
+
+bool nd_log_init_wel(void) {
+ nd_log.eventlog.etw = false;
+ return nd_log_init_windows();
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// we pass all our fields to the windows events logs
+// numbered the same way we have them in memory.
+//
+// to avoid runtime memory allocations, we use a static allocations with ready to use buffers
+// which are immediately available for logging.
+
+#define SMALL_WIDE_BUFFERS_SIZE 256
+#define MEDIUM_WIDE_BUFFERS_SIZE 2048
+#define BIG_WIDE_BUFFERS_SIZE 16384
+static wchar_t small_wide_buffers[_NDF_MAX][SMALL_WIDE_BUFFERS_SIZE];
+static wchar_t medium_wide_buffers[2][MEDIUM_WIDE_BUFFERS_SIZE];
+static wchar_t big_wide_buffers[2][BIG_WIDE_BUFFERS_SIZE];
+
+static struct {
+ size_t size;
+ wchar_t *buf;
+} fields_buffers[_NDF_MAX] = { 0 };
+
+#if defined(HAVE_ETW)
+static EVENT_DATA_DESCRIPTOR etw_eventData[_NDF_MAX - 1];
+#endif
+
+static LPCWSTR wel_messages[_NDF_MAX - 1];
+
+__attribute__((constructor)) void wevents_initialize_buffers(void) {
+ for(size_t i = 0; i < _NDF_MAX ;i++) {
+ fields_buffers[i].buf = small_wide_buffers[i];
+ fields_buffers[i].size = SMALL_WIDE_BUFFERS_SIZE;
+ }
+
+ fields_buffers[NDF_NIDL_INSTANCE].buf = medium_wide_buffers[0];
+ fields_buffers[NDF_NIDL_INSTANCE].size = MEDIUM_WIDE_BUFFERS_SIZE;
+
+ fields_buffers[NDF_REQUEST].buf = big_wide_buffers[0];
+ fields_buffers[NDF_REQUEST].size = BIG_WIDE_BUFFERS_SIZE;
+ fields_buffers[NDF_MESSAGE].buf = big_wide_buffers[1];
+ fields_buffers[NDF_MESSAGE].size = BIG_WIDE_BUFFERS_SIZE;
+
+ for(size_t i = 1; i < _NDF_MAX ;i++)
+ wel_messages[i - 1] = fields_buffers[i].buf;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+#define is_field_set(fields, fields_max, field) ((field) < (fields_max) && (fields)[field].entry.set)
+
+static const char *get_field_value_unsafe(struct log_field *fields, ND_LOG_FIELD_ID i, size_t fields_max, BUFFER **tmp) {
+ if(!is_field_set(fields, fields_max, i) || !fields[i].eventlog)
+ return "";
+
+ static char number_str[MAX(MAX(UINT64_MAX_LENGTH, DOUBLE_MAX_LENGTH), UUID_STR_LEN)];
+
+ const char *s = NULL;
+ if (fields[i].annotator)
+ s = fields[i].annotator(&fields[i]);
+
+ else
+ switch (fields[i].entry.type) {
+ case NDFT_TXT:
+ s = fields[i].entry.txt;
+ break;
+ case NDFT_STR:
+ s = string2str(fields[i].entry.str);
+ break;
+ case NDFT_BFR:
+ s = buffer_tostring(fields[i].entry.bfr);
+ break;
+ case NDFT_U64:
+ print_uint64(number_str, fields[i].entry.u64);
+ s = number_str;
+ break;
+ case NDFT_I64:
+ print_int64(number_str, fields[i].entry.i64);
+ s = number_str;
+ break;
+ case NDFT_DBL:
+ print_netdata_double(number_str, fields[i].entry.dbl);
+ s = number_str;
+ break;
+ case NDFT_UUID:
+ if (!uuid_is_null(*fields[i].entry.uuid)) {
+ uuid_unparse_lower(*fields[i].entry.uuid, number_str);
+ s = number_str;
+ }
+ break;
+ case NDFT_CALLBACK:
+ if (!*tmp)
+ *tmp = buffer_create(1024, NULL);
+ else
+ buffer_flush(*tmp);
+
+ if (fields[i].entry.cb.formatter(*tmp, fields[i].entry.cb.formatter_data))
+ s = buffer_tostring(*tmp);
+ else
+ s = NULL;
+ break;
+
+ default:
+ s = "UNHANDLED";
+ break;
+ }
+
+ if(!s || !*s) return "";
+ return s;
+}
+static void etw_replace_percent_with_unicode(wchar_t *s, size_t size) {
+ size_t original_len = wcslen(s);
+
+ // Traverse the string, replacing '%' with the Unicode fullwidth percent sign
+ for (size_t i = 0; i < original_len && i < size - 1; i++) {
+ if (s[i] == L'%' && iswdigit(s[i + 1])) {
+ // s[i] = 0xFF05; // Replace '%' with fullwidth percent sign 'ï¼…'
+ // s[i] = 0x29BC; // ⦼
+ s[i] = 0x2105; // â„…
+ }
+ }
+
+ // Ensure null termination if needed
+ s[size - 1] = L'\0';
+}
+
+static void wevt_generate_all_fields_unsafe(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
+ for (size_t i = 0; i < fields_max; i++) {
+ fields_buffers[i].buf[0] = L'\0';
+
+ if (!fields[i].entry.set || !fields[i].eventlog)
+ continue;
+
+ const char *s = get_field_value_unsafe(fields, i, fields_max, tmp);
+ if (s && *s) {
+ utf8_to_utf16(fields_buffers[i].buf, (int) fields_buffers[i].size, s, -1);
+
+ if(nd_log.eventlog.etw)
+ // UNBELIEVABLE! they do recursive parameter expansion in ETW...
+ etw_replace_percent_with_unicode(fields_buffers[i].buf, fields_buffers[i].size);
+ }
+ }
+}
+
+static bool has_user_role_permissions(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
+ const char *t;
+
+ t = get_field_value_unsafe(fields, NDF_USER_NAME, fields_max, tmp);
+ if (*t) return true;
+
+ t = get_field_value_unsafe(fields, NDF_USER_ROLE, fields_max, tmp);
+ if (*t && strcmp(t, "none") != 0) return true;
+
+ t = get_field_value_unsafe(fields, NDF_USER_ACCESS, fields_max, tmp);
+ if (*t && strcmp(t, "0x0") != 0) return true;
+
+ return false;
+}
+
+static bool nd_logger_windows(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
+ if (!nd_log.eventlog.initialized)
+ return false;
+
+ ND_LOG_FIELD_PRIORITY priority = NDLP_INFO;
+ if (fields[NDF_PRIORITY].entry.set)
+ priority = (ND_LOG_FIELD_PRIORITY) fields[NDF_PRIORITY].entry.u64;
+
+ DWORD wType = get_event_type_from_priority(priority);
+ (void) wType;
+
+ CLEAN_BUFFER *tmp = NULL;
+
+ static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
+ spinlock_lock(&spinlock);
+ wevt_generate_all_fields_unsafe(fields, fields_max, &tmp);
+
+ MESSAGE_ID messageID;
+ switch (source->source) {
+ default:
+ case NDLS_DEBUG:
+ case NDLS_DAEMON:
+ case NDLS_COLLECTORS:
+ messageID = MSGID_MESSAGE_ONLY;
+ break;
+
+ case NDLS_HEALTH:
+ messageID = MSGID_ALERT_TRANSITION;
+ break;
+
+ case NDLS_ACCESS:
+ if (is_field_set(fields, fields_max, NDF_MESSAGE)) {
+ messageID = MSGID_ACCESS_MESSAGE;
+
+ if (has_user_role_permissions(fields, fields_max, &tmp))
+ messageID = MSGID_ACCESS_MESSAGE_USER;
+ else if (*get_field_value_unsafe(fields, NDF_REQUEST, fields_max, &tmp))
+ messageID = MSGID_ACCESS_MESSAGE_REQUEST;
+ } else if (is_field_set(fields, fields_max, NDF_RESPONSE_CODE)) {
+ messageID = MSGID_ACCESS;
+
+ if (*get_field_value_unsafe(fields, NDF_SRC_FORWARDED_FOR, fields_max, &tmp))
+ messageID = MSGID_ACCESS_FORWARDER;
+
+ if (has_user_role_permissions(fields, fields_max, &tmp)) {
+ if (messageID == MSGID_ACCESS)
+ messageID = MSGID_ACCESS_USER;
+ else
+ messageID = MSGID_ACCESS_FORWARDER_USER;
+ }
+ } else
+ messageID = MSGID_REQUEST_ONLY;
+ break;
+
+ case NDLS_ACLK:
+ messageID = MSGID_MESSAGE_ONLY;
+ break;
+ }
+
+ if (messageID == MSGID_MESSAGE_ONLY && (
+ *get_field_value_unsafe(fields, NDF_ERRNO, fields_max, &tmp) ||
+ *get_field_value_unsafe(fields, NDF_WINERROR, fields_max, &tmp))) {
+ messageID = MSGID_MESSAGE_ERRNO;
+ }
+
+ DWORD eventID = construct_event_id(source->source, priority, messageID);
+
+ // wType
+ //
+ // without a manifest => this determines the Level of the event
+ // with a manifest => Level from the manifest is used (wType ignored)
+ // [however it is good to have, in case the manifest is not accessible somehow]
+ //
+
+ // wCategory
+ //
+ // without a manifest => numeric Task values appear
+ // with a manifest => Task from the manifest is used (wCategory ignored)
+
+ BOOL rc;
+#if defined(HAVE_ETW)
+ if (nd_log.eventlog.etw) {
+ // metadata based logging - ETW
+
+ for (size_t i = 1; i < _NDF_MAX; i++)
+ EventDataDescCreate(&etw_eventData[i - 1], fields_buffers[i].buf,
+ (wcslen(fields_buffers[i].buf) + 1) * sizeof(WCHAR));
+
+ EVENT_DESCRIPTOR EventDesc = {
+ .Id = eventID & EVENT_ID_CODE_MASK, // ETW needs the raw event id
+ .Version = 0,
+ .Channel = source->channelID,
+ .Level = get_level_from_priority(priority),
+ .Opcode = source->Opcode,
+ .Task = source->Task,
+ .Keyword = source->Keyword,
+ };
+
+ rc = ERROR_SUCCESS == EventWrite(regHandle, &EventDesc, _NDF_MAX - 1, etw_eventData);
+
+ }
+ else
+#endif
+ {
+ // eventID based logging - WEL
+ rc = ReportEventW(source->hEventLog, wType, 0, eventID, NULL, _NDF_MAX - 1, 0, wel_messages, NULL);
+ }
+
+ spinlock_unlock(&spinlock);
+
+ return rc == TRUE;
+}
+
+#if defined(HAVE_ETW)
+bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
+ return nd_logger_windows(source, fields, fields_max);
+}
+#endif
+
+#if defined(HAVE_WEL)
+bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
+ return nd_logger_windows(source, fields, fields_max);
+}
+#endif
+
+#endif
diff --git a/src/libnetdata/log/nd_log.c b/src/libnetdata/log/nd_log.c
new file mode 100644
index 0000000000..a605fe4604
--- /dev/null
+++ b/src/libnetdata/log/nd_log.c
@@ -0,0 +1,465 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the
+// source code that makes the calls, allowing our loggers to log the lines of source code that actually log
+#define SD_JOURNAL_SUPPRESS_LOCATION
+
+#include "../libnetdata.h"
+#include "nd_log-internals.h"
+
+const char *program_name = "";
+uint64_t debug_flags = 0;
+int aclklog_enabled = 0;
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void errno_clear(void) {
+ errno = 0;
+
+#if defined(OS_WINDOWS)
+ SetLastError(ERROR_SUCCESS);
+#endif
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// logger router
+
+static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
+ *spinlock = NULL;
+ ND_LOG_METHOD output = nd_log.sources[source].method;
+
+ switch(output) {
+ case NDLM_JOURNAL:
+ if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
+ output = NDLM_FILE;
+ *fpp = stderr;
+ *spinlock = &nd_log.std_error.spinlock;
+ }
+ else {
+ *fpp = NULL;
+ *spinlock = NULL;
+ }
+ break;
+
+#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
+#if defined(HAVE_ETW)
+ case NDLM_ETW:
+#endif
+#if defined(HAVE_WEL)
+ case NDLM_WEL:
+#endif
+ if(unlikely(!nd_log.eventlog.initialized)) {
+ output = NDLM_FILE;
+ *fpp = stderr;
+ *spinlock = &nd_log.std_error.spinlock;
+ }
+ else {
+ *fpp = NULL;
+ *spinlock = NULL;
+ }
+ break;
+#endif
+
+ case NDLM_SYSLOG:
+ if(unlikely(!nd_log.syslog.initialized)) {
+ output = NDLM_FILE;
+ *spinlock = &nd_log.std_error.spinlock;
+ *fpp = stderr;
+ }
+ else {
+ *spinlock = NULL;
+ *fpp = NULL;
+ }
+ break;
+
+ case NDLM_FILE:
+ if(!nd_log.sources[source].fp) {
+ *fpp = stderr;
+ *spinlock = &nd_log.std_error.spinlock;
+ }
+ else {
+ *fpp = nd_log.sources[source].fp;
+ *spinlock = &nd_log.sources[source].spinlock;
+ }
+ break;
+
+ case NDLM_STDOUT:
+ output = NDLM_FILE;
+ *fpp = stdout;
+ *spinlock = &nd_log.std_output.spinlock;
+ break;
+
+ default:
+ case NDLM_DEFAULT:
+ case NDLM_STDERR:
+ output = NDLM_FILE;
+ *fpp = stderr;
+ *spinlock = &nd_log.std_error.spinlock;
+ break;
+
+ case NDLM_DISABLED:
+ case NDLM_DEVNULL:
+ output = NDLM_DISABLED;
+ *fpp = NULL;
+ *spinlock = NULL;
+ break;
+ }
+
+ return output;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// high level logger
+
+static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
+ ND_LOG_METHOD output, struct nd_log_source *source,
+ struct log_field *fields, size_t fields_max) {
+ if(spinlock)
+ spinlock_lock(spinlock);
+
+ // check the limits
+ if(limit && nd_log_limit_reached(source))
+ goto cleanup;
+
+ if(output == NDLM_JOURNAL) {
+ if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
+ // we can't log to journal, let's log to stderr
+ if(spinlock)
+ spinlock_unlock(spinlock);
+
+ output = NDLM_FILE;
+ spinlock = &nd_log.std_error.spinlock;
+ fp = stderr;
+
+ if(spinlock)
+ spinlock_lock(spinlock);
+ }
+ }
+
+#if defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+ if(output == NDLM_ETW) {
+ if(!nd_logger_etw(source, fields, fields_max)) {
+ // we can't log to windows events, let's log to stderr
+ if(spinlock)
+ spinlock_unlock(spinlock);
+
+ output = NDLM_FILE;
+ spinlock = &nd_log.std_error.spinlock;
+ fp = stderr;
+
+ if(spinlock)
+ spinlock_lock(spinlock);
+ }
+ }
+#endif
+#if defined(HAVE_WEL)
+ if(output == NDLM_WEL) {
+ if(!nd_logger_wel(source, fields, fields_max)) {
+ // we can't log to windows events, let's log to stderr
+ if(spinlock)
+ spinlock_unlock(spinlock);
+
+ output = NDLM_FILE;
+ spinlock = &nd_log.std_error.spinlock;
+ fp = stderr;
+
+ if(spinlock)
+ spinlock_lock(spinlock);
+ }
+ }
+#endif
+#endif
+
+ if(output == NDLM_SYSLOG)
+ nd_logger_syslog(priority, source->format, fields, fields_max);
+
+ if(output == NDLM_FILE)
+ nd_logger_file(fp, source->format, fields, fields_max);
+
+
+cleanup:
+ if(spinlock)
+ spinlock_unlock(spinlock);
+}
+
+static void nd_logger_unset_all_thread_fields(void) {
+ size_t fields_max = THREAD_FIELDS_MAX;
+ for(size_t i = 0; i < fields_max ; i++)
+ thread_log_fields[i].entry.set = false;
+}
+
+static void nd_logger_merge_log_stack_to_thread_fields(void) {
+ for(size_t c = 0; c < thread_log_stack_next ;c++) {
+ struct log_stack_entry *lgs = thread_log_stack_base[c];
+
+ for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
+ if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
+ continue;
+
+ struct log_stack_entry *e = &lgs[i];
+ ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
+
+ // do not add empty / unset fields
+ if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
+ (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
+ (type == NDFT_STR && !e->str) ||
+ (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
+ (type == NDFT_CALLBACK && !e->cb.formatter) ||
+ type == NDFT_UNSET)
+ continue;
+
+ thread_log_fields[lgs[i].id].entry = *e;
+ }
+ }
+}
+
+static void nd_logger(const char *file, const char *function, const unsigned long line,
+ ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,
+ int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {
+
+ SPINLOCK *spinlock;
+ FILE *fp;
+ ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
+ if(!IS_FINAL_LOG_METHOD(output))
+ return;
+
+ // mark all fields as unset
+ nd_logger_unset_all_thread_fields();
+
+ // flatten the log stack into the fields
+ nd_logger_merge_log_stack_to_thread_fields();
+
+ // set the common fields that are automatically set by the logging subsystem
+
+ if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
+ thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
+
+ if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
+ thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
+ else {
+ ND_LOG_SOURCES src = source;
+
+ if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
+ src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
+ else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
+ src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
+
+ if(src != source && src < _NDLS_MAX) {
+ source = src;
+ output = nd_logger_select_output(source, &fp, &spinlock);
+ if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
+ return;
+ }
+ }
+
+ if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
+ thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
+
+ if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
+ thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
+ thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
+ thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
+ }
+
+ if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
+ thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
+ }
+
+ if(likely(!thread_log_fields[NDF_TID].entry.set))
+ thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached());
+
+ if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
+ const char *thread_tag = nd_thread_tag();
+ thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
+
+ // TODO: fix the ND_MODULE in logging by setting proper module name in threads
+// if(!thread_log_fields[NDF_MODULE].entry.set)
+// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
+ }
+
+ if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
+ thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
+
+ if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
+ thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
+
+ if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)
+ thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);
+
+ CLEAN_BUFFER *wb = NULL;
+ if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
+ wb = buffer_create(1024, NULL);
+ buffer_vsprintf(wb, fmt, ap);
+ thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
+ }
+
+ nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
+ thread_log_fields, THREAD_FIELDS_MAX);
+
+ if(nd_log.sources[source].pending_msg) {
+ // log a pending message
+
+ nd_logger_unset_all_thread_fields();
+
+ thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
+ .set = true,
+ .type = NDFT_U64,
+ .u64 = now_realtime_usec(),
+ };
+
+ thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
+ .set = true,
+ .type = NDFT_TXT,
+ .txt = nd_log_id2source(source),
+ };
+
+ thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
+ .set = true,
+ .type = NDFT_TXT,
+ .txt = program_name,
+ };
+
+ thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
+ .set = true,
+ .type = NDFT_TXT,
+ .txt = nd_log.sources[source].pending_msg,
+ };
+
+ nd_logger_log_fields(spinlock, fp, false, priority, output,
+ &nd_log.sources[source],
+ thread_log_fields, THREAD_FIELDS_MAX);
+
+ freez((void *)nd_log.sources[source].pending_msg);
+ nd_log.sources[source].pending_msg = NULL;
+ }
+
+ errno_clear();
+}
+
+static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
+ if(source >= _NDLS_MAX)
+ source = NDLS_DAEMON;
+
+ if(nd_log.overwrite_process_source)
+ source = nd_log.overwrite_process_source;
+
+ return source;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// public API for loggers
+
+void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )
+{
+ int saved_errno = errno;
+
+ size_t saved_winerror = 0;
+#if defined(OS_WINDOWS)
+ saved_winerror = GetLastError();
+#endif
+
+ source = nd_log_validate_source(source);
+
+ if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
+ return;
+
+ va_list args;
+ va_start(args, fmt);
+ nd_logger(file, function, line, source, priority,
+ source == NDLS_DAEMON || source == NDLS_COLLECTORS,
+ saved_errno, saved_winerror, fmt, args);
+ va_end(args);
+}
+
+void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
+ int saved_errno = errno;
+
+ size_t saved_winerror = 0;
+#if defined(OS_WINDOWS)
+ saved_winerror = GetLastError();
+#endif
+
+ source = nd_log_validate_source(source);
+
+ if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
+ return;
+
+ if(erl->sleep_ut)
+ sleep_usec(erl->sleep_ut);
+
+ spinlock_lock(&erl->spinlock);
+
+ erl->count++;
+ time_t now = now_boottime_sec();
+ if(now - erl->last_logged < erl->log_every) {
+ spinlock_unlock(&erl->spinlock);
+ return;
+ }
+
+ spinlock_unlock(&erl->spinlock);
+
+ va_list args;
+ va_start(args, fmt);
+ nd_logger(file, function, line, source, priority,
+ source == NDLS_DAEMON || source == NDLS_COLLECTORS,
+ saved_errno, saved_winerror, fmt, args);
+ va_end(args);
+ erl->last_logged = now;
+ erl->count = 0;
+}
+
+void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ int saved_errno = errno;
+
+ size_t saved_winerror = 0;
+#if defined(OS_WINDOWS)
+ saved_winerror = GetLastError();
+#endif
+
+ ND_LOG_SOURCES source = NDLS_DAEMON;
+ source = nd_log_validate_source(source);
+
+ va_list args;
+ va_start(args, fmt);
+ nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);
+ va_end(args);
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
+
+ char action_data[70+1];
+ snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
+
+ const char *thread_tag = nd_thread_tag();
+ const char *tag_to_send = thread_tag;
+
+ // anonymize thread names
+ if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
+ tag_to_send = THREAD_TAG_STREAM_RECEIVER;
+ if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
+ tag_to_send = THREAD_TAG_STREAM_SENDER;
+
+ char action_result[60+1];
+ snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
+
+#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
+ int fd = nd_log.sources[NDLS_DAEMON].fd;
+ if(fd == -1)
+ fd = STDERR_FILENO;
+
+ int nptrs;
+ void *buffer[10000];
+
+ nptrs = backtrace(buffer, sizeof(buffer));
+ if(nptrs)
+ backtrace_symbols_fd(buffer, nptrs, fd);
+#endif
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ abort();
+#endif
+
+ netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);
+}
+
diff --git a/src/libnetdata/log/log.h b/src/libnetdata/log/nd_log.h
similarity index 58%
rename from src/libnetdata/log/log.h
rename to src/libnetdata/log/nd_log.h
index 79df798a5f..1e9634d51d 100644
--- a/src/libnetdata/log/log.h
+++ b/src/libnetdata/log/nd_log.h
@@ -1,149 +1,18 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#ifndef NETDATA_LOG_H
-#define NETDATA_LOG_H 1
+#ifndef NETDATA_ND_LOG_H
+#define NETDATA_ND_LOG_H 1
# ifdef __cplusplus
extern "C" {
# endif
#include "../libnetdata.h"
+#include "nd_log-common.h"
#define ND_LOG_DEFAULT_THROTTLE_LOGS 1000
#define ND_LOG_DEFAULT_THROTTLE_PERIOD 60
-typedef enum __attribute__((__packed__)) {
- NDLS_UNSET = 0, // internal use only
- NDLS_ACCESS, // access.log
- NDLS_ACLK, // aclk.log
- NDLS_COLLECTORS, // collector.log
- NDLS_DAEMON, // error.log
- NDLS_HEALTH, // health.log
- NDLS_DEBUG, // debug.log
-
- // terminator
- _NDLS_MAX,
-} ND_LOG_SOURCES;
-
-typedef enum __attribute__((__packed__)) {
- NDLP_EMERG = LOG_EMERG,
- NDLP_ALERT = LOG_ALERT,
- NDLP_CRIT = LOG_CRIT,
- NDLP_ERR = LOG_ERR,
- NDLP_WARNING = LOG_WARNING,
- NDLP_NOTICE = LOG_NOTICE,
- NDLP_INFO = LOG_INFO,
- NDLP_DEBUG = LOG_DEBUG,
-} ND_LOG_FIELD_PRIORITY;
-
-typedef enum __attribute__((__packed__)) {
- // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
- // so that it easy to audit for missing fields
-
- NDF_STOP = 0,
- NDF_TIMESTAMP_REALTIME_USEC, // the timestamp of the log message - added automatically
- NDF_SYSLOG_IDENTIFIER, // the syslog identifier of the application - added automatically
- NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call
- NDF_PRIORITY, // the syslog priority (severity) - set at the log call
- NDF_ERRNO, // the ERRNO at the time of the log call - added automatically
-#if defined(OS_WINDOWS)
- NDF_WINERROR, // Windows GetLastError()
-#endif
- NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically
- NDF_LINE, // the source code file line number - added automatically
- NDF_FILE, // the source code filename - added automatically
- NDF_FUNC, // the source code function - added automatically
- NDF_TID, // the thread ID of the thread logging - added automatically
- NDF_THREAD_TAG, // the thread tag of the thread logging - added automatically
- NDF_MESSAGE_ID, // for specific events
- NDF_MODULE, // for internal plugin module, all other get the NDF_THREAD_TAG
-
- NDF_NIDL_NODE, // the node / rrdhost currently being worked
- NDF_NIDL_INSTANCE, // the instance / rrdset currently being worked
- NDF_NIDL_CONTEXT, // the context of the instance currently being worked
- NDF_NIDL_DIMENSION, // the dimension / rrddim currently being worked
-
- // web server, aclk and stream receiver
- NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd
-
- // Netdata Cloud Related
- NDF_ACCOUNT_ID,
- NDF_USER_NAME,
- NDF_USER_ROLE,
- NDF_USER_ACCESS,
-
- // web server and stream receiver
- NDF_SRC_IP, // the streaming / web server source IP
- NDF_SRC_PORT, // the streaming / web server source Port
- NDF_SRC_FORWARDED_HOST,
- NDF_SRC_FORWARDED_FOR,
- NDF_SRC_CAPABILITIES, // the stream receiver capabilities
-
- // stream sender (established links)
- NDF_DST_TRANSPORT, // the transport we send the request, one of: http, https
- NDF_DST_IP, // the destination streaming IP
- NDF_DST_PORT, // the destination streaming Port
- NDF_DST_CAPABILITIES, // the destination streaming capabilities
-
- // web server, aclk and stream receiver
- NDF_REQUEST_METHOD, // for http like requests, the http request method
- NDF_RESPONSE_CODE, // for http like requests, the http response code, otherwise a status string
-
- // web server (all), aclk (queries)
- NDF_CONNECTION_ID, // the web server connection ID
- NDF_TRANSACTION_ID, // the web server and API transaction ID
- NDF_RESPONSE_SENT_BYTES, // for http like requests, the response bytes
- NDF_RESPONSE_SIZE_BYTES, // for http like requests, the uncompressed response size
- NDF_RESPONSE_PREPARATION_TIME_USEC, // for http like requests, the preparation time
- NDF_RESPONSE_SENT_TIME_USEC, // for http like requests, the time to send the response back
- NDF_RESPONSE_TOTAL_TIME_USEC, // for http like requests, the total time to complete the response
-
- // health alerts
- NDF_ALERT_ID,
- NDF_ALERT_UNIQUE_ID,
- NDF_ALERT_EVENT_ID,
- NDF_ALERT_TRANSITION_ID,
- NDF_ALERT_CONFIG_HASH,
- NDF_ALERT_NAME,
- NDF_ALERT_CLASS,
- NDF_ALERT_COMPONENT,
- NDF_ALERT_TYPE,
- NDF_ALERT_EXEC,
- NDF_ALERT_RECIPIENT,
- NDF_ALERT_DURATION,
- NDF_ALERT_VALUE,
- NDF_ALERT_VALUE_OLD,
- NDF_ALERT_STATUS,
- NDF_ALERT_STATUS_OLD,
- NDF_ALERT_SOURCE,
- NDF_ALERT_UNITS,
- NDF_ALERT_SUMMARY,
- NDF_ALERT_INFO,
- NDF_ALERT_NOTIFICATION_REALTIME_USEC,
- // NDF_ALERT_FLAGS,
-
- // put new items here
- // leave the request URL and the message last
-
- NDF_REQUEST, // the request we are currently working on
- NDF_MESSAGE, // the log message, if any
-
- // terminator
- _NDF_MAX,
-} ND_LOG_FIELD_ID;
-
-typedef enum __attribute__((__packed__)) {
- NDFT_UNSET = 0,
- NDFT_TXT,
- NDFT_STR,
- NDFT_BFR,
- NDFT_U64,
- NDFT_I64,
- NDFT_DBL,
- NDFT_UUID,
- NDFT_CALLBACK,
-} ND_LOG_STACK_FIELD_TYPE;
-
void errno_clear(void);
void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);
void nd_log_set_facility(const char *facility);
@@ -156,7 +25,7 @@ void nd_log_set_flood_protection(size_t logs, time_t period);
void nd_log_initialize_for_external_plugins(const char *name);
void nd_log_reopen_log_files_for_spawn_server(void);
bool nd_log_journal_socket_available(void);
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len);
+ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len);
int nd_log_priority2id(const char *priority);
const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
const char *nd_log_method_for_external_plugins(const char *s);
@@ -306,4 +175,4 @@ void netdata_logger_fatal( const char *file, const char *function, unsigned long
}
# endif
-#endif /* NETDATA_LOG_H */
+#endif /* NETDATA_ND_LOG_H */
diff --git a/src/libnetdata/log/nd_log_limit.c b/src/libnetdata/log/nd_log_limit.c
new file mode 100644
index 0000000000..272138196c
--- /dev/null
+++ b/src/libnetdata/log/nd_log_limit.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "nd_log_limit.h"
+
+void nd_log_limits_reset(void) {
+ usec_t now_ut = now_monotonic_usec();
+
+ spinlock_lock(&nd_log.std_output.spinlock);
+ spinlock_lock(&nd_log.std_error.spinlock);
+
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
+ spinlock_lock(&nd_log.sources[i].spinlock);
+ nd_log.sources[i].limits.prevented = 0;
+ nd_log.sources[i].limits.counter = 0;
+ nd_log.sources[i].limits.started_monotonic_ut = now_ut;
+ nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
+ spinlock_unlock(&nd_log.sources[i].spinlock);
+ }
+
+ spinlock_unlock(&nd_log.std_output.spinlock);
+ spinlock_unlock(&nd_log.std_error.spinlock);
+}
+
+void nd_log_limits_unlimited(void) {
+ nd_log_limits_reset();
+ for(size_t i = 0; i < _NDLS_MAX ;i++) {
+ nd_log.sources[i].limits.logs_per_period = 0;
+ }
+}
+
+bool nd_log_limit_reached(struct nd_log_source *source) {
+ if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
+ return false;
+
+ usec_t now_ut = now_monotonic_usec();
+ if(!source->limits.started_monotonic_ut)
+ source->limits.started_monotonic_ut = now_ut;
+
+ source->limits.counter++;
+
+ if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
+ if(source->limits.prevented) {
+ BUFFER *wb = buffer_create(1024, NULL);
+ buffer_sprintf(wb,
+ "LOG FLOOD PROTECTION: resuming logging "
+ "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
+ source->limits.prevented,
+ source->limits.throttle_period);
+
+ if(source->pending_msg)
+ freez((void *)source->pending_msg);
+
+ source->pending_msg = strdupz(buffer_tostring(wb));
+
+ buffer_free(wb);
+ }
+
+ // restart the period accounting
+ source->limits.started_monotonic_ut = now_ut;
+ source->limits.counter = 1;
+ source->limits.prevented = 0;
+
+ // log this error
+ return false;
+ }
+
+ if(source->limits.counter > source->limits.logs_per_period) {
+ if(!source->limits.prevented) {
+ BUFFER *wb = buffer_create(1024, NULL);
+ buffer_sprintf(wb,
+ "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
+ "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
+ source->limits.counter,
+ (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
+ source->limits.logs_per_period,
+ source->limits.throttle_period,
+ program_name,
+ (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC)
+ );
+
+ if(source->pending_msg)
+ freez((void *)source->pending_msg);
+
+ source->pending_msg = strdupz(buffer_tostring(wb));
+
+ buffer_free(wb);
+ }
+
+ source->limits.prevented++;
+
+ // prevent logging this error
+#ifdef NETDATA_INTERNAL_CHECKS
+ return false;
+#else
+ return true;
+#endif
+ }
+
+ return false;
+}
diff --git a/src/libnetdata/log/nd_log_limit.h b/src/libnetdata/log/nd_log_limit.h
new file mode 100644
index 0000000000..5486abde93
--- /dev/null
+++ b/src/libnetdata/log/nd_log_limit.h
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_ND_LOG_LIMIT_H
+#define NETDATA_ND_LOG_LIMIT_H
+
+#include "../libnetdata.h"
+
+struct nd_log_source;
+bool nd_log_limit_reached(struct nd_log_source *source);
+
+struct nd_log_limit {
+ usec_t started_monotonic_ut;
+ uint32_t counter;
+ uint32_t prevented;
+
+ uint32_t throttle_period;
+ uint32_t logs_per_period;
+ uint32_t logs_per_period_backup;
+};
+
+#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
+#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
+
+#include "nd_log-internals.h"
+
+#endif //NETDATA_ND_LOG_LIMIT_H
diff --git a/src/libnetdata/log/nd_wevents_manifest.xml b/src/libnetdata/log/nd_wevents_manifest.xml
new file mode 100644
index 0000000000..9e326c1cbc
--- /dev/null
+++ b/src/libnetdata/log/nd_wevents_manifest.xml
@@ -0,0 +1,295 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<instrumentationManifest
+ xmlns="http://schemas.microsoft.com/win/2004/08/events"
+ xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <instrumentation>
+ <events>
+
+ <provider name="Netdata"
+ guid="{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
+ symbol="ND_PROVIDER_NAME"
+ messageFileName="%SystemRoot%\System32\nd_wevents.dll"
+ resourceFileName="%SystemRoot%\System32\nd_wevents.dll"
+ parameterFileName="%SystemRoot%\System32\nd_wevents.dll"
+ message="$(string.ND_PROVIDER_NAME)">
+
+ <!-- Define the channels -->
+ <channels>
+ <channel name="Netdata/Daemon"
+ symbol="ND_CHANNEL_DAEMON"
+ type="Operational"/>
+
+ <channel name="Netdata/Collectors"
+ symbol="ND_CHANNEL_COLLECTORS"
+ type="Operational"/>
+
+ <channel name="Netdata/Access"
+ symbol="ND_CHANNEL_ACCESS"
+ type="Operational"/>
+
+ <channel symbol="ND_CHANNEL_HEALTH"
+ name="Netdata/Alerts"
+ type="Operational"/>
+
+ <channel name="Netdata/ACLK"
+ symbol="ND_CHANNEL_ACLK"
+ type="Operational"/>
+ </channels>
+
+ <levels>
+ </levels>
+
+ <opcodes>
+ </opcodes>
+
+ <tasks>
+ <task name="Daemon" value="1" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Daemon)"/>
+ <task name="Collector" value="2" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Collector)"/>
+ <task name="Access" value="3" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Access)"/>
+ <task name="Health" value="4" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Health)"/>
+ <task name="Aclk" value="5" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Aclk)"/>
+ </tasks>
+
+ <templates>
+ <template tid="NetdataLogTemplate">
+ <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->
+ <data name="Timestamp" inType="win:UnicodeString"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->
+ <data name="Program" inType="win:UnicodeString"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->
+ <data name="NetdataLogSource" inType="win:UnicodeString"/> <!-- 3 (NDF_LOG_SOURCE) -->
+ <data name="Level" inType="win:UnicodeString"/> <!-- 4 (NDF_PRIORITY) -->
+ <data name="UnixErrno" inType="win:UnicodeString"/> <!-- 5 (NDF_ERRNO) -->
+ <data name="WindowsLastError" inType="win:UnicodeString"/> <!-- 6 (NDF_WINERROR) -->
+ <data name="InvocationID" inType="win:UnicodeString"/> <!-- 7 (NDF_INVOCATION_ID) -->
+ <data name="CodeLine" inType="win:UInt32"/> <!-- 8 (NDF_LINE) -->
+ <data name="CodeFile" inType="win:UnicodeString"/> <!-- 9 (NDF_FILE) -->
+ <data name="CodeFunction" inType="win:UnicodeString"/> <!-- 10 (NDF_FUNC) -->
+ <data name="ThreadID" inType="win:UInt32"/> <!-- 11 (NDF_TID) -->
+ <data name="ThreadName" inType="win:UnicodeString"/> <!-- 12 (NDF_THREAD_TAG) -->
+ <data name="MessageID" inType="win:UnicodeString"/> <!-- 13 (NDF_MESSAGE_ID) -->
+ <data name="Module" inType="win:UnicodeString"/> <!-- 14 (NDF_MODULE) -->
+ <data name="Node" inType="win:UnicodeString"/> <!-- 15 (NDF_NIDL_NODE) -->
+ <data name="Instance" inType="win:UnicodeString"/> <!-- 16 (NDF_NIDL_INSTANCE) -->
+ <data name="Context" inType="win:UnicodeString"/> <!-- 17 (NDF_NIDL_CONTEXT) -->
+ <data name="Dimension" inType="win:UnicodeString"/> <!-- 18 (NDF_NIDL_DIMENSION) -->
+ <data name="SourceTransport" inType="win:UnicodeString"/> <!-- 19 (NDF_SRC_TRANSPORT) -->
+ <data name="AccountID" inType="win:UnicodeString"/> <!-- 20 (NDF_ACCOUNT_ID) -->
+ <data name="UserName" inType="win:UnicodeString"/> <!-- 21 (NDF_USER_NAME) -->
+ <data name="UserRole" inType="win:UnicodeString"/> <!-- 22 (NDF_USER_ROLE) -->
+ <data name="UserPermissions" inType="win:UnicodeString"/> <!-- 23 (NDF_USER_ACCESS) -->
+ <data name="SourceIP" inType="win:UnicodeString"/> <!-- 24 (NDF_SRC_IP) -->
+ <data name="SourceForwardedHost" inType="win:UnicodeString"/> <!-- 25 (NDF_SRC_PORT) -->
+ <data name="SourceForwardedFor" inType="win:UnicodeString"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->
+ <data name="SourcePort" inType="win:UInt32"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->
+ <data name="SourceCapabilities" inType="win:UnicodeString"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->
+ <data name="DestinationTransport" inType="win:UnicodeString"/> <!-- 29 (NDF_DST_TRANSPORT) -->
+ <data name="DestinationIP" inType="win:UnicodeString"/> <!-- 30 (NDF_DST_IP) -->
+ <data name="DestinationPort" inType="win:UInt32"/> <!-- 31 (NDF_DST_PORT) -->
+ <data name="DestinationCapabilities" inType="win:UnicodeString"/> <!-- 32 (NDF_DST_CAPABILITIES) -->
+ <data name="RequestMethod" inType="win:UnicodeString"/> <!-- 33 (NDF_REQUEST_METHOD) -->
+ <data name="ResponseCode" inType="win:UInt32"/> <!-- 34 (NDF_RESPONSE_CODE) -->
+ <data name="ConnectionID" inType="win:UnicodeString"/> <!-- 35 (NDF_CONNECTION_ID) -->
+ <data name="TransactionID" inType="win:UnicodeString"/> <!-- 36 (NDF_TRANSACTION_ID) -->
+ <data name="ResponseSentBytes" inType="win:UInt64"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->
+ <data name="ResponseSizeBytes" inType="win:UInt64"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->
+ <data name="ResponsePreparationTimeUsec" inType="win:UInt64"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->
+ <data name="ResponseSentTimeUsec" inType="win:UInt64"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->
+ <data name="ResponseTotalTimeUsec" inType="win:UInt64"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->
+ <data name="AlertID" inType="win:UnicodeString"/> <!-- 42 (NDF_ALERT_ID) -->
+ <data name="AlertUniqueID" inType="win:UnicodeString"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->
+ <data name="AlertTransitionID" inType="win:UnicodeString"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->
+ <data name="AlertEventID" inType="win:UnicodeString"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->
+ <data name="AlertConfig" inType="win:UnicodeString"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->
+ <data name="AlertName" inType="win:UnicodeString"/> <!-- 47 (NDF_ALERT_NAME) -->
+ <data name="AlertClass" inType="win:UnicodeString"/> <!-- 48 (NDF_ALERT_CLASS) -->
+ <data name="AlertComponent" inType="win:UnicodeString"/> <!-- 49 (NDF_ALERT_COMPONENT) -->
+ <data name="AlertType" inType="win:UnicodeString"/> <!-- 50 (NDF_ALERT_TYPE) -->
+ <data name="AlertExec" inType="win:UnicodeString"/> <!-- 51 (NDF_ALERT_EXEC) -->
+ <data name="AlertRecipient" inType="win:UnicodeString"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->
+ <data name="AlertDuration" inType="win:UInt64"/> <!-- 53 (NDF_ALERT_DURATION) -->
+ <data name="AlertValue" inType="win:Double"/> <!-- 54 (NDF_ALERT_VALUE) -->
+ <data name="AlertOldValue" inType="win:Double"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->
+ <data name="AlertStatus" inType="win:UnicodeString"/> <!-- 56 (NDF_ALERT_STATUS) -->
+ <data name="AlertOldStatus" inType="win:UnicodeString"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->
+ <data name="Source" inType="win:UnicodeString"/> <!-- 58 (NDF_ALERT_SOURCE) -->
+ <data name="AlertUnits" inType="win:UnicodeString"/> <!-- 59 (NDF_ALERT_UNITS) -->
+ <data name="AlertSummary" inType="win:UnicodeString"/> <!-- 60 (NDF_ALERT_SUMMARY) -->
+ <data name="AlertInfo" inType="win:UnicodeString"/> <!-- 61 (NDF_ALERT_INFO) -->
+ <data name="AlertNotificationTime" inType="win:UInt64"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->
+ <data name="Request" inType="win:UnicodeString"/> <!-- 63 (NDF_REQUEST) -->
+ <data name="Message" inType="win:UnicodeString"/> <!-- 64 (NDF_MESSAGE) -->
+ </template>
+ </templates>
+
+ <events>
+ <!-- Daemon Events -->
+ <event symbol="ND_EVENT_DAEMON_INFO"
+ value="0x1000"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Daemon"
+ level="win:Informational"
+ task="Daemon"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_DAEMON_WARNING"
+ value="0x1001"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Daemon"
+ level="win:Warning"
+ task="Daemon"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_DAEMON_ERROR"
+ value="0x1002"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Daemon"
+ level="win:Error"
+ task="Daemon"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <!-- Collector Events -->
+ <event symbol="ND_EVENT_COLLECTOR_INFO"
+ value="0x2000"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Collectors"
+ level="win:Informational"
+ task="Collector"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_COLLECTOR_WARNING"
+ value="0x2001"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Collectors"
+ level="win:Warning"
+ task="Collector"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_COLLECTOR_ERROR"
+ value="0x2002"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/Collectors"
+ level="win:Error"
+ task="Collector"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <!-- Access Events -->
+ <event symbol="ND_EVENT_ACCESS_INFO"
+ value="0x3000"
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
+ channel="Netdata/Access"
+ level="win:Informational"
+ task="Access"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_ACCESS_WARNING"
+ value="0x3001"
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
+ channel="Netdata/Access"
+ level="win:Warning"
+ task="Access"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_ACCESS_ERROR"
+ value="0x3002"
+ message="$(string.ND_ACCESS_EVENT_MESSAGE)"
+ channel="Netdata/Access"
+ level="win:Error"
+ task="Access"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <!-- Health Events -->
+ <event symbol="ND_EVENT_HEALTH_INFO"
+ value="0x4000"
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
+ channel="Netdata/Alerts"
+ level="win:Informational"
+ task="Health"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_HEALTH_WARNING"
+ value="0x4001"
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
+ channel="Netdata/Alerts"
+ level="win:Warning"
+ task="Health"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_HEALTH_ERROR"
+ value="0x4002"
+ message="$(string.ND_HEALTH_EVENT_MESSAGE)"
+ channel="Netdata/Alerts"
+ level="win:Error"
+ task="Health"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <!-- ACLK Events -->
+ <event symbol="ND_EVENT_ACLK_INFO"
+ value="0x5000"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/ACLK"
+ level="win:Informational"
+ task="Aclk"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_ACLK_WARNING"
+ value="0x5001"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/ACLK"
+ level="win:Warning"
+ task="Aclk"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ <event symbol="ND_EVENT_ACLK_ERROR"
+ value="0x5002"
+ message="$(string.ND_GENERIC_LOG_MESSAGE)"
+ channel="Netdata/ACLK"
+ level="win:Error"
+ task="Aclk"
+ opcode="win:Info"
+ template="NetdataLogTemplate"/>
+
+ </events>
+ </provider>
+ </events>
+ </instrumentation>
+
+ <localization>
+ <resources culture="en-US">
+ <stringTable>
+ <string id="Task.Daemon" value="ND Daemon Log"/>
+ <string id="Task.Collector" value="ND Collector Log"/>
+ <string id="Task.Access" value="ND Access Log"/>
+ <string id="Task.Health" value="ND Health Log"/>
+ <string id="Task.Aclk" value="ND ACLK Log"/>
+
+ <string id="ND_PROVIDER_NAME" value="Netdata"/>
+ <string id="ND_GENERIC_LOG_MESSAGE" value="%64"/>
+ <string id="ND_ACCESS_EVENT_MESSAGE"
+ value="Transaction %36, method: %33, path: %63
+
+ Source IP : %24, Forwarded-For: %27
+ User : %21, role: %22, permissions: %23
+ Timings (usec): prep %39, sent %40, total %41
+ Response Size : sent %37, uncompressed %38
+ Response Code : %34
+"/>
+ <string id="ND_HEALTH_EVENT_MESSAGE"
+ value="Alert '%47' of instance '%16' on node '%15', transitioned from %57 to %56"/>
+ </stringTable>
+ </resources>
+ </localization>
+</instrumentationManifest>
diff --git a/src/libnetdata/log/systemd-cat-native.c b/src/libnetdata/log/systemd-cat-native.c
index e20bc7f481..71d74abdde 100644
--- a/src/libnetdata/log/systemd-cat-native.c
+++ b/src/libnetdata/log/systemd-cat-native.c
@@ -611,7 +611,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
if(equal) {
const char *field = line->buffer;
size_t field_len = equal - line->buffer;
- ND_LOG_FIELD_ID id = nd_log_field_id_by_name(field, field_len);
+ ND_LOG_FIELD_ID id = nd_log_field_id_by_journal_name(field, field_len);
if(id != NDF_STOP) {
const char *value = ++equal;
diff --git a/src/libnetdata/log/journal.c b/src/libnetdata/log/systemd-journal-helpers.c
similarity index 99%
rename from src/libnetdata/log/journal.c
rename to src/libnetdata/log/systemd-journal-helpers.c
index 2182212f6d..24553364b9 100644
--- a/src/libnetdata/log/journal.c
+++ b/src/libnetdata/log/systemd-journal-helpers.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#include "journal.h"
+#include "systemd-journal-helpers.h"
bool is_path_unix_socket(const char *path) {
// Check if the path is valid
diff --git a/src/libnetdata/log/journal.h b/src/libnetdata/log/systemd-journal-helpers.h
similarity index 75%
rename from src/libnetdata/log/journal.h
rename to src/libnetdata/log/systemd-journal-helpers.h
index df8ece18b0..a85f8e85a9 100644
--- a/src/libnetdata/log/journal.h
+++ b/src/libnetdata/log/systemd-journal-helpers.h
@@ -2,8 +2,8 @@
#include "../libnetdata.h"
-#ifndef NETDATA_LOG_JOURNAL_H
-#define NETDATA_LOG_JOURNAL_H
+#ifndef NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
+#define NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
#define JOURNAL_DIRECT_SOCKET "/run/systemd/journal/socket"
@@ -15,4 +15,4 @@ bool journal_direct_send(int fd, const char *msg, size_t msg_len);
bool is_path_unix_socket(const char *path);
bool is_stderr_connected_to_journal(void);
-#endif //NETDATA_LOG_JOURNAL_H
+#endif // NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
diff --git a/src/libnetdata/log/wevt_netdata_compile.bat b/src/libnetdata/log/wevt_netdata_compile.bat
new file mode 100644
index 0000000000..279b6c31b6
--- /dev/null
+++ b/src/libnetdata/log/wevt_netdata_compile.bat
@@ -0,0 +1,121 @@
+@echo off
+setlocal enabledelayedexpansion
+
+echo PATH=%PATH%
+
+if "%~1"=="" (
+ echo Error: Missing .mc file path.
+ goto :usage
+)
+if "%~2"=="" (
+ echo Error: Missing destination directory.
+ goto :usage
+)
+
+REM Set variables
+set "SRC_DIR=%~1"
+set "BIN_DIR=%~2"
+set "MC_FILE=%BIN_DIR%\wevt_netdata.mc"
+set "MAN_FILE=%BIN_DIR%\wevt_netdata_manifest.xml"
+set "BASE_NAME=wevt_netdata"
+set "SDK_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
+set "VS_PATH=C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
+
+if not exist "%SRC_DIR%" (
+ echo Error: Source directory does not exist.
+ exit /b 1
+)
+
+if not exist "%BIN_DIR%" (
+ echo Error: Destination directory does not exist.
+ exit /b 1
+)
+
+if not exist "%MC_FILE%" (
+ echo Error: %MC_FILE% not found.
+ exit /b 1
+)
+
+if not exist "%MAN_FILE%" (
+ echo Error: %MAN_FILE% not found.
+ exit /b 1
+)
+
+REM Add SDK paths to PATH
+set "PATH=C:\Windows\System32;%SDK_PATH%;%VS_PATH%;%PATH%"
+
+REM Check if commands are available
+where mc >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: mc.exe not found in PATH.
+ exit /b 1
+)
+where rc >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: rc.exe not found in PATH.
+ exit /b 1
+)
+where link >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: link.exe not found in PATH.
+ exit /b 1
+)
+where wevtutil >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: wevtutil.exe not found in PATH.
+ exit /b 1
+)
+
+REM Change to the destination directory
+cd /d "%BIN_DIR%"
+
+echo.
+echo Running mc.exe...
+mc -v -b -U "%MC_FILE%" "%MAN_FILE%"
+if %errorlevel% neq 0 (
+ echo Error: mc.exe failed on messages.
+ exit /b 1
+)
+
+if not exist "%BASE_NAME%.rc" (
+ echo Error: %BASE_NAME%.rc not found.
+ exit /b 1
+)
+
+echo.
+echo Modifying %BASE_NAME%.rc to include the manifest...
+copy "%MAN_FILE%" %BASE_NAME%_manifest.man
+echo 1 2004 "%BASE_NAME%_manifest.man" >> %BASE_NAME%.rc
+
+echo.
+echo %BASE_NAME%.rc contents:
+type %BASE_NAME%.rc
+
+echo.
+echo Running rc.exe...
+rc /v /fo %BASE_NAME%.res %BASE_NAME%.rc
+if %errorlevel% neq 0 (
+ echo Error: rc.exe failed.
+ exit /b 1
+)
+
+if not exist "%BASE_NAME%.res" (
+ echo Error: %BASE_NAME%.res not found.
+ exit /b 1
+)
+
+echo.
+echo Running link.exe...
+link /dll /noentry /machine:x64 /out:%BASE_NAME%.dll %BASE_NAME%.res
+if %errorlevel% neq 0 (
+ echo Error: link.exe failed.
+ exit /b 1
+)
+
+echo.
+echo Process completed successfully.
+exit /b 0
+
+:usage
+echo Usage: %~nx0 [path_to_mc_file] [destination_directory]
+exit /b 1
diff --git a/src/libnetdata/log/wevt_netdata_compile.sh b/src/libnetdata/log/wevt_netdata_compile.sh
new file mode 100644
index 0000000000..eae510645d
--- /dev/null
+++ b/src/libnetdata/log/wevt_netdata_compile.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+mylocation=$(dirname "${0}")
+
+# Check if both parameters are provided
+if [ $# -ne 2 ]; then
+ echo "Error: Incorrect number of parameters."
+ echo "Usage: $0 <source_directory> <destination_directory>"
+ exit 1
+fi
+
+# Get the parameters
+src_dir="$1"
+dest_dir="$2"
+
+# Get the directory of this script
+SCRIPT_DIR="$(dirname "$0")"
+
+# Create a temporary batch file
+temp_bat=$(mktemp --suffix=.bat)
+
+# Write the contents to the temporary batch file
+# Use cygpath directly within the heredoc
+cat << EOF > "$temp_bat"
+@echo off
+set "PATH=%SYSTEMROOT%;$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --sdk -w);$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --visualstudio -w)"
+call "$(cygpath -w -a "$SCRIPT_DIR/wevt_netdata_compile.bat")" "$(cygpath -w -a "$src_dir")" "$(cygpath -w -a "$dest_dir")"
+EOF
+
+# Execute the temporary batch file
+echo
+echo "Executing Windows Batch File..."
+echo
+cat "$temp_bat"
+cmd.exe //c "$(cygpath -w -a "$temp_bat")"
+exit_status=$?
+
+# Remove the temporary batch file
+rm "$temp_bat"
+
+# Check the exit status
+if [ $exit_status -eq 0 ]; then
+ echo "nd_wevents_compile.bat executed successfully."
+else
+ echo "nd_wevents_compile.bat failed with exit status $exit_status."
+fi
+
+exit $exit_status
diff --git a/src/libnetdata/log/wevt_netdata_install.bat b/src/libnetdata/log/wevt_netdata_install.bat
new file mode 100644
index 0000000000..5156075927
--- /dev/null
+++ b/src/libnetdata/log/wevt_netdata_install.bat
@@ -0,0 +1,52 @@
+@echo off
+setlocal enabledelayedexpansion
+
+set "MAN_SRC=%~dp0wevt_netdata_manifest.xml"
+set "DLL_SRC=%~dp0wevt_netdata.dll"
+set "DLL_DST=%SystemRoot%\System32\wevt_netdata.dll"
+
+where wevtutil >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: wevtutil.exe not found in PATH.
+ exit /b 1
+)
+
+echo.
+echo Uninstalling previous manifest (if any)...
+wevtutil um "%MAN_SRC%"
+
+echo.
+echo Copying %DLL_SRC% to %DLL_DST%
+copy /y "%DLL_SRC%" "%DLL_DST%"
+if %errorlevel% neq 0 (
+ echo Error: Failed to copy %DLL_SRC% to %DLL_DST%
+ exit /b 1
+)
+
+echo.
+echo Granting access to %DLL_DST% for Windows Event Logging...
+icacls "%DLL_DST%" /grant "NT SERVICE\EventLog":R
+if %errorlevel% neq 0 (
+ echo Error: Failed to grant access to %DLL_DST%.
+ exit /b 1
+)
+
+echo.
+echo Importing the manifest...
+wevtutil im "%MAN_SRC%" /rf:"%DLL_DST%" /mf:"%DLL_DST%"
+if %errorlevel% neq 0 (
+ echo Error: Failed to import the manifest.
+ exit /b 1
+)
+
+echo.
+echo Verifying Netdata Publisher for Event Tracing for Windows (ETW)...
+wevtutil gp "Netdata"
+if %errorlevel% neq 0 (
+ echo Error: Failed to get publisher Netdata.
+ exit /b 1
+)
+
+echo.
+echo Netdata Event Tracing for Windows manifest installed successfully.
+exit /b 0
diff --git a/src/libnetdata/log/wevt_netdata_mc_generate.c b/src/libnetdata/log/wevt_netdata_mc_generate.c
new file mode 100644
index 0000000000..5ab2bdf179
--- /dev/null
+++ b/src/libnetdata/log/wevt_netdata_mc_generate.c
@@ -0,0 +1,518 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdbool.h>
+#include <string.h>
+
+// from winnt.h
+#define EVENTLOG_SUCCESS 0x0000
+#define EVENTLOG_ERROR_TYPE 0x0001
+#define EVENTLOG_WARNING_TYPE 0x0002
+#define EVENTLOG_INFORMATION_TYPE 0x0004
+#define EVENTLOG_AUDIT_SUCCESS 0x0008
+#define EVENTLOG_AUDIT_FAILURE 0x0010
+
+// the severities we define in .mc file
+#define STATUS_SEVERITY_INFORMATIONAL 0x1
+#define STATUS_SEVERITY_WARNING 0x2
+#define STATUS_SEVERITY_ERROR 0x3
+
+#define FACILITY_APPLICATION 0x0fff
+
+#include "nd_log-common.h"
+#include "nd_log-to-windows-common.h"
+
+const char *get_msg_symbol(MESSAGE_ID msg) {
+ switch(msg) {
+ case MSGID_MESSAGE_ONLY:
+ return "MESSAGE_ONLY";
+
+ case MSGID_MESSAGE_ERRNO:
+ return "MESSAGE_ERRNO";
+
+ case MSGID_REQUEST_ONLY:
+ return "REQUEST_ONLY";
+
+ case MSGID_ACCESS_MESSAGE:
+ return "ACCESS_MESSAGE";
+
+ case MSGID_ACCESS_MESSAGE_REQUEST:
+ return "ACCESS_MESSAGE_REQUEST";
+
+ case MSGID_ACCESS_MESSAGE_USER:
+ return "ACCESS_MESSAGE_USER";
+
+ case MSGID_ACCESS:
+ return "ACCESS";
+
+ case MSGID_ACCESS_USER:
+ return "ACCESS_USER";
+
+ case MSGID_ACCESS_FORWARDER:
+ return "ACCESS_FORWARDER";
+
+ case MSGID_ACCESS_FORWARDER_USER:
+ return "ACCESS_FORWARDER_USER";
+
+ case MSGID_ALERT_TRANSITION:
+ return "ALERT_TRANSITION";
+
+ default:
+ fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
+ exit(1);
+ }
+}
+
+const char *get_msg_format(MESSAGE_ID msg) {
+ switch(msg) {
+ case MSGID_MESSAGE_ONLY:
+ return "%2(%12): %64\r\n";
+
+ case MSGID_MESSAGE_ERRNO:
+ return "%2(%12): %64%n\r\n"
+ "%n\r\n"
+ " Unix Errno : %5%n\r\n"
+ " Windows Error: %6%n\r\n"
+ ;
+
+ case MSGID_REQUEST_ONLY:
+ return "%2(%12): %63\r\n";
+
+ case MSGID_ACCESS_MESSAGE:
+ return "%64\r\n";
+
+ case MSGID_ACCESS_MESSAGE_REQUEST:
+ return "%64%n\r\n"
+ "%n\r\n"
+ " Request: %63%n\r\n"
+ ;
+
+ case MSGID_ACCESS_MESSAGE_USER:
+ return "%64%n\r\n"
+ "%n\r\n"
+ " User: %21, role: %22, permissions: %23%n\r\n"
+ ;
+
+ case MSGID_ACCESS:
+ return "%33 %63%n\r\n"
+ "%n\r\n"
+ " Response Code : %34%n\r\n"
+ " Transaction ID: %36%n\r\n"
+ " Source IP : %24%n\r\n"
+ ;
+
+ case MSGID_ACCESS_USER:
+ return "%33 %63%n\r\n"
+ "%n\r\n"
+ " Response Code : %34%n\r\n"
+ " Transaction ID: %36%n\r\n"
+ " Source IP : %24%n\r\n"
+ " User : %21, role: %22, permissions: %23%n\r\n"
+ ;
+
+ case MSGID_ACCESS_FORWARDER:
+ return "%33 %63%n\r\n"
+ "%n\r\n"
+ " Response Code : %34%n\r\n"
+ " Transaction ID: %36%n\r\n"
+ " Source IP : %24, For %27%n\r\n"
+ ;
+
+ case MSGID_ACCESS_FORWARDER_USER:
+ return "%33 %63%n\r\n"
+ "%n\r\n"
+ " Response Code : %34%n\r\n"
+ " Transaction ID: %36%n\r\n"
+ " Source IP : %24, For %27%n\r\n"
+ " User : %21, role: %22, permissions: %23%n\r\n"
+ ;
+
+ case MSGID_ALERT_TRANSITION:
+ return "Alert '%47' of instance '%16' on node '%15' transitioned from %57 to %56\r\n";
+
+ default:
+ fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
+ exit(1);
+ }
+}
+
+int main(int argc, const char **argv) {
+ (void)argc; (void)argv;
+
+ const char *header = NULL, *footer = NULL, *s_header = NULL, *s_footer = NULL;
+
+ bool manifest = false;
+ if(argc == 2 && strcmp(argv[1], "--manifest") == 0) {
+ manifest = true;
+
+ header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
+ "<!--\r\n"
+ "\r\n"
+ " THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
+ "\r\n"
+ " This XML file can be verified by running mc.exe (the MS tool) with this manifest as param.\r\n"
+ "\r\n"
+ " \"c:\\Program Files (x86)\\Windows Kits\\10\\bin\\10.0.26100.0\\x64\\mc.exe\" wevt_netdata_manifest.xml wevt_netdata.mc\r\n"
+ "\r\n"
+ " -->\r\n"
+ "<instrumentationManifest\r\n"
+ " xmlns=\"http://schemas.microsoft.com/win/2004/08/events\"\r\n"
+ " xmlns:win=\"http://manifests.microsoft.com/win/2004/08/windows/events\"\r\n"
+ " xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\r\n"
+ " <instrumentation>\r\n"
+ " <events>\r\n"
+ "\r\n"
+ " <provider name=\"" NETDATA_ETW_PROVIDER_NAME "\"\r\n"
+ " guid=\"" NETDATA_ETW_PROVIDER_GUID_STR "\"\r\n"
+ " symbol=\"NETDATA_ETW_PROVIDER_GUID\"\r\n"
+ " messageFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
+ " resourceFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
+ " message=\"$(string.ND_PROVIDER_NAME)\">\r\n"
+ "\r\n"
+ " <!-- Define the provider sub-channels -->\r\n"
+ " <channels>\r\n"
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON "\"\r\n"
+ " symbol=\"CHANNEL_DAEMON\"\r\n"
+ " type=\"Operational\"\r\n"
+ " message=\"$(string.Channel.Daemon)\"\r\n"
+ " enabled=\"true\"\r\n"
+ " />\r\n"
+ "\r\n"
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS "\"\r\n"
+ " symbol=\"CHANNEL_COLLECTORS\"\r\n"
+ " type=\"Operational\"\r\n"
+ " message=\"$(string.Channel.Collectors)\"\r\n"
+ " enabled=\"true\"\r\n"
+ " />\r\n"
+ "\r\n"
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS "\"\r\n"
+ " symbol=\"CHANNEL_ACCESS\"\r\n"
+ " type=\"Operational\"\r\n"
+ " message=\"$(string.Channel.Access)\"\r\n"
+ " enabled=\"true\"\r\n"
+ " />\r\n"
+ "\r\n"
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH "\"\r\n"
+ " symbol=\"CHANNEL_HEALTH\"\r\n"
+ " type=\"Operational\"\r\n"
+ " message=\"$(string.Channel.Health)\"\r\n"
+ " enabled=\"true\"\r\n"
+ " />\r\n"
+ "\r\n"
+ " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK "\"\r\n"
+ " symbol=\"CHANNEL_ACLK\"\r\n"
+ " type=\"Operational\"\r\n"
+ " message=\"$(string.Channel.Aclk)\"\r\n"
+ " enabled=\"true\"\r\n"
+ " />\r\n"
+ " </channels>\r\n"
+ "\r\n"
+ " <levels>\r\n"
+ " </levels>\r\n"
+ "\r\n"
+ " <opcodes>\r\n"
+ " </opcodes>\r\n"
+ "\r\n"
+ " <tasks>\r\n"
+ " </tasks>\r\n"
+ "\r\n"
+ " <templates>\r\n"
+ " <template tid=\"AllFieldsTemplate\">\r\n"
+ " <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->\r\n"
+ " <data name=\"Timestamp\" inType=\"win:UnicodeString\"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->\r\n"
+ " <data name=\"Program\" inType=\"win:UnicodeString\"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->\r\n"
+ " <data name=\"NetdataLogSource\" inType=\"win:UnicodeString\"/> <!-- 3 (NDF_LOG_SOURCE) -->\r\n"
+ " <data name=\"Level\" inType=\"win:UnicodeString\"/> <!-- 4 (NDF_PRIORITY) -->\r\n"
+ " <data name=\"UnixErrno\" inType=\"win:UnicodeString\"/> <!-- 5 (NDF_ERRNO) -->\r\n"
+ " <data name=\"WindowsLastError\" inType=\"win:UnicodeString\"/> <!-- 6 (NDF_WINERROR) -->\r\n"
+ " <data name=\"InvocationID\" inType=\"win:UnicodeString\"/> <!-- 7 (NDF_INVOCATION_ID) -->\r\n"
+ " <data name=\"CodeLine\" inType=\"win:UnicodeString\"/> <!-- 8 (NDF_LINE) -->\r\n"
+ " <data name=\"CodeFile\" inType=\"win:UnicodeString\"/> <!-- 9 (NDF_FILE) -->\r\n"
+ " <data name=\"CodeFunction\" inType=\"win:UnicodeString\"/> <!-- 10 (NDF_FUNC) -->\r\n"
+ " <data name=\"ThreadID\" inType=\"win:UnicodeString\"/> <!-- 11 (NDF_TID) -->\r\n"
+ " <data name=\"ThreadName\" inType=\"win:UnicodeString\"/> <!-- 12 (NDF_THREAD_TAG) -->\r\n"
+ " <data name=\"MessageID\" inType=\"win:UnicodeString\"/> <!-- 13 (NDF_MESSAGE_ID) -->\r\n"
+ " <data name=\"Module\" inType=\"win:UnicodeString\"/> <!-- 14 (NDF_MODULE) -->\r\n"
+ " <data name=\"Node\" inType=\"win:UnicodeString\"/> <!-- 15 (NDF_NIDL_NODE) -->\r\n"
+ " <data name=\"Instance\" inType=\"win:UnicodeString\"/> <!-- 16 (NDF_NIDL_INSTANCE) -->\r\n"
+ " <data name=\"Context\" inType=\"win:UnicodeString\"/> <!-- 17 (NDF_NIDL_CONTEXT) -->\r\n"
+ " <data name=\"Dimension\" inType=\"win:UnicodeString\"/> <!-- 18 (NDF_NIDL_DIMENSION) -->\r\n"
+ " <data name=\"SourceTransport\" inType=\"win:UnicodeString\"/> <!-- 19 (NDF_SRC_TRANSPORT) -->\r\n"
+ " <data name=\"AccountID\" inType=\"win:UnicodeString\"/> <!-- 20 (NDF_ACCOUNT_ID) -->\r\n"
+ " <data name=\"UserName\" inType=\"win:UnicodeString\"/> <!-- 21 (NDF_USER_NAME) -->\r\n"
+ " <data name=\"UserRole\" inType=\"win:UnicodeString\"/> <!-- 22 (NDF_USER_ROLE) -->\r\n"
+ " <data name=\"UserPermissions\" inType=\"win:UnicodeString\"/> <!-- 23 (NDF_USER_ACCESS) -->\r\n"
+ " <data name=\"SourceIP\" inType=\"win:UnicodeString\"/> <!-- 24 (NDF_SRC_IP) -->\r\n"
+ " <data name=\"SourceForwardedHost\" inType=\"win:UnicodeString\"/> <!-- 25 (NDF_SRC_PORT) -->\r\n"
+ " <data name=\"SourceForwardedFor\" inType=\"win:UnicodeString\"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->\r\n"
+ " <data name=\"SourcePort\" inType=\"win:UnicodeString\"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->\r\n"
+ " <data name=\"SourceCapabilities\" inType=\"win:UnicodeString\"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->\r\n"
+ " <data name=\"DestinationTransport\" inType=\"win:UnicodeString\"/> <!-- 29 (NDF_DST_TRANSPORT) -->\r\n"
+ " <data name=\"DestinationIP\" inType=\"win:UnicodeString\"/> <!-- 30 (NDF_DST_IP) -->\r\n"
+ " <data name=\"DestinationPort\" inType=\"win:UnicodeString\"/> <!-- 31 (NDF_DST_PORT) -->\r\n"
+ " <data name=\"DestinationCapabilities\" inType=\"win:UnicodeString\"/> <!-- 32 (NDF_DST_CAPABILITIES) -->\r\n"
+ " <data name=\"RequestMethod\" inType=\"win:UnicodeString\"/> <!-- 33 (NDF_REQUEST_METHOD) -->\r\n"
+ " <data name=\"ResponseCode\" inType=\"win:UnicodeString\"/> <!-- 34 (NDF_RESPONSE_CODE) -->\r\n"
+ " <data name=\"ConnectionID\" inType=\"win:UnicodeString\"/> <!-- 35 (NDF_CONNECTION_ID) -->\r\n"
+ " <data name=\"TransactionID\" inType=\"win:UnicodeString\"/> <!-- 36 (NDF_TRANSACTION_ID) -->\r\n"
+ " <data name=\"ResponseSentBytes\" inType=\"win:UnicodeString\"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->\r\n"
+ " <data name=\"ResponseSizeBytes\" inType=\"win:UnicodeString\"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->\r\n"
+ " <data name=\"ResponsePreparationTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->\r\n"
+ " <data name=\"ResponseSentTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->\r\n"
+ " <data name=\"ResponseTotalTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->\r\n"
+ " <data name=\"AlertID\" inType=\"win:UnicodeString\"/> <!-- 42 (NDF_ALERT_ID) -->\r\n"
+ " <data name=\"AlertUniqueID\" inType=\"win:UnicodeString\"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->\r\n"
+ " <data name=\"AlertTransitionID\" inType=\"win:UnicodeString\"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->\r\n"
+ " <data name=\"AlertEventID\" inType=\"win:UnicodeString\"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->\r\n"
+ " <data name=\"AlertConfig\" inType=\"win:UnicodeString\"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->\r\n"
+ " <data name=\"AlertName\" inType=\"win:UnicodeString\"/> <!-- 47 (NDF_ALERT_NAME) -->\r\n"
+ " <data name=\"AlertClass\" inType=\"win:UnicodeString\"/> <!-- 48 (NDF_ALERT_CLASS) -->\r\n"
+ " <data name=\"AlertComponent\" inType=\"win:UnicodeString\"/> <!-- 49 (NDF_ALERT_COMPONENT) -->\r\n"
+ " <data name=\"AlertType\" inType=\"win:UnicodeString\"/> <!-- 50 (NDF_ALERT_TYPE) -->\r\n"
+ " <data name=\"AlertExec\" inType=\"win:UnicodeString\"/> <!-- 51 (NDF_ALERT_EXEC) -->\r\n"
+ " <data name=\"AlertRecipient\" inType=\"win:UnicodeString\"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->\r\n"
+ " <data name=\"AlertDuration\" inType=\"win:UnicodeString\"/> <!-- 53 (NDF_ALERT_DURATION) -->\r\n"
+ " <data name=\"AlertValue\" inType=\"win:UnicodeString\"/> <!-- 54 (NDF_ALERT_VALUE) -->\r\n"
+ " <data name=\"AlertOldValue\" inType=\"win:UnicodeString\"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->\r\n"
+ " <data name=\"AlertStatus\" inType=\"win:UnicodeString\"/> <!-- 56 (NDF_ALERT_STATUS) -->\r\n"
+ " <data name=\"AlertOldStatus\" inType=\"win:UnicodeString\"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->\r\n"
+ " <data name=\"Source\" inType=\"win:UnicodeString\"/> <!-- 58 (NDF_ALERT_SOURCE) -->\r\n"
+ " <data name=\"AlertUnits\" inType=\"win:UnicodeString\"/> <!-- 59 (NDF_ALERT_UNITS) -->\r\n"
+ " <data name=\"AlertSummary\" inType=\"win:UnicodeString\"/> <!-- 60 (NDF_ALERT_SUMMARY) -->\r\n"
+ " <data name=\"AlertInfo\" inType=\"win:UnicodeString\"/> <!-- 61 (NDF_ALERT_INFO) -->\r\n"
+ " <data name=\"AlertNotificationTime\" inType=\"win:UnicodeString\"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->\r\n"
+ " <data name=\"Request\" inType=\"win:UnicodeString\"/> <!-- 63 (NDF_REQUEST) -->\r\n"
+ " <data name=\"Message\" inType=\"win:UnicodeString\"/> <!-- 64 (NDF_MESSAGE) -->\r\n"
+ " </template>\r\n"
+ " </templates>\r\n"
+ "\r\n"
+ " <events>\r\n"
+ ;
+
+ footer = " </events>\r\n"
+ " </provider>\r\n"
+ " </events>\r\n"
+ " </instrumentation>\r\n"
+ ;
+
+ s_header = " <localization>\r\n"
+ " <resources culture=\"en-US\">\r\n"
+ " <stringTable>\r\n"
+ " <string id=\"ND_PROVIDER_NAME\" value=\"" NETDATA_ETW_PROVIDER_NAME "\"/>\r\n"
+ "\r\n"
+ " <string id=\"Channel.Daemon\" value=\"Daemon\"/>\r\n"
+ " <string id=\"Channel.Collectors\" value=\"Collectors\"/>\r\n"
+ " <string id=\"Channel.Access\" value=\"Access\"/>\r\n"
+ " <string id=\"Channel.Health\" value=\"Health\"/>\r\n"
+ " <string id=\"Channel.Aclk\" value=\"Aclk\"/>\r\n"
+ "\r\n"
+ ;
+
+ s_footer = " </stringTable>\r\n"
+ " </resources>\r\n"
+ " </localization>\r\n"
+ "</instrumentationManifest>\r\n"
+ ;
+ }
+ else {
+ header = ";// THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
+ "\r\n"
+ "MessageIdTypedef=DWORD\r\n"
+ "\r\n"
+ "SeverityNames=(\r\n"
+ " Informational=0x1:STATUS_SEVERITY_INFORMATIONAL\r\n"
+ " Warning=0x2:STATUS_SEVERITY_WARNING\r\n"
+ " Error=0x3:STATUS_SEVERITY_ERROR\r\n"
+ " )\r\n"
+ "\r\n"
+ "FacilityNames=(\r\n"
+ " " NETDATA_CHANNEL_NAME "=0x0FFF:FACILITY_NETDATA\r\n"
+ " )\r\n"
+ "\r\n"
+ "LanguageNames=(\r\n"
+ " English=0x409:MSG00409\r\n"
+ " )\r\n"
+ "\r\n"
+ ;
+
+ footer = "";
+ }
+
+ bool done[UINT16_MAX] = { 0 };
+ char symbol[1024];
+
+ printf("%s", header);
+ for(size_t src = 1; src < _NDLS_MAX ;src++) {
+ for(size_t pri = 0; pri < _NDLP_MAX ;pri++) {
+ uint8_t severity = get_severity_from_priority(pri);
+
+ for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
+
+ if(src >= 16) {
+ fprintf(stderr, "\n\nSource %zu is bigger than 4 bits!\n\n", src);
+ return 1;
+ }
+
+ if(pri >= 16) {
+ fprintf(stderr, "\n\nPriority %zu is bigger than 4 bits!\n\n", pri);
+ return 1;
+ }
+
+ if(msg >= 256) {
+ fprintf(stderr, "\n\nMessageID %zu is bigger than 8 bits!\n\n", msg);
+ return 1;
+ }
+
+ uint16_t eventID = construct_event_code(src, pri, msg);
+ if((eventID & 0xFFFF) != eventID) {
+ fprintf(stderr, "\n\nEventID 0x%x is bigger than 16 bits!\n\n", eventID);
+ return 1;
+ }
+
+ if(done[eventID]) continue;
+ done[eventID] = true;
+
+ const char *level = get_level_from_priority_str(pri);
+ const char *pri_txt;
+ switch(pri) {
+ case NDLP_EMERG:
+ pri_txt = "EMERG";
+ break;
+
+ case NDLP_CRIT:
+ pri_txt = "CRIT";
+ break;
+
+ case NDLP_ALERT:
+ pri_txt = "ALERT";
+ break;
+
+ case NDLP_ERR:
+ pri_txt = "ERR";
+ break;
+
+ case NDLP_WARNING:
+ pri_txt = "WARN";
+ break;
+
+ case NDLP_INFO:
+ pri_txt = "INFO";
+ break;
+
+ case NDLP_NOTICE:
+ pri_txt = "NOTICE";
+ break;
+
+ case NDLP_DEBUG:
+ pri_txt = "DEBUG";
+ break;
+
+ default:
+ fprintf(stderr, "\n\nInvalid priority %zu!\n\n\n", pri);
+ return 1;
+ }
+
+ const char *channel;
+ const char *src_txt;
+ switch(src) {
+ case NDLS_COLLECTORS:
+ src_txt = "COLLECTORS";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS;
+ break;
+
+ case NDLS_ACCESS:
+ src_txt = "ACCESS";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS;
+ break;
+
+ case NDLS_HEALTH:
+ src_txt = "HEALTH";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH;
+ break;
+
+ case NDLS_DEBUG:
+ src_txt = "DEBUG";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
+ break;
+
+ case NDLS_DAEMON:
+ src_txt = "DAEMON";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
+ break;
+
+ case NDLS_ACLK:
+ src_txt = "ACLK";
+ channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK;
+ break;
+
+ default:
+ fprintf(stderr, "\n\nInvalid source %zu!\n\n\n", src);
+ return 1;
+ }
+
+ const char *msg_txt = get_msg_symbol(msg);
+ const char *format = get_msg_format(msg);
+
+ const char *severity_txt;
+ switch (severity) {
+ case STATUS_SEVERITY_INFORMATIONAL:
+ severity_txt = "Informational";
+ break;
+
+ case STATUS_SEVERITY_ERROR:
+ severity_txt = "Error";
+ break;
+
+ case STATUS_SEVERITY_WARNING:
+ severity_txt = "Warning";
+ break;
+
+ default:
+ fprintf(stderr, "\n\nInvalid severity id %u!\n\n\n", severity);
+ return 1;
+ }
+
+ if(manifest)
+ snprintf(symbol, sizeof(symbol), "ED_%s_%s_%s", src_txt, pri_txt, msg_txt);
+ else
+ snprintf(symbol, sizeof(symbol), "MC_%s_%s_%s", src_txt, pri_txt, msg_txt);
+
+ if(manifest)
+ printf(" <event symbol=\"%s\"\r\n"
+ " value=\"0x%x\"\r\n"
+ " message=\"$(string.msg.MAN_%s)\"\r\n"
+ " channel=\"%s\"\r\n"
+ " level=\"%s\"\r\n"
+ " task=\"win:None\"\r\n"
+ " opcode=\"win:Info\"\r\n"
+ " template=\"AllFieldsTemplate\"/>\r\n\r\n",
+ symbol, eventID, msg_txt, channel, level);
+ else
+ printf("MessageId=0x%x\r\n"
+ "Severity=%s\r\n"
+ "Facility=" NETDATA_CHANNEL_NAME "\r\n"
+ "SymbolicName=%s\r\n"
+ "Language=English\r\n"
+ "%s"
+ ".\r\n"
+ "\r\n",
+ eventID, severity_txt, symbol, format);
+ }
+ }
+ }
+ printf("%s", footer);
+
+ if(s_header) {
+ printf("%s", s_header);
+
+ for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
+ const char *msg_txt = get_msg_symbol(msg);
+ const char *format = get_msg_format(msg);
+ printf(" <string id=\"msg.MAN_%s\" value=\"%s\"/>\r\n", msg_txt, format);
+ }
+
+ printf("%s", s_footer);
+ }
+}
+
diff --git a/src/libnetdata/os/windows-perflib/perflib.c b/src/libnetdata/os/windows-perflib/perflib.c
index 940b3c6e60..947397af98 100644
--- a/src/libnetdata/os/windows-perflib/perflib.c
+++ b/src/libnetdata/os/windows-perflib/perflib.c
@@ -386,44 +386,30 @@ static inline PERF_COUNTER_DEFINITION *getCounterDefinition(PERF_DATA_BLOCK *pDa
// --------------------------------------------------------------------------------------------------------------------
static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodePage, char *start, DWORD length) {
+ static __thread wchar_t unicode[PERFLIB_MAX_NAME_LENGTH];
+
WCHAR *tempBuffer; // Temporary buffer for Unicode data
DWORD charsCopied = 0;
- BOOL free_tempBuffer;
if (CodePage == 0) {
// Input is already Unicode (UTF-16)
tempBuffer = (WCHAR *)start;
charsCopied = length / sizeof(WCHAR); // Convert byte length to number of WCHARs
- free_tempBuffer = FALSE;
}
else {
- // Convert the multi-byte instance name to Unicode (UTF-16)
- // Calculate maximum possible characters in UTF-16
-
- int charCount = MultiByteToWideChar(CodePage, 0, start, (int)length, NULL, 0);
- tempBuffer = (WCHAR *)malloc(charCount * sizeof(WCHAR));
- if (!tempBuffer) return FALSE;
-
- charsCopied = MultiByteToWideChar(CodePage, 0, start, (int)length, tempBuffer, charCount);
- if (charsCopied == 0) {
- free(tempBuffer);
- dst[0] = '\0';
- return FALSE;
- }
-
- free_tempBuffer = TRUE;
+ tempBuffer = unicode;
+ charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length);
+ if(!charsCopied) return FALSE;
}
// Now convert from Unicode (UTF-16) to UTF-8
int bytesCopied = WideCharToMultiByte(CP_UTF8, 0, tempBuffer, (int)charsCopied, dst, (int)dst_len, NULL, NULL);
if (bytesCopied == 0) {
- if (free_tempBuffer) free(tempBuffer);
dst[0] = '\0'; // Ensure the buffer is null-terminated even on failure
return FALSE;
}
- dst[bytesCopied] = '\0'; // Ensure buffer is null-terminated
- if (free_tempBuffer) free(tempBuffer); // Free temporary buffer if used
+ dst[bytesCopied - 1] = '\0'; // Ensure buffer is null-terminated
return TRUE;
}
diff --git a/src/libnetdata/os/windows-perflib/perflib.h b/src/libnetdata/os/windows-perflib/perflib.h
index 0d853edcc3..4394a8a152 100644
--- a/src/libnetdata/os/windows-perflib/perflib.h
+++ b/src/libnetdata/os/windows-perflib/perflib.h
@@ -25,6 +25,7 @@ const char *RegistryFindNameByID(DWORD id);
const char *RegistryFindHelpByID(DWORD id);
DWORD RegistryFindIDByName(const char *name);
#define PERFLIB_REGISTRY_NAME_NOT_FOUND (DWORD)-1
+#define PERFLIB_MAX_NAME_LENGTH 1024
PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id);
void perflibFreePerformanceData(void);
diff --git a/src/libnetdata/spawn_server/log-forwarder.c b/src/libnetdata/spawn_server/log-forwarder.c
new file mode 100644
index 0000000000..5c4db55ea0
--- /dev/null
+++ b/src/libnetdata/spawn_server/log-forwarder.c
@@ -0,0 +1,322 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+#include "log-forwarder.h"
+
+typedef struct LOG_FORWARDER_ENTRY {
+ int fd;
+ char *cmd;
+ pid_t pid;
+ BUFFER *wb;
+ size_t pfds_idx;
+ bool delete;
+
+ struct LOG_FORWARDER_ENTRY *prev;
+ struct LOG_FORWARDER_ENTRY *next;
+} LOG_FORWARDER_ENTRY;
+
+typedef struct LOG_FORWARDER {
+ LOG_FORWARDER_ENTRY *entries;
+ ND_THREAD *thread;
+ SPINLOCK spinlock;
+ int pipe_fds[2]; // Pipe for notifications
+ bool running;
+} LOG_FORWARDER;
+
+static void *log_forwarder_thread_func(void *arg);
+
+// --------------------------------------------------------------------------------------------------------------------
+// helper functions
+
+static inline LOG_FORWARDER_ENTRY *log_forwarder_find_entry_unsafe(LOG_FORWARDER *lf, int fd) {
+ for (LOG_FORWARDER_ENTRY *entry = lf->entries; entry; entry = entry->next) {
+ if (entry->fd == fd)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static inline void log_forwarder_del_entry_unsafe(LOG_FORWARDER *lf, LOG_FORWARDER_ENTRY *entry) {
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(lf->entries, entry, prev, next);
+ buffer_free(entry->wb);
+ freez(entry->cmd);
+ close(entry->fd);
+ freez(entry);
+}
+
+static inline void log_forwarder_wake_up_worker(LOG_FORWARDER *lf) {
+ char ch = 0;
+ ssize_t bytes_written = write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
+ if (bytes_written != 1)
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to write to notification pipe");
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// starting / stopping
+
+LOG_FORWARDER *log_forwarder_start(void) {
+ LOG_FORWARDER *lf = callocz(1, sizeof(LOG_FORWARDER));
+
+ spinlock_init(&lf->spinlock);
+ if (pipe(lf->pipe_fds) != 0) {
+ freez(lf);
+ return NULL;
+ }
+
+ // make sure read() will not block on this pipe
+ sock_setnonblock(lf->pipe_fds[PIPE_READ]);
+
+ lf->running = true;
+ lf->thread = nd_thread_create("log-fw", NETDATA_THREAD_OPTION_JOINABLE, log_forwarder_thread_func, lf);
+
+ return lf;
+}
+
+static inline void mark_all_entries_for_deletion_unsafe(LOG_FORWARDER *lf) {
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ;entry = entry->next)
+ entry->delete = true;
+}
+
+void log_forwarder_stop(LOG_FORWARDER *lf) {
+ if(!lf || !lf->running) return;
+
+ // Signal the thread to stop
+ spinlock_lock(&lf->spinlock);
+ lf->running = false;
+
+ // mark them all for deletion
+ mark_all_entries_for_deletion_unsafe(lf);
+
+ // Send a byte to the pipe to wake up the thread
+ char ch = 0;
+ write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
+ spinlock_unlock(&lf->spinlock);
+
+ // Wait for the thread to finish
+ close(lf->pipe_fds[PIPE_WRITE]); // force it to quit
+ nd_thread_join(lf->thread);
+ close(lf->pipe_fds[PIPE_READ]);
+
+ freez(lf);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// managing entries
+
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd) {
+ if(!lf || !lf->running || fd < 0) return;
+
+ LOG_FORWARDER_ENTRY *entry = callocz(1, sizeof(LOG_FORWARDER_ENTRY));
+ entry->fd = fd;
+ entry->cmd = NULL;
+ entry->pid = 0;
+ entry->pfds_idx = 0;
+ entry->delete = false;
+ entry->wb = buffer_create(0, NULL);
+
+ spinlock_lock(&lf->spinlock);
+
+ // Append to the entries list
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(lf->entries, entry, prev, next);
+
+ // Send a byte to the pipe to wake up the thread
+ log_forwarder_wake_up_worker(lf);
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd) {
+ if(!lf || !lf->running || fd < 0) return false;
+
+ bool ret = false;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if(entry) {
+ entry->delete = true;
+
+ // Send a byte to the pipe to wake up the thread
+ log_forwarder_wake_up_worker(lf);
+
+ ret = true;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+
+ return ret;
+}
+
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd) {
+ if(!lf || !lf->running || fd < 0 || !cmd || !*cmd) return;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if (entry) {
+ freez(entry->cmd);
+ entry->cmd = strdupz(cmd);
+ }
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid) {
+ if(!lf || !lf->running || fd < 0) return;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if (entry)
+ entry->pid = pid;
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// log forwarder thread
+
+static inline void log_forwarder_log(LOG_FORWARDER *lf __maybe_unused, LOG_FORWARDER_ENTRY *entry, const char *msg) {
+ const char *s = msg;
+ while(*s && isspace((uint8_t)*s)) s++;
+ if(*s == '\0') return; // do not log empty lines
+
+ ND_LOG_STACK lgs[] = {
+ ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, entry->cmd ? entry->cmd : "unknown"),
+ ND_LOG_FIELD_I64(NDF_TID, entry->pid),
+ ND_LOG_FIELD_END(),
+ };
+ ND_LOG_STACK_PUSH(lgs);
+
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "STDERR: %s", msg);
+}
+
+// returns the number of entries active
+static inline size_t log_forwarder_remove_deleted_unsafe(LOG_FORWARDER *lf) {
+ size_t entries = 0;
+
+ LOG_FORWARDER_ENTRY *entry = lf->entries;
+ while(entry) {
+ LOG_FORWARDER_ENTRY *next = entry->next;
+
+ if(entry->delete) {
+ if (buffer_strlen(entry->wb))
+ // there is something not logged in it - log it
+ log_forwarder_log(lf, entry, buffer_tostring(entry->wb));
+
+ log_forwarder_del_entry_unsafe(lf, entry);
+ }
+ else
+ entries++;
+
+ entry = next;
+ }
+
+ return entries;
+}
+
+static void *log_forwarder_thread_func(void *arg) {
+ LOG_FORWARDER *lf = (LOG_FORWARDER *)arg;
+
+ while (1) {
+ spinlock_lock(&lf->spinlock);
+ if (!lf->running) {
+ mark_all_entries_for_deletion_unsafe(lf);
+ log_forwarder_remove_deleted_unsafe(lf);
+ spinlock_unlock(&lf->spinlock);
+ break;
+ }
+
+ // Count the number of fds
+ size_t nfds = 1 + log_forwarder_remove_deleted_unsafe(lf);
+
+ struct pollfd pfds[nfds];
+
+ // First, the notification pipe
+ pfds[0].fd = lf->pipe_fds[PIPE_READ];
+ pfds[0].events = POLLIN;
+
+ int idx = 1;
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next, idx++) {
+ pfds[idx].fd = entry->fd;
+ pfds[idx].events = POLLIN;
+ entry->pfds_idx = idx;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+
+ int timeout = 200; // 200ms
+ int ret = poll(pfds, nfds, timeout);
+
+ if (ret > 0) {
+ // Check the notification pipe
+ if (pfds[0].revents & POLLIN) {
+ // Read and discard the data
+ char buf[256];
+ ssize_t bytes_read = read(lf->pipe_fds[PIPE_READ], buf, sizeof(buf));
+ // Ignore the data; proceed regardless of the result
+ if (bytes_read == -1) {
+ if (errno != EAGAIN && errno != EWOULDBLOCK) {
+ // Handle read error if necessary
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to read from notification pipe");
+ return NULL;
+ }
+ }
+ }
+
+ // Now check the other fds
+ spinlock_lock(&lf->spinlock);
+
+ size_t to_remove = 0;
+
+ // read or mark them for deletion
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next) {
+ if (entry->pfds_idx < 1 || entry->pfds_idx >= nfds || !(pfds[entry->pfds_idx].revents & POLLIN))
+ continue;
+
+ BUFFER *wb = entry->wb;
+ buffer_need_bytes(wb, 1024);
+
+ ssize_t bytes_read = read(entry->fd, &wb->buffer[wb->len], wb->size - wb->len - 1);
+ if(bytes_read > 0)
+ wb->len += bytes_read;
+ else if(bytes_read == 0 || (bytes_read == -1 && errno != EINTR && errno != EAGAIN)) {
+ // EOF or error
+ entry->delete = true;
+ to_remove++;
+ }
+
+ // log as many lines are they have been received
+ char *start = (char *)buffer_tostring(wb);
+ char *newline = strchr(start, '\n');
+ while(newline) {
+ *newline = '\0';
+ log_forwarder_log(lf, entry, start);
+
+ start = ++newline;
+ newline = strchr(newline, '\n');
+ }
+
+ if(start != wb->buffer) {
+ wb->len = strlen(start);
+ if (wb->len)
+ memmove(wb->buffer, start, wb->len);
+ }
+
+ entry->pfds_idx = 0;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+ }
+ else if (ret == 0) {
+ // Timeout, nothing to do
+ continue;
+
+ }
+ else
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Log forwarder: poll() error");
+ }
+
+ return NULL;
+}
diff --git a/src/libnetdata/spawn_server/log-forwarder.h b/src/libnetdata/spawn_server/log-forwarder.h
new file mode 100644
index 0000000000..344601c1f0
--- /dev/null
+++ b/src/libnetdata/spawn_server/log-forwarder.h
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOG_FORWARDER_H
+#define NETDATA_LOG_FORWARDER_H
+
+#include "../libnetdata.h"
+
+typedef struct LOG_FORWARDER LOG_FORWARDER;
+
+LOG_FORWARDER *log_forwarder_start(void); // done once, at spawn_server_create()
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd); // to add a new fd
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd); // set the syslog identifier
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid); // set the pid of the child process
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd); // to remove an fd
+void log_forwarder_stop(LOG_FORWARDER *lf); // done once, at spawn_server_destroy()
+
+#endif //NETDATA_LOG_FORWARDER_H
diff --git a/src/libnetdata/spawn_server/spawn_server_internals.h b/src/libnetdata/spawn_server/spawn_server_internals.h
index 9802f0771e..198442ae7b 100644
--- a/src/libnetdata/spawn_server/spawn_server_internals.h
+++ b/src/libnetdata/spawn_server/spawn_server_internals.h
@@ -6,6 +6,7 @@
#include "../libnetdata.h"
#include "spawn_server.h"
#include "spawn_library.h"
+#include "log-forwarder.h"
#if defined(OS_WINDOWS)
#define SPAWN_SERVER_VERSION_WINDOWS 1
@@ -62,6 +63,7 @@ struct spawn_server {
#endif
#if defined(SPAWN_SERVER_VERSION_WINDOWS)
+ LOG_FORWARDER *log_forwarder;
#endif
};
@@ -70,6 +72,7 @@ struct spawn_instance {
int sock;
int write_fd;
int read_fd;
+ int stderr_fd;
pid_t child_pid;
#if defined(SPAWN_SERVER_VERSION_UV)
diff --git a/src/libnetdata/spawn_server/spawn_server_nofork.c b/src/libnetdata/spawn_server/spawn_server_nofork.c
index f345561b45..be060fc7d8 100644
--- a/src/libnetdata/spawn_server/spawn_server_nofork.c
+++ b/src/libnetdata/spawn_server/spawn_server_nofork.c
@@ -59,6 +59,7 @@ static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *rq) {
// close all open file descriptors of the parent, but keep ours
os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
+ nd_log_reopen_log_files_for_spawn_server();
// set the process name
os_setproctitle("spawn-child", server->argc, server->argv);
@@ -363,7 +364,6 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN
else if (pid == 0) {
// the child
- gettid_uncached(); // make sure the logger logs valid pids
spawn_server_run_child(server, rq);
exit(63);
}
@@ -984,22 +984,24 @@ static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) {
}
static void replace_stdio_with_dev_null() {
+ // we cannot log in this function - the logger is not yet initialized after fork()
+
int dev_null_fd = open("/dev/null", O_RDWR);
if (dev_null_fd == -1) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
return;
}
// Redirect stdin (fd 0)
if (dup2(dev_null_fd, STDIN_FILENO) == -1) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
close(dev_null_fd);
return;
}
// Redirect stdout (fd 1)
if (dup2(dev_null_fd, STDOUT_FILENO) == -1) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
+ // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
close(dev_null_fd);
return;
}
@@ -1070,7 +1072,6 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
pid_t pid = fork();
if (pid == 0) {
// the child - the spawn server
- gettid_uncached(); // make sure the logger logs valid pids
{
char buf[15];
diff --git a/src/libnetdata/spawn_server/spawn_server_windows.c b/src/libnetdata/spawn_server/spawn_server_windows.c
index 8c7d76cd24..f80925a24b 100644
--- a/src/libnetdata/spawn_server/spawn_server_windows.c
+++ b/src/libnetdata/spawn_server/spawn_server_windows.c
@@ -40,11 +40,18 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, c
server->name = strdupz(name);
else
server->name = strdupz("unnamed");
+
+ server->log_forwarder = log_forwarder_start();
+
return server;
}
void spawn_server_destroy(SPAWN_SERVER *server) {
if (server) {
+ if (server->log_forwarder) {
+ log_forwarder_stop(server->log_forwarder);
+ server->log_forwarder = NULL;
+ }
freez((void *)server->name);
freez(server);
}
@@ -136,13 +143,13 @@ int set_fd_blocking(int fd) {
// }
//}
-SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
+SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd __maybe_unused, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
if (type != SPAWN_INSTANCE_TYPE_EXEC)
return NULL;
- int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 };
+ int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 }, pipe_stderr[2] = { -1, -1 };
errno_clear();
@@ -166,12 +173,21 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
goto cleanup;
}
+ if (pipe(pipe_stderr) == -1) {
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ "SPAWN PARENT: Cannot create stderr pipe() for request No %zu, command: %s",
+ instance->request_id, command);
+ goto cleanup;
+ }
+
// Ensure pipes are in blocking mode
if (set_fd_blocking(pipe_stdin[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdin[PIPE_WRITE]) == -1 ||
- set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1) {
+ set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1 ||
+ set_fd_blocking(pipe_stderr[PIPE_READ]) == -1 || set_fd_blocking(pipe_stderr[PIPE_WRITE]) == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Failed to set blocking I/O on pipes for request No %zu, command: %s",
instance->request_id, command);
+ goto cleanup;
}
// do not run multiple times this section
@@ -181,9 +197,9 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Convert POSIX file descriptors to Windows handles
HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[PIPE_READ]);
HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[PIPE_WRITE]);
- HANDLE stderr_handle = (HANDLE)_get_osfhandle(stderr_fd);
+ HANDLE stderr_write_handle = (HANDLE)_get_osfhandle(pipe_stderr[PIPE_WRITE]);
- if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_handle == INVALID_HANDLE_VALUE) {
+ if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_write_handle == INVALID_HANDLE_VALUE) {
spinlock_unlock(&spinlock);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s",
@@ -194,7 +210,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Set handle inheritance
if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
!SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
- !SetHandleInformation(stderr_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
+ !SetHandleInformation(stderr_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
spinlock_unlock(&spinlock);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s",
@@ -210,18 +226,18 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdInput = stdin_read_handle;
si.hStdOutput = stdout_write_handle;
- si.hStdError = stderr_handle;
+ si.hStdError = stderr_write_handle;
// Retrieve the current environment block
char* env_block = GetEnvironmentStrings();
// print_environment_block(env_block);
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: Running request No %zu, command: '%s'",
instance->request_id, command);
- int fds[3] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], stderr_fd };
- os_close_all_non_std_open_fds_except(fds, 3, CLOSE_RANGE_CLOEXEC);
+ int fds_to_keep_open[] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], pipe_stderr[PIPE_WRITE] };
+ os_close_all_non_std_open_fds_except(fds_to_keep_open, 3, CLOSE_RANGE_CLOEXEC);
// Spawn the process
errno_clear();
@@ -247,6 +263,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Close unused pipe ends
close(pipe_stdin[PIPE_READ]); pipe_stdin[PIPE_READ] = -1;
close(pipe_stdout[PIPE_WRITE]); pipe_stdout[PIPE_WRITE] = -1;
+ close(pipe_stderr[PIPE_WRITE]); pipe_stderr[PIPE_WRITE] = -1;
// Store process information in instance
instance->dwProcessId = pi.dwProcessId;
@@ -256,9 +273,15 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Convert handles to POSIX file descriptors
instance->write_fd = pipe_stdin[PIPE_WRITE];
instance->read_fd = pipe_stdout[PIPE_READ];
+ instance->stderr_fd = pipe_stderr[PIPE_READ];
+
+ // Add stderr_fd to the log forwarder
+ log_forwarder_add_fd(server->log_forwarder, instance->stderr_fd);
+ log_forwarder_annotate_fd_name(server->log_forwarder, instance->stderr_fd, command);
+ log_forwarder_annotate_fd_pid(server->log_forwarder, instance->stderr_fd, spawn_server_instance_pid(instance));
errno_clear();
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: created process for request No %zu, pid %d (winpid %d), command: %s",
instance->request_id, (int)instance->child_pid, (int)pi.dwProcessId, command);
@@ -269,6 +292,8 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
if (pipe_stdin[PIPE_WRITE] >= 0) close(pipe_stdin[PIPE_WRITE]);
if (pipe_stdout[PIPE_READ] >= 0) close(pipe_stdout[PIPE_READ]);
if (pipe_stdout[PIPE_WRITE] >= 0) close(pipe_stdout[PIPE_WRITE]);
+ if (pipe_stderr[PIPE_READ] >= 0) close(pipe_stderr[PIPE_READ]);
+ if (pipe_stderr[PIPE_WRITE] >= 0) close(pipe_stderr[PIPE_WRITE]);
freez(instance);
return NULL;
}
@@ -314,7 +339,7 @@ static void TerminateChildProcesses(SPAWN_INSTANCE *si) {
if (pe.th32ParentProcessID == si->dwProcessId) {
HANDLE hChildProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pe.th32ProcessID);
if (hChildProcess) {
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING,
"SPAWN PARENT: killing subprocess %u of request No %zu, pid %d (winpid %u)",
pe.th32ProcessID, si->request_id, (int)si->child_pid, si->dwProcessId);
@@ -378,6 +403,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
// to have them, to avoid abnormal shutdown of the plugins
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
+ if(si->stderr_fd != -1) {
+ if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
+ close(si->stderr_fd);
+
+ si->stderr_fd = -1;
+ }
errno_clear();
if(TerminateProcess(si->process_handle, STATUS_CONTROL_C_EXIT) == 0)
@@ -394,6 +425,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) {
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
+ if(si->stderr_fd != -1) {
+ if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
+ close(si->stderr_fd);
+
+ si->stderr_fd = -1;
+ }
// wait for the process to end
WaitForSingleObject(si->process_handle, INFINITE);
@@ -404,7 +441,7 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
char *err = GetErrorString(exit_code);
- nd_log(NDLS_COLLECTORS, NDLP_ERR,
+ nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: child of request No %zu, pid %d (winpid %u), exited with code %u (0x%x): %s",
si->request_id, (int)si->child_pid, si->dwProcessId,
(unsigned)exit_code, (unsigned)exit_code, err ? err : "(no reason text)");
diff --git a/src/libnetdata/string/utf8.c b/src/libnetdata/string/utf8.c
new file mode 100644
index 0000000000..e0a6745da4
--- /dev/null
+++ b/src/libnetdata/string/utf8.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#if defined(OS_WINDOWS)
+#include <windows.h>
+
+/*
+ * Convert any CodePage to UTF16
+ * Goals:
+ * 1. Destination is always NULL terminated
+ * 2. If the destination buffer is not enough, return as much as possible data (truncate)
+ * 3. Always return the number of wide characters written, including the null terminator
+ */
+
+size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len) {
+ if(!src || src_len == 0) {
+ // invalid input
+ if(dst && dst_size)
+ *dst = L'\0';
+ return 0;
+ }
+
+ if(!dst || !dst_size) {
+ // the caller wants to know the buffer to allocate for the conversion
+ int required = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
+ if(required <= 0) return 0; // error in the conversion
+
+ // Add 1 for null terminator only if src_len is not -1
+ // so that the caller can call us again to get the entire string (not truncated)
+ return (size_t)required + ((src_len != -1) ? 1 : 0);
+ }
+
+ // do the conversion directly to the destination buffer
+ int rc = MultiByteToWideChar(CodePage, 0, src, src_len, dst, (int)dst_size);
+ if(rc <= 0) {
+ // conversion failed, let's see why...
+ DWORD status = GetLastError();
+ if(status == ERROR_INSUFFICIENT_BUFFER) {
+ // it cannot fit entirely, let's allocate a new buffer to convert it
+ // and then truncate it to the destination buffer
+
+ // clear errno and LastError to clear the error of the
+ // MultiByteToWideChar() that failed
+ errno_clear();
+
+ // get the required size
+ int required_size = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
+
+ // mallocz() never fails (exits the program on NULL)
+ wchar_t *tmp = mallocz(required_size * sizeof(wchar_t));
+
+ // convert it, now it should fit
+ rc = MultiByteToWideChar(CodePage, 0, src, src_len, tmp, required_size);
+ if (rc <= 0) {
+ // it failed!
+ *dst = L'\0';
+ freez(tmp);
+ return 0;
+ }
+
+ size_t len = rc;
+
+ // copy as much as we can
+ memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(wchar_t));
+
+ // null terminate it
+ dst[MIN(len, (dst_size - 1))] = L'\0';
+
+ // free the temporary buffer
+ freez(tmp);
+
+ // return the actual bytes written
+ return MIN(len, dst_size);
+ }
+
+ // empty the destination
+ *dst = L'\0';
+ return 0;
+ }
+
+ size_t len = rc;
+
+ if(len >= dst_size) {
+ // truncate it to fit the null
+ dst[dst_size - 1] = L'\0';
+ return dst_size;
+ }
+
+ if(dst[len - 1] != L'\0') {
+ // the result is not null terminated
+ // append the null
+ dst[len] = L'\0';
+ return len + 1;
+ }
+
+ // the result is already null terminated
+ return len;
+}
+
+/*
+ * Convert UTF16 (wide-character string) to UTF8
+ * Goals:
+ * 1. Destination is always NULL terminated
+ * 2. If the destination buffer is not enough, return as much as possible data (truncate)
+ * 3. Always return the number of bytes written, including the null terminator
+ */
+
+size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len) {
+ if (!src || src_len == 0) {
+ // invalid input
+ if(dst && dst_size)
+ *dst = L'\0';
+ return 0;
+ }
+
+ if (!dst || dst_size == 0) {
+ // The caller wants to know the buffer size required for the conversion
+ int required = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
+ if (required <= 0) return 0; // error in the conversion
+
+ // Add 1 for null terminator only if src_len is not -1
+ return (size_t)required + ((src_len != -1) ? 1 : 0);
+ }
+
+ // Perform the conversion directly into the destination buffer
+ int rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, dst, (int)dst_size, NULL, NULL);
+ if (rc <= 0) {
+ // Conversion failed, let's see why...
+ DWORD status = GetLastError();
+ if (status == ERROR_INSUFFICIENT_BUFFER) {
+ // It cannot fit entirely, let's allocate a new buffer to convert it
+ // and then truncate it to the destination buffer
+
+ // Clear errno and LastError to clear the error of the
+ // WideCharToMultiByte() that failed
+ errno_clear();
+
+ // Get the required size
+ int required_size = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
+
+ // mallocz() never fails (exits the program on NULL)
+ char *tmp = mallocz(required_size * sizeof(char));
+
+ // Convert it, now it should fit
+ rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, tmp, required_size, NULL, NULL);
+ if (rc <= 0) {
+ // Conversion failed
+ *dst = '\0';
+ freez(tmp);
+ return 0;
+ }
+
+ size_t len = rc;
+
+ // Copy as much as we can
+ memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(char));
+
+ // Null-terminate it
+ dst[MIN(len, (dst_size - 1))] = '\0';
+
+ // Free the temporary buffer
+ freez(tmp);
+
+ // Return the actual bytes written
+ return MIN(len, dst_size);
+ }
+
+ // Empty the destination
+ *dst = '\0';
+ return 0;
+ }
+
+ size_t len = rc;
+
+ if (len >= dst_size) {
+ // Truncate it to fit the null terminator
+ dst[dst_size - 1] = '\0';
+ return dst_size;
+ }
+
+ if (dst[len - 1] != '\0') {
+ // The result is not null-terminated
+ // Append the null terminator
+ dst[len] = '\0';
+ return len + 1;
+ }
+
+ // The result is already null-terminated
+ return len;
+}
+#endif
diff --git a/src/libnetdata/string/utf8.h b/src/libnetdata/string/utf8.h
index 6d934b543d..ee9fe12b20 100644
--- a/src/libnetdata/string/utf8.h
+++ b/src/libnetdata/string/utf8.h
@@ -6,4 +6,25 @@
#define IS_UTF8_BYTE(x) ((uint8_t)(x) & (uint8_t)0x80)
#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x) && ((uint8_t)(x) & (uint8_t)0x40))
+#ifndef _countof
+#define _countof(x) (sizeof(x) / sizeof(*(x)))
+#endif
+
+#if defined(OS_WINDOWS)
+
+// return an always null terminated wide string, truncate to given size if destination is not big enough,
+// src_len can be -1 use all of it.
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
+size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len);
+
+// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
+#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len)
+
+// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
+// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
+size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len);
+
+#endif
+
#endif /* NETDATA_STRING_UTF8_H */
diff --git a/src/libnetdata/template-enum.h b/src/libnetdata/template-enum.h
index 82487336ac..2170ee86ba 100644
--- a/src/libnetdata/template-enum.h
+++ b/src/libnetdata/template-enum.h
@@ -41,6 +41,7 @@
#define BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(type) \
type type ## _2id_one(const char *str); \
+ const char *type##_2str_one(type id); \
const char *type##_2json(BUFFER *wb, const char *key, type id);
#define BITMAP_STR_DEFINE_FUNCTIONS(type, def, def_str) \
@@ -57,6 +58,16 @@
return def; \
} \
\
+ const char *type##_2str_one(type id) \
+ { \
+ for (size_t i = 0; type ## _names[i].name; i++) { \
+ if (id == type ## _names[i].id) \
+ return type ## _names[i].name; \
+ } \
+ \
+ return def_str; \
+ } \
+ \
const char *type##_2json(BUFFER *wb, const char *key, type id) \
{ \
buffer_json_member_add_array(wb, key); \