archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-02 20:48:06 +00:00
Code Issues Projects Releases Wiki Activity

Log to windows ()

Browse source 
* split netdata logger into multiple files - no acctual code changes

* move around some more code

* base for implementing windows events logging

* fix for the last commit

* working logging to windows events, but not pretty yet

* fix compilation on linux

* added scripts for compiling the resource file and importing the manifest

* added validation that the provider is available

* working manifest for ETW (Event Tracing for Windows)

* compile the messages dll with msys tools

* handle wevents configuration

* when starting under clion, do not start as service

* unify conversion to utf16

* fix bug in windows-events.plugin that was incorrectly not processing right the publishers that do not have a UUID

* enable wevents as default logging for all methods, under windows

* log to windows using EventCreate.exe for the messages

* do not log all the fields

* added log-forwarder to spawn-server-windows

* fix last character being cut-off when converting from utf-16

* updated info

* updated any_to_utf16() to be always consistent

* added utf16_to_utf8()

* external plugins inherit windows events

* fix wrong log source

* fix spawn server logs

* log to multiple event log sources

* generate custom messages dll for event viewer - working

* removed debugging code

* cleanup log forwarder entries from the thread, to avoid bad file descriptor in poll()

* .mc and its manifest are automatically generated

* sanitizers should not remove trailing underscores

* use the resources dll for the netdata directory; set the default maxSize to windows events

* do not set customer flag on event ids; use the same naming for channels and providers

* work to unify manifest and resources

* netdata now logs using ETW

* implemented etw and wel logging in netdata

* minor changes

* updated windows installer to install the manifest

* do not install etw if the manifest is not there

* allow loggings to WEL and ETW at the same time

* fix the installer conditions

* fix nsi

* detect ci paths for sys utils

* enable ETW is CI

* better integration of spawn server with logger

* use script to find SDK path

* use auto-discovery of sdk and visual studio

* fix overlapping link.exe with msys; do not escape percentage when it is not followed by a number; added more documentation about windows

* debug info for path

* fixes compilation scripts

* ETW and WEL are always required on Windows

* in progress for supporting full text search queries

* find mvc versions

* improve find-sdk-path.sh

* fix the script once again

* fetch event data for full text search

* fix script again

* fix script, yes again

* fts using event data

* code renames and cleanup for clarity

* update documentation

* full text search switches plugin to load everything synchronously

* full text search using the individual event data fields, without using XML

* close all idle provider handles after 5 mins

* added EventsAPI field

* supported exposing all system fields; started documentation about windows events plugin

* avoid crash because of unitialized memory

* remove debugging

* do not add qualifiers and version when they are zero

* updated docs

* copy the manifest too

* rework on installing manifest and dll

* completed documentation

* work on windows-events sources list

* fix windows installer logic

* removed unecessary include

* added image to documentation
This commit is contained in:
Costa Tsaousis 2024-10-15 09:04:59 +03:00 • committed by GitHub
parent ccfc0444b0
commit f6141cc4f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
72 changed files with 8021 additions and 3442 deletions

117
CMakeLists.txt
View file

@ -670,10 +670,10 @@ set(LIBNETDATA_FILES
src/libnetdata/libnetdata.h
src/libnetdata/locks/locks.c
src/libnetdata/locks/locks.h
src/libnetdata/log/journal.c
src/libnetdata/log/journal.h
src/libnetdata/log/log.c
src/libnetdata/log/log.h
src/libnetdata/log/systemd-journal-helpers.c
src/libnetdata/log/systemd-journal-helpers.h
src/libnetdata/log/nd_log.c
src/libnetdata/log/nd_log.h
src/libnetdata/os/os.c
src/libnetdata/os/os.h
src/libnetdata/simple_hashtable.h
@ -822,6 +822,25 @@ set(LIBNETDATA_FILES
src/libnetdata/sanitizers/sanitizers-functions.h
src/libnetdata/sanitizers/sanitizers-pluginsd.c
src/libnetdata/sanitizers/sanitizers-pluginsd.h
src/libnetdata/log/nd_log-internals.c
src/libnetdata/log/nd_log-internals.h
src/libnetdata/log/nd_log_limit.c
src/libnetdata/log/nd_log_limit.h
src/libnetdata/log/nd_log-config.c
src/libnetdata/log/nd_log-init.c
src/libnetdata/log/nd_log-to-syslog.c
src/libnetdata/log/nd_log-to-systemd-journal.c
src/libnetdata/log/nd_log-annotators.c
src/libnetdata/log/nd_log-field-formatters.c
src/libnetdata/log/nd_log-format-logfmt.c
src/libnetdata/log/nd_log-format-json.c
src/libnetdata/log/nd_log-to-file.c
src/libnetdata/log/nd_log-to-windows-events.c
src/libnetdata/string/utf8.c
src/libnetdata/spawn_server/log-forwarder.c
src/libnetdata/spawn_server/log-forwarder.h
src/libnetdata/log/nd_log-common.h
src/libnetdata/log/nd_log-to-windows-common.h
)
if(ENABLE_PLUGIN_EBPF)
@ -1439,12 +1458,13 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
src/collectors/windows-events.plugin/windows-events-unicode.h
src/collectors/windows-events.plugin/windows-events-xml.c
src/collectors/windows-events.plugin/windows-events-xml.h
src/collectors/windows-events.plugin/windows-events-publishers.c
src/collectors/windows-events.plugin/windows-events-publishers.h
src/collectors/windows-events.plugin/windows-events-providers.c
src/collectors/windows-events.plugin/windows-events-providers.h
src/collectors/windows-events.plugin/windows-events-fields-cache.c
src/collectors/windows-events.plugin/windows-events-fields-cache.h
src/collectors/windows-events.plugin/windows-events-query-builder.c
src/collectors/windows-events.plugin/windows-events-query-builder.h
src/collectors/windows-events.plugin/windows-events-query-evt-variant.c
)
set(WINDOWS_PLUGIN_FILES
@ -1746,10 +1766,90 @@ target_include_directories(libnetdata BEFORE PUBLIC ${CONFIG_H_DIR} ${CMAKE_SOUR
target_link_libraries(libnetdata PUBLIC
"$<$<NOT:$<BOOL:${HAVE_BUILTIN_ATOMICS}>>:atomic>"
"$<$<OR:$<BOOL:${OS_LINUX}>,$<BOOL:${OS_FREEBSD}>>:pthread;rt>"
"$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt>"
"$<$<BOOL:${OS_WINDOWS}>:kernel32;advapi32;winmm;rpcrt4;bcrypt;wevtapi>"
"$<$<BOOL:${LINK_LIBM}>:m>"
"${SYSTEMD_LDFLAGS}")
if(OS_WINDOWS)
set(HAVE_ETW True)
set(HAVE_WEL True)
# Output the results for debugging purposes
message(STATUS "Have Event Tracing for Windows (ETW): ${HAVE_ETW}")
message(STATUS "Have Windows Event Log (WEL): ${HAVE_WEL}")
if(HAVE_WEL OR HAVE_ETW)
# Define the source and generated file paths
set(WEVT_GEN_SRC_H_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/nd_log-to-windows-common.h")
set(WEVT_GEN_SRC_C_FILE "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_mc_generate.c")
set(WEVT_GEN_BIN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_mc_generate")
set(WEVT_BUILD_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_compile.sh")
set(WEVT_MC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.mc")
set(WEVT_MAN_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.xml")
set(WEVT_RC_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.rc")
set(WEVT_MC_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.h")
set(WEVT_MAN_H_FILE "${CMAKE_BINARY_DIR}/wevt_netdata_manifest.h")
set(WEVT_RES_OBJECT "${CMAKE_BINARY_DIR}/wevt_netdata_res.o")
set(WEVT_DLL_FILE "${CMAKE_BINARY_DIR}/wevt_netdata.dll")
set(WEVT_ETW_INSTALL_SCRIPT "${CMAKE_SOURCE_DIR}/src/libnetdata/log/wevt_netdata_install.bat")
# we compile ${WEVT_GEN_BIN_FILE}, which generates the manifest, the .mc,
# and the headers required for compiling libnetdata/logs
if(HAVE_ETW)
# ETW method also supports WEL
# but it requires Microsoft tools mc, rc, and link
add_custom_command(
OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_MAN_H_FILE}" "${WEVT_DLL_FILE}"
COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
COMMAND "${WEVT_BUILD_SCRIPT}" "${CMAKE_SOURCE_DIR}/src/libnetdata/log" "${CMAKE_BINARY_DIR}"
DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
)
else()
# WEL method can be built with windmc, windres and the normal linker
add_custom_command(
OUTPUT "${WEVT_MC_H_FILE}" "${WEVT_DLL_FILE}"
COMMAND "${CMAKE_C_COMPILER}" -o "${WEVT_GEN_BIN_FILE}" "${WEVT_GEN_SRC_C_FILE}"
COMMAND "${WEVT_GEN_BIN_FILE}" >"${WEVT_MC_FILE}"
COMMAND "${WEVT_GEN_BIN_FILE}" --manifest >"${WEVT_MAN_FILE}"
COMMAND windmc -r "${CMAKE_BINARY_DIR}" -h "${CMAKE_BINARY_DIR}" ${WEVT_MC_FILE}
COMMAND echo "1 2004" "wevt_netdata_manifest.xml" >> "${WEVT_RC_FILE}"
COMMAND windres ${WEVT_RC_FILE} -o ${WEVT_RES_OBJECT}
COMMAND ${CMAKE_LINKER} -dll --entry 0 -nostdlib -o ${WEVT_DLL_FILE} ${WEVT_RES_OBJECT}
DEPENDS "${WEVT_GEN_SRC_C_FILE}" "${WEVT_GEN_SRC_H_FILE}"
COMMENT "Compiling ${WEVT_MC_FILE} to generate ${WEVT_MC_H_FILE} and ${WEVT_DLL_FILE}"
)
endif()
# Create a custom target for the DLL
add_custom_target(wevt_netdata ALL DEPENDS ${WEVT_DLL_FILE})
set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
OBJECT_DEPENDS "${WEVT_MC_H_FILE}")
if(HAVE_ETW)
set_source_files_properties(src/libnetdata/log/nd_log-to-windows-events.c PROPERTIES
OBJECT_DEPENDS "${WEVT_MAN_H_FILE}")
install(FILES "${WEVT_DLL_FILE}" "${WEVT_MAN_FILE}" "${WEVT_ETW_INSTALL_SCRIPT}"
COMPONENT wevt_netdata_dll
DESTINATION "${BINDIR}")
else()
# do not install the manifest in this case
# the nsi installer will skip registering the ETW publisher
install(FILES "${WEVT_DLL_FILE}"
COMPONENT wevt_netdata_dll
DESTINATION "${BINDIR}")
endif()
endif()
endif()
# ebpf
if(ENABLE_PLUGIN_EBPF)
netdata_add_libbpf_to_target(libnetdata)
@ -1925,8 +2025,7 @@ if(ENABLE_PLUGIN_APPS)
add_executable(apps.plugin ${APPS_PLUGIN_FILES})
target_link_libraries(apps.plugin libnetdata ${CAP_LIBRARIES}
"$<$<BOOL:${OS_WINDOWS}>:Version>"
"$<$<BOOL:${OS_WINDOWS}>:ntdll>")
"$<$<BOOL:${OS_WINDOWS}>:Version;ntdll>")
target_include_directories(apps.plugin PRIVATE ${CAP_INCLUDE_DIRS})
target_compile_options(apps.plugin PRIVATE ${CAP_CFLAGS_OTHER})

2
packaging/cmake/config.cmake.h.in
View file

@ -169,6 +169,8 @@
#cmakedefine HAVE_LIBYAML
#cmakedefine HAVE_LIBMNL
#cmakedefine HAVE_WEL
#cmakedefine HAVE_ETW
#cmakedefine RUN_UNDER_CLION
// /* Enable GNU extensions on systems that have them. */

11
packaging/utils/compile-on-windows.sh → packaging/utils/compile-and-run-windows.sh
View file

@ -70,15 +70,20 @@ then
${NULL}
fi
ninja -v -C "${build}" || ninja -v -C "${build}" -j 1
echo "Stopping service Netdata"
sc stop "Netdata" || echo "Failed"
ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
# register the event log publisher
cmd.exe //c "$(cygpath -w -a "/opt/netdata/usr/bin/wevt_netdata_install.bat")"
#echo
#echo "Compile with:"
#echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
echo "Stopping service Netdata"
sc stop "Netdata" || echo "Failed"
echo "starting netdata..."
# enable JIT debug with gdb
export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"

4
packaging/windows/clion-msys-msys-environment.bat
View file

@ -13,7 +13,9 @@ set MSYSTEM=MSYS
:: go exists only mingw64 / ucrt64 / etc, not under msys profile
set GOROOT=C:\msys64\mingw64
set PATH="%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
set "PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
set "PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
set "PATH=%PATH%;C:\msys64\usr\bin;C:\msys64\bin;C:\msys64\mingw64\bin"
::set PKG_CONFIG_EXECUTABLE=C:\msys64\mingw64\bin\pkg-config.exe
::set CMAKE_C_COMPILER=C:\msys64\mingw64\bin\gcc.exe
::set CMAKE_CC_COMPILER=C:\msys64\mingw64\bin\g++.exe

217
packaging/windows/find-sdk-path.sh Normal file
View file

@ -0,0 +1,217 @@
#!/bin/bash
# Function to output the path in Windows format (convert from MSYS2/Unix format using cygpath)
convert_to_windows_format() {
cygpath -w -a "$1"
}
# Function to display help message
display_help() {
echo "Usage: $0 [-s|--sdk] [-v|--visualstudio] [-w|--windows] [--help]"
echo
echo "Options:"
echo " -s, --sdk Search for tools in the Windows SDK."
echo " -v, --visualstudio Search for tools in Visual Studio."
echo " -w, --windows Output the path in Windows format (using cygpath)."
echo " --help Display this help message."
exit 0
}
# Function to find tools in the Windows SDK
find_sdk_tools() {
sdk_base_path="/c/Program Files (x86)/Windows Kits/10/bin"
if [ ! -d "$sdk_base_path" ]; then
echo "ERROR: SDK base path \"$sdk_base_path\" does not exist. No SDK installations found." >&2
echo "$system_root"
return 1
fi
echo "SDK base path exists: \"$sdk_base_path\"" >&2
# Find all SDK versions
sdk_versions=($(ls "$sdk_base_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
echo "Found SDK versions: ${sdk_versions[*]}" >&2
if [ ${#sdk_versions[@]} -eq 0 ]; then
echo "ERROR: No valid Windows SDK versions found in \"$sdk_base_path\"." >&2
echo "$system_root"
return 1
fi
# Sort versions and pick the latest
sorted_versions=$(printf '%s\n' "${sdk_versions[@]}" | sort -V)
latest_sdk_version=$(echo "$sorted_versions" | tail -n 1)
sdk_tool_path="$sdk_base_path/$latest_sdk_version/x64"
echo "Latest SDK version: \"$latest_sdk_version\"" >&2
if [ ! -d "$sdk_tool_path" ]; then
echo "ERROR: Tool path \"$sdk_tool_path\" does not exist." >&2
echo "$system_root"
return 1
fi
# Check if required tools exist
tools=("mc.exe" "rc.exe")
for tool in "${tools[@]}"; do
if [ ! -f "$sdk_tool_path/$tool" ]; then
echo "ERROR: $tool not found in \"$sdk_tool_path\"" >&2
echo "$system_root"
return 1
else
echo "$tool found in \"$sdk_tool_path\"" >&2
fi
done
echo >&2
echo "DONE: All required tools found in \"$sdk_tool_path\"" >&2
echo >&2
echo "$sdk_tool_path"
}
# Function to find tools in Visual Studio
find_visual_studio_tools() {
studio_base_path="/c/Program Files/Microsoft Visual Studio/2022"
echo "Checking for Visual Studio installations in: \"$studio_base_path\"" >&2
if [ ! -d "$studio_base_path" ]; then
echo "ERROR: Visual Studio base path \"$studio_base_path\" does not exist. No Visual Studio installations found." >&2
echo "$system_root"
return 1
fi
# Visual Studio editions we want to check
editions=("Enterprise" "Professional" "Community")
available_editions=()
# Loop through each edition and check for tools
for edition in "${editions[@]}"; do
edition_path="$studio_base_path/$edition/VC/Tools/MSVC"
if [ -d "$edition_path" ]; then
available_editions+=("$edition")
echo "Checking edition: $edition in $studio_base_path" >&2
# Find all MSVC versions and sort them
msvc_versions=($(ls "$edition_path" | tr ' ' '\n' | grep -E "^[0-9]+\..*$"))
echo "Found MSVC versions in $edition: ${msvc_versions[*]}" >&2
if [ ${#msvc_versions[@]} -gt 0 ]; then
sorted_versions=$(printf '%s\n' "${msvc_versions[@]}" | sort -V)
latest_msvc_version=$(echo "${sorted_versions[@]}" | tail -n 1)
vs_tool_path="$edition_path/$latest_msvc_version/bin/Hostx64/x64"
echo "Latest MSVC version: \"$latest_msvc_version\" in $edition" >&2
if [ ! -d "$vs_tool_path" ]; then
echo "WARNING: Tool path \"$vs_tool_path\" does not exist." >&2
continue
fi
# Check if required tools exist
tools=("link.exe")
missing_tool=0
for tool in "${tools[@]}"; do
if [ ! -f "$vs_tool_path/$tool" ]; then
echo "WARNING: $tool not found in \"$vs_tool_path\" for $edition" >&2
missing_tool=1
else
echo "$tool found in \"$vs_tool_path\"" >&2
fi
done
if [ $missing_tool -eq 0 ]; then
echo >&2
echo "All required tools found in \"$vs_tool_path\"" >&2
echo >&2
echo "$vs_tool_path"
return 0
else
echo "WARNING: skipping edition '$edition', directory does not exist." >&2
fi
else
echo "WARNING: skipping edition '$edition', MSVC directory does not exist." >&2
fi
else
echo "WARNING: skipping edition '$edition', directory does not exist." >&2
fi
done
echo "ERROR: No valid Visual Studio editions found in \"$studio_base_path\"." >&2
echo "$system_root"
return 1
}
# Parse options using getopt
TEMP=$(getopt -o svwh --long sdk,visualstudio,windows,help -- "$@")
if [ $? != 0 ]; then
echo "ERROR: Invalid options provided." >&2
exit 1
fi
eval set -- "$TEMP"
search_mode="sdk"
windows_format=0
system_root="/usr/bin"
# Process getopt options
while true; do
case "$1" in
-s|--sdk)
search_mode="sdk"
shift
;;
-v|--visualstudio)
search_mode="visualstudio"
shift
;;
-w|--windows)
system_root="%SYSTEMROOT%"
windows_format=1
shift
;;
--help|-h)
display_help
;;
--)
shift
break
;;
*)
echo "ERROR: Invalid option: $1" >&2
exit 1
;;
esac
done
# Ensure that one of --sdk or --visualstudio is selected
if [ -z "$search_mode" ]; then
echo "ERROR: You must specify either --sdk or --visualstudio." >&2
display_help
fi
# Determine which function to call based on the search mode
if [ "$search_mode" = "sdk" ]; then
tool_path=$(find_sdk_tools)
else
tool_path=$(find_visual_studio_tools)
fi
# If a valid path is found, output it
if [ "$tool_path" != "$system_root" ]; then
if [ "$windows_format" -eq 1 ]; then
windows_tool_path=$(convert_to_windows_format "$tool_path")
echo "$windows_tool_path"
else
echo "$tool_path"
fi
else
echo "$system_root"
exit 1
fi
exit 0

103
packaging/windows/installer.nsi
View file

@ -257,29 +257,97 @@ Function NetdataUninstallRegistry
end:
FunctionEnd
Function InstallDLL
; Check if certutil is available
nsExec::ExecToStack 'where certutil'
Pop $R0
StrCmp $R0 "" NoCertUtil FoundCertUtil
NoCertUtil:
DetailPrint "certutil not found, assuming files are different."
Goto CopyDLL
FoundCertUtil:
; Calculate hash of the existing DLL
nsExec::ExecToStack 'certutil -hashfile "$SYSDIR\wevt_netdata.dll" MD5'
Pop $R0
; Calculate hash of the new DLL
nsExec::ExecToStack 'certutil -hashfile "$INSTDIR\usr\bin\wevt_netdata.dll" MD5'
Pop $R1
StrCmp $R0 $R1 SetPermissions
CopyDLL:
ClearErrors
CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata.dll" "$SYSDIR"
IfErrors RetryPrompt SetPermissions
RetryPrompt:
MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata.dll probably because it is in use. Please close the Event Viewer (or other Event Log applications) and press Retry."
StrCmp $R0 IDRETRY CopyDLL
StrCmp $R0 IDCANCEL ExitInstall
Goto End
SetPermissions:
nsExec::ExecToLog 'icacls "$SYSDIR\wevt_netdata.dll" /grant "NT SERVICE\EventLog":R'
Goto End
ExitInstall:
Abort
End:
FunctionEnd
Function InstallManifest
IfFileExists "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" CopyManifest End
CopyManifest:
ClearErrors
CopyFiles /SILENT "$INSTDIR\usr\bin\wevt_netdata_manifest.xml" "$SYSDIR"
IfErrors RetryPrompt InstallManifest
RetryPrompt:
MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "Failed to copy wevt_netdata_manifest.xml."
StrCmp $R0 IDRETRY CopyManifest
StrCmp $R0 IDCANCEL ExitInstall
InstallManifest:
nsExec::ExecToLog 'wevtutil im "$SYSDIR\wevt_netdata_manifest.xml" "/mf:$SYSDIR\wevt_netdata.dll" "/rf:$SYSDIR\wevt_netdata.dll"'
Goto End
ExitInstall:
Abort
End:
FunctionEnd
Section "Install Netdata"
SetOutPath $INSTDIR
SetCompress off
SetOutPath $INSTDIR
SetCompress off
File /r "C:\msys64\opt\netdata\*.*"
File /r "C:\msys64\opt\netdata\*.*"
ClearErrors
ClearErrors
nsExec::ExecToLog '$SYSDIR\sc.exe create Netdata binPath= "$INSTDIR\usr\bin\netdata.exe" start= delayed-auto'
pop $0
${If} $0 != 0
DetailPrint "Warning: Failed to create Netdata service."
DetailPrint "Warning: Failed to create Netdata service."
${EndIf}
ClearErrors
ClearErrors
nsExec::ExecToLog '$SYSDIR\sc.exe description Netdata "Real-time system monitoring service"'
pop $0
${If} $0 != 0
DetailPrint "Warning: Failed to add Netdata service description."
DetailPrint "Warning: Failed to add Netdata service description."
${EndIf}
WriteUninstaller "$INSTDIR\Uninstall.exe"
Call NetdataUninstallRegistry
Call InstallDLL
Call InstallManifest
StrLen $0 $cloudToken
StrLen $1 $cloudRooms
@ -325,9 +393,24 @@ Section "Uninstall"
DetailPrint "Warning: Failed to delete Netdata service."
${EndIf}
# https://nsis.sourceforge.io/Reference/RMDir
RMDir /r /REBOOTOK "$INSTDIR"
; Check if the manifest exists before uninstalling it
IfFileExists "$SYSDIR\wevt_netdata_manifest.xml" ManifestExistsForUninstall ManifestNotExistsForUninstall
ManifestExistsForUninstall:
nsExec::ExecToLog 'wevtutil um "$SYSDIR\wevt_netdata_manifest.xml"'
pop $0
${If} $0 != 0
DetailPrint "Warning: Failed to uninstall the event manifest."
${EndIf}
Goto DoneUninstall
ManifestNotExistsForUninstall:
DetailPrint "Manifest not found, skipping manifest uninstall."
DoneUninstall:
; Remove files
RMDir /r /REBOOTOK "$INSTDIR"
DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\Netdata"
SectionEnd

8
src/collectors/apps.plugin/apps_os_windows.c
View file

@ -523,14 +523,10 @@ static char *wchar_to_utf8(WCHAR *s) {
static char *ansi_to_utf8(LPCSTR str) {
static __thread WCHAR unicode[PATH_MAX];
static __thread int unicode_size = sizeof(unicode) / sizeof(*unicode);
// Step 1: Convert ANSI string (LPSTR) to wide string (UTF-16)
int wideLength = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0);
if (wideLength == 0 || wideLength > unicode_size)
return NULL;
MultiByteToWideChar(CP_ACP, 0, str, -1, unicode, wideLength);
size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1);
if (!count) return NULL;
return wchar_to_utf8(unicode);
}

12
src/collectors/apps.plugin/apps_pid.c
View file

@ -327,15 +327,11 @@ static bool is_filename(const char *s) {
(*s == '/' && s[1] == '/' && isalpha((uint8_t)s[2]) && s[3] == '/')) { // windows native "//x/"
WCHAR ws[FILENAME_MAX];
int wlen = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0);
if (wlen <= 0 || (size_t)wlen > sizeof(ws) / sizeof(*ws)) {
return false; // Failed to convert UTF-8 to UTF-16
if(utf8_to_utf16(ws, _countof(ws), s, -1) > 0) {
DWORD attributes = GetFileAttributesW(ws);
if (attributes != INVALID_FILE_ATTRIBUTES)
return true;
}
MultiByteToWideChar(CP_UTF8, 0, s, -1, ws, wlen);
DWORD attributes = GetFileAttributesW(ws);
if (attributes != INVALID_FILE_ATTRIBUTES)
return true;
}
#endif

1
src/collectors/systemd-journal.plugin/systemd-journal.c
View file

@ -85,6 +85,7 @@ struct lqs_extension {
#define LQS_SOURCE_TYPE SD_JOURNAL_FILE_SOURCE_TYPE
#define LQS_SOURCE_TYPE_ALL SDJF_ALL
#define LQS_SOURCE_TYPE_NONE SDJF_NONE
#define LQS_PARAMETER_SOURCE_NAME "Journal Sources" // this is how it is shown to users
#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) get_internal_source_type(value)
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) available_journal_file_sources_to_json_array(wb)
#include "libnetdata/facets/logs_query_status.h"

289
src/collectors/windows-events.plugin/README.md Normal file
View file

@ -0,0 +1,289 @@
# Windows Events plugin
[KEY FEATURES](#key-features) | [EVENTS SOURCES](#events-sources) | [EVENT FIELDS](#event-fields) |
[PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [PERFORMANCE](#query-performance) |
[CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq)
The Windows Events plugin by Netdata makes viewing, exploring and analyzing Windows Events simple and
efficient.
![image](https://github.com/user-attachments/assets/71a1ab1d-5b7b-477e-a4e6-a30275a5710b)
## Key features
- Supports **Windows Event Logs (WEL)**.
- Supports **Event Tracing for Windows (ETW)** and **TraceLogging (TL)**, when events are routed to Event Log.
- Allows filtering on all System Events fields.
- Allows **full text search** (`grep`) on all System and User fields.
- Provides a **histogram** for log entries over time, with a break down per field-value, for any System Event field and any
time-frame.
- Supports coloring log entries based on severity.
- In PLAY mode it "tails" all the Events, showing new log entries immediately after they are received.
### Prerequisites
`windows-events.plugin` is a Netdata Function Plugin.
To protect your privacy, as with all Netdata Functions, a free Netdata Cloud user account is required to access it.
For more information check [this discussion](https://github.com/netdata/netdata/discussions/16136).
## Events Sources
The plugin automatically detects all the available channels and offers a list of "Event Channels".
By default, it aggregates events from all event channels, providing a unified systems view of all events.
> To improve query performance, we recommend selecting the relevant event channels, before doing more
> analysis on the events.
In the list of events channels, several shortcuts are added, aggregating events according to various attributes:
- `All`, aggregates events from all available channels. This provides a holistic view of all events in the system.
- `All-Admin`, `All-Operational`, `All-Analytic` and `All-Debug` aggregates events from channels marked `Admin`, `Operational`, `Analytic` and `Debug`, respectively.
- `All-Windows`, aggregates events from `Application`, `Security`, `System` and `Setup`.
- `All-Enabled` and `All-Disabled` aggregates events from channels depending on their status.
- `All-Forwarded` aggregates events from channels owned by `Microsoft-Windows-EventCollector`.
- `All-Classic` aggregates events from channels using the Classic Event Log API.
- `All-Of-X`, where `X` is a provider name, is offered for all providers having more than a channel.
- `All-In-X`, where `X` is `Backup-Mode`, `Overwrite-Mode`, `StopWhenFull-Mode` and `RetainAndBackup-Mode`, aggregate events based on their channel retention policy.
Channels that are configured but are not queryable, and channels that do not have any events in them, are automatically excluded from the channels list.
## Event Fields
Windows Events are structured with both system-defined fields and user-defined fields.
The Windows Events plugin primarily works with the system-defined fields, which are consistently available
across all event types.
### System-defined fields
The system-defined fields are:
1. **EventRecordID**
A unique, sequential identifier for the event within the channel. This ID increases as new events are logged.
2. **Version**
The version of the event, indicating possible structural changes or updates to the event definition.
Netdata adds this field automatically when it is not zero.
3. **Level**
The severity or importance of the event. Levels can include:
- 0: LogAlways (reserved)
- 1: Critical
- 2: Error
- 3: Warning
- 4: Informational
- 5: Verbose
Additionally, applications may define their own levels.
Netdata provides 2 fields: `Level` and `LevelID` for the text and numeric representation of it.
4. **Opcode**
The action or state within a provider when the event was logged.
Netdata provides 2 fields: `Opcode` and `OpcodeID` for the text and numeric representation of it.
5. **EventID**
This identifies the event template, linking it to a specific message or event type. Event IDs are provider-specific.
6. **Task**
Defines a higher-level categorization or logical grouping for the event, often related to a specific function within the application or provider.
Netdata provides 2 fields: `Task` and `TaskID` for the text and numeric representation of it.
7. **Qualifiers**
Provides additional detail for interpreting the event and is often specific to the event source.
Netdata adds this field automatically when it is not zero.
8. **ProcessID**
The ID of the process that generated the event, useful for pinpointing the source of the event within the system.
9. **ThreadID**
The ID of the thread within the process that generated the event, which helps in more detailed debugging scenarios.
10. **Keywords**
A categorization field that can be used for event filtering. Keywords are bit flags that represent categories or purposes of the event, providing additional context.
Netdata provides 2 fields: `Keywords` and `keywordsID` for the text and numeric representation of it.
11. **Provider**
The unique identifier (GUID) of the event provider. This is essential for knowing which application or system component generated the event.
Netdata provides 2 fields: `Provider` and `ProviderGUID` for its name and GUID of it.
12. **ActivityID**
A GUID that correlates events generated as part of the same operation or transaction, helping to track activities across different components or stages.
Netdata adds this field automatically when it is not zero.
13. **RelatedActivityID**
A GUID that links related operations or transactions, allowing for tracing complex workflows where one event triggers or relates to another.
Netdata adds this field automatically when it is not zero.
14. **Timestamp**
The timestamp when the event was created. This provides precise timing information about when the event occurred.
15. **User**
The system user who logged this event.
Netdata provides 3 fields: `UserAccount`, `UserDomain` and `UserSID`.
### User-defined fields
Each event log entry can include up to 100 user-defined fields (per event-id).
Unfortunately, accessing these fields is significantly slower, to a level that is not practical to do so
when there are more than few thousand log entries to explore. So, Netdata presents them
with lazy loading.
This prevents Netdata for offering filtering for user-defined fields, although Netdata does support
full text search on user-defined field values.
### Event fields as columns in the table
The system fields mentioned above are offered as columns on the UI. Use the gear button above the table to
select visible columns.
### Event fields as filters
The plugin presents the system fields as filters for the query, with counters for each of the possible values
for the field. This list can be used to quickly check which fields and values are available for the entire
time-frame of the query, across multiple providers and channels.
### Event fields as histogram sources
The histogram can be based on any of the system fields that are available as filters. This allows you to
visualize the distribution of events over time based on different criteria such as Level, Provider, or EventID.
## PLAY mode
The PLAY mode in this plugin allows real-time monitoring of new events as they are added to the Windows Event
Log. This feature works by continuously querying for new events and updating the display.
## Full-text search
The plugin supports searching for text within all system and user fields of the events. This means that while
user-defined fields are not directly filterable, they are searchable through the full-text search feature.
Keep in mind that query performance is slower while doing full text search, mainly because the plugin
needs to ask from the system to provide all the user fields values.
## Query performance
The plugin is optimized to work efficiently with Event Logs. It uses several layers of caching and
similar techniques to offload as much work as possible from the system, offering quick responses even when
hundreds of thousands of events are within the visible timeframe.
To achieve this level of efficiency, the plugin:
- pre-loads ETW providers' manifests for resolving numeric Levels, Opcodes, Tasks and Keywords to text.
- caches number to text maps for Levels, Opcodes, Tasks and Keywords per provider for WEL providers.
- caches user SID to account and domain maps.
- lazy loads the "expensive" event Message and XML, so that the system is queried only for the visible events.
For Full Text Search:
- requests only the Message and the values of the user-fields from the system, avoiding the "expensive" XML call (which is still lazy-loaded).
The result is a system that is highly efficient for working with moderate volumes (hundreds of thousands) of events.
## Configuration and maintenance
This Netdata plugin does not require any specific configuration. It automatically detects available event logs
on the system.
## FAQ
### Can I use this plugin on event centralization servers?
Yes. You can centralize your Windows Events using Windows Event Forwarding (WEF) or other event collection
mechanisms, and then install Netdata on this events centralization server to explore the events of all your
infrastructure.
This plugin will automatically provide multi-node views of your events and also give you the ability to
combine the events of multiple servers, as you see fit.
### Can I use this plugin from a parent Netdata?
Yes. When your nodes are connected to a Netdata parent, all their functions are available via the parent's UI.
So, from the parent UI, you can access the functions of all your nodes.
Keep in mind that to protect your privacy, in order to access Netdata functions, you need a free Netdata Cloud
account.
### Is any of my data exposed to Netdata Cloud from this plugin?
No. When you access the agent directly, none of your data passes through Netdata Cloud. You need a free Netdata
Cloud account only to verify your identity and enable the use of Netdata Functions. Once this is done, all the
data flow directly from your Netdata agent to your web browser.
When you access Netdata via https://app.netdata.cloud, your data travel via Netdata Cloud, but they are not stored
in Netdata Cloud. This is to allow you access your Netdata agents from anywhere. All communication from/to
Netdata Cloud is encrypted.
### What are the different types of event logs supported by this plugin?
The plugin supports all the kinds of event logs currently supported by the Windows Event Viewer:
- Windows Event Logs (WEL): The traditional event logging system in Windows.
- Event Tracing for Windows (ETW): A more detailed and efficient event tracing system.
- TraceLogging (TL): An extension of ETW that simplifies the process of adding events to your code.
The plugin can access all of these when they are routed to the Windows Event Log.
### How does this plugin handle user-defined fields in Windows Events?
User-defined fields are not directly exposed as table columns or filters in the plugin interface. However,
they are included in the XML representation of each event, which can be viewed in the info sidebar when
clicking on an event entry. Additionally, the full-text search feature does search through these
user-defined fields, allowing you to find specific information even if it's not in the main system fields.
### Can I use this plugin to monitor real-time events?
Yes, the plugin supports a PLAY mode that allows you to monitor events in real-time. When activated, it
continuously updates to show new events as they are logged, similar to the "tail" functionality in
Unix-like systems.
### How does the plugin handle large volumes of events?
The plugin is designed to handle moderate volumes of events (hundreds of thousands of events) efficiently.
It is in our roadmap to port the `systemd-journal` sampling techniques to it, for working with very large
datasets to provide quick responses while still giving accurate representations of the data. However, for
the best performance, we recommend querying smaller time frames or using more specific filters when dealing
with extremely large event volumes.
### Can I use this plugin to analyze events from multiple servers?
Yes, if you have set up Windows Event Forwarding (WEF) or another method of centralizing your Windows Events,
you can use this plugin on the central server to analyze events from multiple sources. The plugin will
automatically detect the available event sources.
### How does the histogram feature work in this plugin?
The histogram feature provides a visual representation of event frequency over time. You can base the
histogram on any of the system fields available as filters (such as Level, Provider, or EventID). This
allows you to quickly identify patterns or anomalies in your event logs.
### Is it possible to export or share the results from this plugin?
While the plugin doesn't have a direct export feature, you can use browser-based methods to save or share
the results. This could include taking screenshots, using browser print/save as PDF functionality, or
copying data from the table view. For more advanced data export needs, you might need to use the Windows
Event Log API directly or other Windows administrative tools.
### How often does the plugin update its data?
The plugin updates its data in real-time when in PLAY mode. In normal mode, it refreshes data based on the
query you've submitted. The plugin is designed to provide the most up-to-date information available in the
Windows Event Logs at the time of the query.
## TODO
1. Support Sampling, so that the plugin can respond faster even on very busy systems (millions of events visible).
2. Support exploring events from live Tracing sessions.
3. Support exploring events in saved Event Trace Log files (`.etl` files).
4. Support exploring events in saved Event Logs files (`.evtx` files).

10
src/collectors/windows-events.plugin/windows-events-fields-cache.c
View file

@ -53,16 +53,16 @@ void field_cache_init(void) {
static inline bool should_zero_provider(WEVT_FIELD_TYPE type, uint64_t value) {
switch(type) {
case WEVT_FIELD_TYPE_LEVEL:
return !is_valid_publisher_level(value, true);
return !is_valid_provider_level(value, true);
case WEVT_FIELD_TYPE_KEYWORDS:
return !is_valid_publisher_keywords(value, true);
case WEVT_FIELD_TYPE_KEYWORD:
return !is_valid_provider_keyword(value, true);
case WEVT_FIELD_TYPE_OPCODE:
return !is_valid_publisher_opcode(value, true);
return !is_valid_provider_opcode(value, true);
case WEVT_FIELD_TYPE_TASK:
return !is_valid_publisher_task(value, true);
return !is_valid_provider_task(value, true);
default:
return false;

2
src/collectors/windows-events.plugin/windows-events-fields-cache.h
View file

@ -8,7 +8,7 @@
typedef enum __attribute__((packed)) {
WEVT_FIELD_TYPE_LEVEL = 0,
WEVT_FIELD_TYPE_OPCODE,
WEVT_FIELD_TYPE_KEYWORDS,
WEVT_FIELD_TYPE_KEYWORD,
WEVT_FIELD_TYPE_TASK,
// terminator

310
src/collectors/windows-events.plugin/windows-events-publishers.c → src/collectors/windows-events.plugin/windows-events-providers.c
View file

@ -1,17 +1,19 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "windows-events-publishers.h"
#include "windows-events-providers.h"
#define MAX_OPEN_HANDLES_PER_PUBLISHER 5
#define MAX_OPEN_HANDLES_PER_PROVIDER 5
struct publisher;
struct provider;
// typedef as PROVIDER_META_HANDLE in include file
struct provider_meta_handle {
pid_t owner; // the owner of the handle, or zero
uint32_t locks; // the number of locks the owner has on this handle
EVT_HANDLE hMetadata; // the handle
struct publisher *publisher; // a pointer back to the publisher
struct provider *provider; // a pointer back to the provider
usec_t created_monotonic_ut; // the monotonic timestamp this handle was created
// double linked list
PROVIDER_META_HANDLE *prev;
@ -32,78 +34,156 @@ struct provider_list {
struct provider_data *array; // the array of entries, sorted (for binary search)
};
typedef struct publisher {
typedef struct provider_key {
ND_UUID uuid; // the Provider GUID
const char *name; // the Provider name (UTF-8)
DWORD len; // the length of the Provider Name
const wchar_t *wname; // the Provider wide-string Name (UTF-16)
} PROVIDER_KEY;
typedef struct provider {
PROVIDER_KEY key;
const char *name; // the Provider Name (UTF-8)
uint32_t total_handles; // the number of handles allocated
uint32_t available_handles; // the number of available handles
uint32_t deleted_handles; // the number of deleted handles
PROVIDER_META_HANDLE *handles; // a double linked list of all the handles
struct provider_list keywords;
WEVT_PROVIDER_PLATFORM platform;
struct provider_list keyword;
struct provider_list tasks;
struct provider_list opcodes;
struct provider_list levels;
} PUBLISHER;
} PROVIDER;
// A hashtable implementation for publishers
// using the provider GUID as key and PUBLISHER as value
#define SIMPLE_HASHTABLE_NAME _PROVIDER_GUID
#define SIMPLE_HASHTABLE_VALUE_TYPE PUBLISHER
#define SIMPLE_HASHTABLE_KEY_TYPE ND_UUID
#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION publisher_value_to_key
#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION publisher_cache_compar
// A hashtable implementation for Providers
// using the Provider GUID as key and PROVIDER as value
#define SIMPLE_HASHTABLE_NAME _PROVIDER
#define SIMPLE_HASHTABLE_VALUE_TYPE PROVIDER
#define SIMPLE_HASHTABLE_KEY_TYPE PROVIDER_KEY
#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION provider_value_to_key
#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION provider_cache_compar
#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
#include "libnetdata/simple_hashtable.h"
static struct {
SPINLOCK spinlock;
uint32_t total_publishers;
uint32_t total_providers;
uint32_t total_handles;
uint32_t deleted_handles;
struct simple_hashtable_PROVIDER_GUID hashtable;
ARAL *aral_publishers;
struct simple_hashtable_PROVIDER hashtable;
ARAL *aral_providers;
ARAL *aral_handles;
} pbc = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
};
static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
static inline ND_UUID *publisher_value_to_key(PUBLISHER *p) {
return &p->uuid;
const char *provider_get_name(PROVIDER_META_HANDLE *p) {
return (p && p->provider && p->provider->name) ? p->provider->name : "__UNKNOWN PROVIDER__";
}
static inline bool publisher_cache_compar(ND_UUID *a, ND_UUID *b) {
return UUIDeq(*a, *b);
ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p) {
return (p && p->provider) ? p->provider->key.uuid : UUID_ZERO;
}
void publisher_cache_init(void) {
simple_hashtable_init_PROVIDER_GUID(&pbc.hashtable, 100000);
pbc.aral_publishers = aral_create("wevt_publishers", sizeof(PUBLISHER), 0, 4096, NULL, NULL, NULL, false, true);
static inline PROVIDER_KEY *provider_value_to_key(PROVIDER *p) {
return &p->key;
}
static inline bool provider_cache_compar(PROVIDER_KEY *a, PROVIDER_KEY *b) {
return a->len == b->len && UUIDeq(a->uuid, b->uuid) && memcmp(a->wname, b->wname, a->len) == 0;
}
void provider_cache_init(void) {
simple_hashtable_init_PROVIDER(&pbc.hashtable, 100000);
pbc.aral_providers = aral_create("wevt_providers", sizeof(PROVIDER), 0, 4096, NULL, NULL, NULL, false, true);
pbc.aral_handles = aral_create("wevt_handles", sizeof(PROVIDER_META_HANDLE), 0, 4096, NULL, NULL, NULL, false, true);
}
PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
if(!providerName || !providerName[0] || UUIDiszero(uuid))
static bool provider_property_get(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
DWORD bufferUsed = 0;
if(!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
DWORD status = GetLastError();
if (status != ERROR_INSUFFICIENT_BUFFER) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
goto cleanup;
}
}
wevt_variant_resize(content, bufferUsed);
if (!EvtGetPublisherMetadataProperty(h->hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
goto cleanup;
}
return true;
cleanup:
return false;
}
static bool provider_string_property_exists(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
if(!provider_property_get(h, content, property_id))
return false;
if(content->data->Type != EvtVarTypeString)
return false;
if(!content->data->StringVal[0])
return false;
return true;
}
static void provider_detect_platform(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content) {
if(UUIDiszero(h->provider->key.uuid))
h->provider->platform = WEVT_PLATFORM_WEL;
else if(h->hMetadata) {
if (provider_string_property_exists(h, content, EvtPublisherMetadataMessageFilePath) ||
provider_string_property_exists(h, content, EvtPublisherMetadataResourceFilePath) ||
provider_string_property_exists(h, content, EvtPublisherMetadataParameterFilePath))
h->provider->platform = WEVT_PLATFORM_ETW;
else
// The provider cannot be opened, does not have any resource files (message, resource, parameter)
h->provider->platform = WEVT_PLATFORM_TL;
}
else h->provider->platform = WEVT_PLATFORM_ETW;
}
WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p) {
return p->provider->platform;
}
PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName) {
if(!providerName || !providerName[0])
return NULL;
// XXH64_hash_t hash = XXH3_64bits(&uuid, sizeof(uuid));
uint64_t hash = uuid.parts.low64 + uuid.parts.hig64;
PROVIDER_KEY key = {
.uuid = uuid,
.len = wcslen(providerName),
.wname = providerName,
};
XXH64_hash_t hash = XXH3_64bits(providerName, wcslen(key.wname) * sizeof(*key.wname));
spinlock_lock(&pbc.spinlock);
SIMPLE_HASHTABLE_SLOT_PROVIDER_GUID *slot =
simple_hashtable_get_slot_PROVIDER_GUID(&pbc.hashtable, hash, &uuid, true);
SIMPLE_HASHTABLE_SLOT_PROVIDER *slot =
simple_hashtable_get_slot_PROVIDER(&pbc.hashtable, hash, &key, true);
bool load_it = false;
PUBLISHER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
if(!p) {
p = aral_callocz(pbc.aral_publishers);
p->uuid = uuid;
simple_hashtable_set_slot_PROVIDER_GUID(&pbc.hashtable, slot, hash, p);
p = aral_callocz(pbc.aral_providers);
p->key.uuid = key.uuid;
p->key.len = key.len;
p->key.wname = wcsdup(key.wname);
p->name = strdupz(provider2utf8(key.wname));
simple_hashtable_set_slot_PROVIDER(&pbc.hashtable, slot, hash, p);
load_it = true;
pbc.total_publishers++;
pbc.total_providers++;
}
pid_t me = gettid_cached();
@ -117,7 +197,8 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
if(!h) {
h = aral_callocz(pbc.aral_handles);
h->publisher = p;
h->provider = p;
h->created_monotonic_ut = now_monotonic_usec();
h->hMetadata = EvtOpenPublisherMetadata(
NULL, // Local machine
providerName, // Provider name
@ -147,10 +228,11 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
WEVT_VARIANT property = { 0 };
TXT_UNICODE unicode = { 0 };
publisher_load_list(h, &content, &property, &unicode, &p->keywords, EvtPublisherMetadataKeywords);
publisher_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
publisher_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
publisher_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
provider_detect_platform(h, &content);
provider_load_list(h, &content, &property, &unicode, &p->keyword, EvtPublisherMetadataKeywords);
provider_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
provider_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
provider_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
txt_unicode_cleanup(&unicode);
wevt_variant_cleanup(&content);
@ -162,42 +244,72 @@ PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
return h;
}
EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h) {
EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h) {
return h ? h->hMetadata : NULL;
}
PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h) {
PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h) {
if(h) h->locks++;
return h;
}
void publisher_release(PROVIDER_META_HANDLE *h) {
static void provider_meta_handle_delete(PROVIDER_META_HANDLE *h) {
PROVIDER *p = h->provider;
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
if(h->hMetadata)
EvtClose(h->hMetadata);
aral_freez(pbc.aral_handles, h);
fatal_assert(pbc.total_handles && p->total_handles && p->available_handles);
pbc.total_handles--;
p->total_handles--;
pbc.deleted_handles++;
p->deleted_handles++;
p->available_handles--;
}
void providers_release_unused_handles(void) {
usec_t now_ut = now_monotonic_usec();
spinlock_lock(&pbc.spinlock);
for(size_t i = 0; i < pbc.hashtable.size ; i++) {
SIMPLE_HASHTABLE_SLOT_PROVIDER *slot = &pbc.hashtable.hashtable[i];
PROVIDER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
if(!p) continue;
PROVIDER_META_HANDLE *h = p->handles;
while(h) {
PROVIDER_META_HANDLE *next = h->next;
if(!h->locks && (now_ut - h->created_monotonic_ut) >= WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT)
provider_meta_handle_delete(h);
h = next;
}
}
spinlock_unlock(&pbc.spinlock);
}
void provider_release(PROVIDER_META_HANDLE *h) {
if(!h) return;
pid_t me = gettid_cached();
fatal_assert(h->owner == me);
fatal_assert(h->locks > 0);
if(--h->locks == 0) {
PUBLISHER *p = h->publisher;
PROVIDER *p = h->provider;
spinlock_lock(&pbc.spinlock);
h->owner = 0;
if(++p->available_handles > MAX_OPEN_HANDLES_PER_PUBLISHER) {
// there are multiple handles on this publisher
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
if(h->hMetadata)
EvtClose(h->hMetadata);
aral_freez(pbc.aral_handles, h);
pbc.total_handles--;
p->total_handles--;
pbc.deleted_handles++;
p->deleted_handles++;
p->available_handles--;
if(++p->available_handles > MAX_OPEN_HANDLES_PER_PROVIDER) {
// there are too many idle handles on this provider
provider_meta_handle_delete(h);
}
else if(h->next) {
// it is not the last, put it at the end
@ -210,7 +322,7 @@ void publisher_release(PROVIDER_META_HANDLE *h) {
}
// --------------------------------------------------------------------------------------------------------------------
// load publisher lists
// load provider lists
static bool wevt_get_property_from_array(WEVT_VARIANT *property, EVT_HANDLE handle, DWORD dwIndex, EVT_PUBLISHER_METADATA_PROPERTY_ID PropertyId) {
DWORD used = 0;
@ -253,7 +365,7 @@ static int compare_ascending(const void *a, const void *b) {
// return 0;
//}
static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
if(!h || !h->hMetadata) return;
EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id;
@ -268,7 +380,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
value_id = EvtPublisherMetadataLevelValue;
value_bits = 32;
compare_func = compare_ascending;
is_valid = is_valid_publisher_level;
is_valid = is_valid_provider_level;
break;
case EvtPublisherMetadataOpcodes:
@ -276,7 +388,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataOpcodeMessageID;
value_id = EvtPublisherMetadataOpcodeValue;
value_bits = 32;
is_valid = is_valid_publisher_opcode;
is_valid = is_valid_provider_opcode;
compare_func = compare_ascending;
break;
@ -285,7 +397,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataTaskMessageID;
value_id = EvtPublisherMetadataTaskValue;
value_bits = 32;
is_valid = is_valid_publisher_task;
is_valid = is_valid_provider_task;
compare_func = compare_ascending;
break;
@ -294,7 +406,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
message_id = EvtPublisherMetadataKeywordMessageID;
value_id = EvtPublisherMetadataKeywordValue;
value_bits = 64;
is_valid = is_valid_publisher_keywords;
is_valid = is_valid_provider_keyword;
compare_func = NULL;
break;
@ -305,23 +417,11 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
EVT_HANDLE hMetadata = h->hMetadata;
EVT_HANDLE hArray = NULL;
DWORD bufferUsed = 0;
DWORD itemCount = 0;
// Get the metadata array for the list (e.g., opcodes, tasks, or levels)
if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
DWORD status = GetLastError();
if (status != ERROR_INSUFFICIENT_BUFFER) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
goto cleanup;
}
}
wevt_variant_resize(content, bufferUsed);
if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
if(!provider_property_get(h, content, property_id))
goto cleanup;
}
// Get the number of items (e.g., levels, tasks, or opcodes)
hArray = content->data->EvtHandleVal;
@ -375,7 +475,7 @@ static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content,
uint32_t messageID = wevt_field_get_uint32(property->data);
if (messageID != (uint32_t)-1) {
if (wevt_get_message_unicode(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
if (EvtFormatMessage_utf16(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) {
size_t len;
d->name = unicode2utf8_strdupz(dst->data, &len);
d->len = len;
@ -414,7 +514,7 @@ cleanup:
// lookup functions
// lookup bitmap metdata (returns a comma separated list of strings)
static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
static bool provider_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
if(!(value & l->mask) || !l->total || !l->array || l->exceeds_data_type)
return false;
@ -445,7 +545,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
memcpy(&dst->data[dst->used], s, slen);
dst->used += slen;
dst->src = TXT_SOURCE_PUBLISHER;
dst->src = TXT_SOURCE_PROVIDER;
added++;
}
}
@ -460,7 +560,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
}
//// lookup a single value (returns its string)
//static bool publisher_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
//static bool provider_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
// if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
// return false;
//
@ -477,7 +577,7 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
//
// memcpy(dst->data, s, slen);
// dst->used = slen;
// dst->src = TXT_SOURCE_PUBLISHER;
// dst->src = TXT_SOURCE_PROVIDER;
//
// break;
// }
@ -493,11 +593,11 @@ static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, ui
// return (dst->used > 0);
//}
static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
static bool provider_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
return false;
// if(l->total < 3) return publisher_value_metadata_linear(dst, l, value);
// if(l->total < 3) return provider_value_metadata_linear(dst, l, value);
dst->used = 0;
@ -519,7 +619,7 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
memcpy(dst->data, s, slen);
dst->used = slen;
dst->data[dst->used++] = 0;
dst->src = TXT_SOURCE_PUBLISHER;
dst->src = TXT_SOURCE_PROVIDER;
}
break;
}
@ -539,38 +639,38 @@ static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uin
// --------------------------------------------------------------------------------------------------------------------
// public API to lookup metadata
bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->publisher->keywords.exceeds_data_type;
bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->provider->keyword.exceeds_data_type;
}
bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->publisher->tasks.exceeds_data_type;
bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->provider->tasks.exceeds_data_type;
}
bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h) {
return h && !h->publisher->levels.exceeds_data_type;
bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h) {
return h && !h->provider->levels.exceeds_data_type;
}
bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->publisher->opcodes.exceeds_data_type;
bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
return h && !h->provider->opcodes.exceeds_data_type;
}
bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
return publisher_bitmap_metadata(dst, &h->publisher->keywords, value);
return provider_bitmap_metadata(dst, &h->provider->keyword, value);
}
bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
return publisher_value_metadata(dst, &h->publisher->levels, value);
return provider_value_metadata(dst, &h->provider->levels, value);
}
bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
return publisher_value_metadata(dst, &h->publisher->tasks, value);
return provider_value_metadata(dst, &h->provider->tasks, value);
}
bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
if(!h) return false;
return publisher_value_metadata(dst, &h->publisher->opcodes, value);
return provider_value_metadata(dst, &h->provider->opcodes, value);
}

41
src/collectors/windows-events.plugin/windows-events-providers.h Normal file
View file

@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_WINDOWS_EVENTS_PROVIDERS_H
#define NETDATA_WINDOWS_EVENTS_PROVIDERS_H
typedef enum __attribute__((packed)) {
WEVT_PLATFORM_UNKNOWN = 0,
WEVT_PLATFORM_WEL,
WEVT_PLATFORM_ETW,
WEVT_PLATFORM_TL,
} WEVT_PROVIDER_PLATFORM;
#include "windows-events.h"
struct provider_meta_handle;
typedef struct provider_meta_handle PROVIDER_META_HANDLE;
PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName);
void provider_release(PROVIDER_META_HANDLE *h);
EVT_HANDLE provider_handle(PROVIDER_META_HANDLE *h);
PROVIDER_META_HANDLE *provider_dup(PROVIDER_META_HANDLE *h);
void providers_release_unused_handles(void);
const char *provider_get_name(PROVIDER_META_HANDLE *p);
ND_UUID provider_get_uuid(PROVIDER_META_HANDLE *p);
void provider_cache_init(void);
bool provider_keyword_cacheable(PROVIDER_META_HANDLE *h);
bool provider_tasks_cacheable(PROVIDER_META_HANDLE *h);
bool is_useful_provider_for_levels(PROVIDER_META_HANDLE *h);
bool provider_opcodes_cacheable(PROVIDER_META_HANDLE *h);
bool provider_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool provider_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool provider_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool provider_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
WEVT_PROVIDER_PLATFORM provider_get_platform(PROVIDER_META_HANDLE *p);
#endif //NETDATA_WINDOWS_EVENTS_PROVIDERS_H

28
src/collectors/windows-events.plugin/windows-events-publishers.h
View file

@ -1,28 +0,0 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
#define NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
#include "windows-events.h"
struct provider_meta_handle;
typedef struct provider_meta_handle PROVIDER_META_HANDLE;
PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName);
void publisher_release(PROVIDER_META_HANDLE *h);
EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h);
PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h);
void publisher_cache_init(void);
bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h);
bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h);
bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h);
bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h);
bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
#endif //NETDATA_WINDOWS_EVENTS_PUBLISHERS_H

353
src/collectors/windows-events.plugin/windows-events-query-evt-variant.c Normal file
View file

@ -0,0 +1,353 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "windows-events.h"
#include <sddl.h> // For SID string conversion
// Function to append the separator if the buffer is not empty
static inline void append_separator_if_needed(BUFFER *b, const char *separator) {
if (buffer_strlen(b) > 0 && separator != NULL)
buffer_strcat(b, separator);
}
// Helper function to convert UTF16 strings to UTF8 and append to the buffer
static inline void append_utf16(BUFFER *b, LPCWSTR utf16Str, const char *separator) {
if (!utf16Str || !*utf16Str) return;
append_separator_if_needed(b, separator);
size_t remaining = b->size - b->len;
if(remaining < 128) {
buffer_need_bytes(b, 128);
remaining = b->size - b->len;
}
size_t used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
if(used >= remaining) {
// oops, we need to resize
size_t needed = utf16_to_utf8(NULL, 0, utf16Str, -1); // find the size needed
buffer_need_bytes(b, needed);
remaining = b->size - b->len;
used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1);
}
if(used) {
b->len += used - 1;
internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
"Buffer length mismatch.");
}
}
// Function to append binary data to the buffer
static inline void append_binary(BUFFER *b, PBYTE data, DWORD size, const char *separator) {
if (data == NULL || size == 0) return;
append_separator_if_needed(b, separator);
buffer_need_bytes(b, size * 4);
for (DWORD i = 0; i < size; i++) {
uint8_t value = data[i];
b->buffer[b->len++] = hex_digits[(value & 0xf0) >> 4];
b->buffer[b->len++] = hex_digits[(value & 0x0f)];
}
}
// Function to append size_t to the buffer
static inline void append_size_t(BUFFER *b, size_t size, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64(b, size);
}
// Function to append HexInt32 in hexadecimal format
static inline void append_uint32_hex(BUFFER *b, UINT32 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64_hex(b, n);
}
// Function to append HexInt64 in hexadecimal format
static inline void append_uint64_hex(BUFFER *b, UINT64 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64_hex(b, n);
}
// Function to append various data types to the buffer
static inline void append_uint64(BUFFER *b, UINT64 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64(b, n);
}
static inline void append_int64(BUFFER *b, INT64 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_int64(b, n);
}
static inline void append_double(BUFFER *b, double n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_netdata_double(b, n);
}
static inline void append_guid(BUFFER *b, GUID *guid, const char *separator) {
fatal_assert(sizeof(GUID) == sizeof(nd_uuid_t));
append_separator_if_needed(b, separator);
ND_UUID *uuid = (ND_UUID *)guid;
buffer_need_bytes(b, UUID_STR_LEN);
uuid_unparse_lower(uuid->uuid, &b->buffer[b->len]);
b->len += UUID_STR_LEN - 1;
internal_fatal(buffer_strlen(b) != strlen(buffer_tostring(b)),
"Buffer length mismatch.");
}
static inline void append_systime(BUFFER *b, SYSTEMTIME *st, const char *separator) {
append_separator_if_needed(b, separator);
buffer_sprintf(b, "%04d-%02d-%02d %02d:%02d:%02d",
st->wYear, st->wMonth, st->wDay, st->wHour, st->wMinute, st->wSecond);
}
static inline void append_filetime(BUFFER *b, FILETIME *ft, const char *separator) {
SYSTEMTIME st;
if (FileTimeToSystemTime(ft, &st))
append_systime(b, &st, separator);
}
static inline void append_sid(BUFFER *b, PSID sid, const char *separator) {
buffer_sid_to_sid_str_and_name(sid, b, separator);
}
static inline void append_sbyte(BUFFER *b, INT8 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_int64(b, n);
}
static inline void append_byte(BUFFER *b, UINT8 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64(b, n);
}
static inline void append_int16(BUFFER *b, INT16 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_int64(b, n);
}
static inline void append_uint16(BUFFER *b, UINT16 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64(b, n);
}
static inline void append_int32(BUFFER *b, INT32 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_int64(b, n);
}
static inline void append_uint32(BUFFER *b, UINT32 n, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64(b, n);
}
// Function to append EVT_HANDLE to the buffer
static inline void append_evt_handle(BUFFER *b, EVT_HANDLE h, const char *separator) {
append_separator_if_needed(b, separator);
buffer_print_uint64_hex(b, (uintptr_t)h);
}
// Function to append XML data (UTF-16) to the buffer
static inline void append_evt_xml(BUFFER *b, LPCWSTR xmlData, const char *separator) {
append_utf16(b, xmlData, separator); // XML data is essentially UTF-16 string
}
void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator) {
if(ev->Type == EvtVarTypeNull) return;
if (ev->Type & EVT_VARIANT_TYPE_ARRAY) {
for (DWORD i = 0; i < ev->Count; i++) {
switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
case EvtVarTypeString:
append_utf16(b, ev->StringArr[i], separator);
break;
case EvtVarTypeAnsiString:
if (ev->AnsiStringArr[i] != NULL) {
append_utf16(b, (LPCWSTR)ev->AnsiStringArr[i], separator);
}
break;
case EvtVarTypeSByte:
append_sbyte(b, ev->SByteArr[i], separator);
break;
case EvtVarTypeByte:
append_byte(b, ev->ByteArr[i], separator);
break;
case EvtVarTypeInt16:
append_int16(b, ev->Int16Arr[i], separator);
break;
case EvtVarTypeUInt16:
append_uint16(b, ev->UInt16Arr[i], separator);
break;
case EvtVarTypeInt32:
append_int32(b, ev->Int32Arr[i], separator);
break;
case EvtVarTypeUInt32:
append_uint32(b, ev->UInt32Arr[i], separator);
break;
case EvtVarTypeInt64:
append_int64(b, ev->Int64Arr[i], separator);
break;
case EvtVarTypeUInt64:
append_uint64(b, ev->UInt64Arr[i], separator);
break;
case EvtVarTypeSingle:
append_double(b, ev->SingleArr[i], separator);
break;
case EvtVarTypeDouble:
append_double(b, ev->DoubleArr[i], separator);
break;
case EvtVarTypeGuid:
append_guid(b, &ev->GuidArr[i], separator);
break;
case EvtVarTypeFileTime:
append_filetime(b, &ev->FileTimeArr[i], separator);
break;
case EvtVarTypeSysTime:
append_systime(b, &ev->SysTimeArr[i], separator);
break;
case EvtVarTypeSid:
append_sid(b, ev->SidArr[i], separator);
break;
case EvtVarTypeBinary:
append_binary(b, ev->BinaryVal, ev->Count, separator);
break;
case EvtVarTypeSizeT:
append_size_t(b, ev->SizeTArr[i], separator);
break;
case EvtVarTypeHexInt32:
append_uint32_hex(b, ev->UInt32Arr[i], separator);
break;
case EvtVarTypeHexInt64:
append_uint64_hex(b, ev->UInt64Arr[i], separator);
break;
case EvtVarTypeEvtHandle:
append_evt_handle(b, ev->EvtHandleVal, separator);
break;
case EvtVarTypeEvtXml:
append_evt_xml(b, ev->XmlValArr[i], separator);
break;
default:
// Skip unknown array types
break;
}
}
} else {
switch (ev->Type & EVT_VARIANT_TYPE_MASK) {
case EvtVarTypeNull:
// Do nothing for null types
break;
case EvtVarTypeString:
append_utf16(b, ev->StringVal, separator);
break;
case EvtVarTypeAnsiString:
append_utf16(b, (LPCWSTR)ev->AnsiStringVal, separator);
break;
case EvtVarTypeSByte:
append_sbyte(b, ev->SByteVal, separator);
break;
case EvtVarTypeByte:
append_byte(b, ev->ByteVal, separator);
break;
case EvtVarTypeInt16:
append_int16(b, ev->Int16Val, separator);
break;
case EvtVarTypeUInt16:
append_uint16(b, ev->UInt16Val, separator);
break;
case EvtVarTypeInt32:
append_int32(b, ev->Int32Val, separator);
break;
case EvtVarTypeUInt32:
append_uint32(b, ev->UInt32Val, separator);
break;
case EvtVarTypeInt64:
append_int64(b, ev->Int64Val, separator);
break;
case EvtVarTypeUInt64:
append_uint64(b, ev->UInt64Val, separator);
break;
case EvtVarTypeSingle:
append_double(b, ev->SingleVal, separator);
break;
case EvtVarTypeDouble:
append_double(b, ev->DoubleVal, separator);
break;
case EvtVarTypeBoolean:
append_separator_if_needed(b, separator);
buffer_strcat(b, ev->BooleanVal ? "true" : "false");
break;
case EvtVarTypeGuid:
append_guid(b, ev->GuidVal, separator);
break;
case EvtVarTypeBinary:
append_binary(b, ev->BinaryVal, ev->Count, separator);
break;
case EvtVarTypeSizeT:
append_size_t(b, ev->SizeTVal, separator);
break;
case EvtVarTypeHexInt32:
append_uint32_hex(b, ev->UInt32Val, separator);
break;
case EvtVarTypeHexInt64:
append_uint64_hex(b, ev->UInt64Val, separator);
break;
case EvtVarTypeEvtHandle:
append_evt_handle(b, ev->EvtHandleVal, separator);
break;
case EvtVarTypeEvtXml:
append_evt_xml(b, ev->XmlVal, separator);
break;
default:
// Skip unknown types
break;
}
}
}

423
src/collectors/windows-events.plugin/windows-events-query.c
View file

@ -2,48 +2,13 @@
#include "windows-events.h"
static void wevt_event_done(WEVT_LOG *log);
static uint64_t wevt_log_file_size(const wchar_t *channel);
#define FIELD_RECORD_NUMBER (0)
#define FIELD_EVENT_ID (1)
#define FIELD_LEVEL (2)
#define FIELD_OPCODE (3)
#define FIELD_KEYWORDS (4)
#define FIELD_VERSION (5)
#define FIELD_TASK (6)
#define FIELD_PROCESS_ID (7)
#define FIELD_THREAD_ID (8)
#define FIELD_TIME_CREATED (9)
#define FIELD_CHANNEL (10)
#define FIELD_COMPUTER_NAME (11)
#define FIELD_PROVIDER_NAME (12)
#define FIELD_EVENT_SOURCE_NAME (13)
#define FIELD_PROVIDER_GUID (14)
#define FIELD_CORRELATION_ACTIVITY_ID (15)
#define FIELD_USER_ID (16)
// --------------------------------------------------------------------------------------------------------------------
// These are the fields we extract from the logs
static const wchar_t *RENDER_ITEMS[] = {
L"/Event/System/EventRecordID",
L"/Event/System/EventID",
L"/Event/System/Level",
L"/Event/System/Opcode",
L"/Event/System/Keywords",
L"/Event/System/Version",
L"/Event/System/Task",
L"/Event/System/Execution/@ProcessID",
L"/Event/System/Execution/@ThreadID",
L"/Event/System/TimeCreated/@SystemTime",
L"/Event/System/Channel",
L"/Event/System/Computer",
L"/Event/System/Provider/@Name",
L"/Event/System/Provider/@EventSourceName",
L"/Event/System/Provider/@Guid",
L"/Event/System/Correlation/@ActivityID",
L"/Event/System/Security/@UserID",
};
static const char *wevt_extended_status(void) {
static const char *EvtGetExtendedStatus_utf8(void) {
static __thread wchar_t wbuf[4096];
static __thread char buf[4096];
DWORD wbuf_used = 0;
@ -62,7 +27,9 @@ static const char *wevt_extended_status(void) {
return buf;
}
bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
// --------------------------------------------------------------------------------------------------------------------
bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) {
dst->used = 0;
DWORD size = 0;
@ -107,56 +74,44 @@ cleanup:
return false;
}
static bool wevt_get_field_from_events_log(
WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
static bool EvtFormatMessage_utf8(
TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent,
TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) {
dst->src = TXT_SOURCE_EVENT_LOG;
if(wevt_get_message_unicode(&log->ops.unicode, publisher_handle(p), hEvent, 0, flags))
return wevt_str_unicode_to_utf8(dst, &log->ops.unicode);
if(EvtFormatMessage_utf16(tmp, provider_handle(p), hEvent, 0, flags))
return wevt_str_unicode_to_utf8(dst, tmp);
wevt_utf8_empty(dst);
return false;
}
bool wevt_get_event_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageEvent);
bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageEvent);
}
bool wevt_get_xml_utf8(WEVT_LOG *log, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
return wevt_get_field_from_events_log(log, p, hEvent, dst, EvtFormatMessageXml);
bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) {
return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageXml);
}
static inline void wevt_event_done(WEVT_LOG *log) {
if (log->publisher) {
publisher_release(log->publisher);
log->publisher = NULL;
}
if (log->hEvent) {
EvtClose(log->hEvent);
log->hEvent = NULL;
}
log->ops.level.src = TXT_SOURCE_UNKNOWN;
log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
log->ops.task.src = TXT_SOURCE_UNKNOWN;
}
// --------------------------------------------------------------------------------------------------------------------
static void wevt_get_field_from_cache(
WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
TXT_UTF8 *dst, const ND_UUID *provider,
WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
WEVT_LOG *log, uint64_t value, PROVIDER_META_HANDLE *h,
TXT_UTF8 *dst, const ND_UUID *provider,
WEVT_FIELD_TYPE cache_type, EVT_FORMAT_MESSAGE_FLAGS flags) {
if (field_cache_get(cache_type, provider, value, dst))
return;
wevt_get_field_from_events_log(log, h, log->hEvent, dst, flags);
EvtFormatMessage_utf8(&log->ops.unicode, h, log->hEvent, dst, flags);
field_cache_set(cache_type, provider, value, dst);
}
// --------------------------------------------------------------------------------------------------------------------
// Level
#define SET_LEN_AND_RETURN(constant) *len = sizeof(constant) - 1; return constant
static inline const char *wevt_level_hardcoded(uint64_t level, size_t *len) {
@ -179,9 +134,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageLevel;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_LEVEL;
bool is_publisher = is_valid_publisher_level(value, true);
bool is_provider = is_valid_provider_level(value, true);
if(!is_publisher) {
if(!is_provider) {
size_t len;
const char *hardcoded = wevt_level_hardcoded(value, &len);
if(hardcoded) {
@ -189,12 +144,12 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
dst->src = TXT_SOURCE_HARDCODED;
}
else {
// since this is not a publisher value
// since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
else if (!publisher_get_level(dst, h, value)) {
else if (!provider_get_level(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@ -203,6 +158,9 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *
dst, WEVT_PREFIX_LEVEL, sizeof(WEVT_PREFIX_LEVEL) - 1, ev->level);
}
// --------------------------------------------------------------------------------------------------------------------
// Opcode
static inline const char *wevt_opcode_hardcoded(uint64_t opcode, size_t *len) {
switch(opcode) {
case WEVT_OPCODE_INFO: SET_LEN_AND_RETURN(WEVT_OPCODE_NAME_INFO);
@ -228,9 +186,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageOpcode;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_OPCODE;
bool is_publisher = is_valid_publisher_opcode(value, true);
bool is_provider = is_valid_provider_opcode(value, true);
if(!is_publisher) {
if(!is_provider) {
size_t len;
const char *hardcoded = wevt_opcode_hardcoded(value, &len);
if(hardcoded) {
@ -238,12 +196,12 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
dst->src = TXT_SOURCE_HARDCODED;
}
else {
// since this is not a publisher value
// since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
else if (!publisher_get_opcode(dst, h, value)) {
else if (!provider_get_opcode(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@ -252,6 +210,9 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE
dst, WEVT_PREFIX_OPCODE, sizeof(WEVT_PREFIX_OPCODE) - 1, ev->opcode);
}
// --------------------------------------------------------------------------------------------------------------------
// Task
static const char *wevt_task_hardcoded(uint64_t task, size_t *len) {
switch(task) {
case WEVT_TASK_NONE: SET_LEN_AND_RETURN(WEVT_TASK_NAME_NONE);
@ -267,9 +228,9 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageTask;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_TASK;
bool is_publisher = is_valid_publisher_task(value, true);
bool is_provider = is_valid_provider_task(value, true);
if(!is_publisher) {
if(!is_provider) {
size_t len;
const char *hardcoded = wevt_task_hardcoded(value, &len);
if(hardcoded) {
@ -277,12 +238,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
dst->src = TXT_SOURCE_HARDCODED;
}
else {
// since this is not a publisher value
// since this is not a provider value
// we expect to get the system description of it
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
}
else if (!publisher_get_task(dst, h, value)) {
else if (!provider_get_task(dst, h, value)) {
// not found in the manifest, get it from the cache
wevt_get_field_from_cache(log, value, h, dst, &ev->provider, cache_type, flags);
}
@ -291,9 +252,12 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h
dst, WEVT_PREFIX_TASK, sizeof(WEVT_PREFIX_TASK) - 1, ev->task);
}
// --------------------------------------------------------------------------------------------------------------------
// Keyword
#define SET_BITS(msk, txt) { .mask = msk, .name = txt, .len = sizeof(txt) - 1, }
static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
static uint64_t wevt_keyword_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
struct {
uint64_t mask;
const char *name;
@ -324,7 +288,7 @@ static uint64_t wevt_keywords_handle_reserved(uint64_t value, TXT_UTF8 *dst) {
return value & 0x0000FFFFFFFFFFFF;
}
static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
static void wevt_get_keyword(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h) {
TXT_UTF8 *dst = &log->ops.keywords;
if(ev->keywords == WEVT_KEYWORD_NONE) {
@ -332,18 +296,18 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
dst->src = TXT_SOURCE_HARDCODED;
}
uint64_t value = wevt_keywords_handle_reserved(ev->keywords, dst);
uint64_t value = wevt_keyword_handle_reserved(ev->keywords, dst);
EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageKeyword;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORDS;
WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_KEYWORD;
if(!value && dst->used <= 1) {
// no hardcoded info in the buffer, make it None
txt_utf8_set(dst, WEVT_KEYWORD_NAME_NONE, sizeof(WEVT_KEYWORD_NAME_NONE) - 1);
dst->src = TXT_SOURCE_HARDCODED;
}
else if (value && !publisher_get_keywords(dst, h, value) && dst->used <= 1) {
// the publisher did not provide any info and the description is still empty.
else if (value && !provider_get_keywords(dst, h, value) && dst->used <= 1) {
// the provider did not provide any info and the description is still empty.
// the system returns 1 keyword, the highest bit, not a list
// so, when we call the system, we pass the original value (ev->keywords)
wevt_get_field_from_cache(log, ev->keywords, h, dst, &ev->provider, cache_type, flags);
@ -353,57 +317,86 @@ static void wevt_get_keywords(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDL
dst, WEVT_PREFIX_KEYWORDS, sizeof(WEVT_PREFIX_KEYWORDS) - 1, ev->keywords);
}
bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
bool ret = false;
// --------------------------------------------------------------------------------------------------------------------
// Fetching Events
// obtain the information from selected events
static inline bool wEvtRender(WEVT_LOG *log, EVT_HANDLE context, WEVT_VARIANT *raw) {
DWORD bytes_used = 0, property_count = 0;
if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
// information exceeds the allocated space
if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, hRenderContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %zu, extended info: %s",
(uintptr_t)log->hRenderContext, (uintptr_t)log->hEvent, (uintptr_t)log->ops.content.data, log->ops.content.size, wevt_extended_status());
goto cleanup;
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtRender() failed, hRenderSystemContext: 0x%lx, hEvent: 0x%lx, content: 0x%lx, size: %u, extended info: %s",
(uintptr_t)context, (uintptr_t)log->hEvent, (uintptr_t)raw->data, raw->size,
EvtGetExtendedStatus_utf8());
return false;
}
wevt_variant_resize(&log->ops.content, bytes_used);
if (!EvtRender(log->hRenderContext, log->hEvent, EvtRenderEventValues, log->ops.content.size, log->ops.content.data, &bytes_used, &property_count)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtRender() failed, after bytes_used increase, extended info: %s",
wevt_extended_status());
goto cleanup;
wevt_variant_resize(raw, bytes_used);
if (!EvtRender(context, log->hEvent, EvtRenderEventValues, raw->size, raw->data, &bytes_used, &property_count)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtRender() failed, after bytes_used increase, extended info: %s",
EvtGetExtendedStatus_utf8());
return false;
}
}
log->ops.content.used = bytes_used;
raw->used = bytes_used;
raw->count = property_count;
EVT_VARIANT *content = log->ops.content.data;
return true;
}
ev->id = wevt_field_get_uint64(&content[FIELD_RECORD_NUMBER]);
ev->event_id = wevt_field_get_uint16(&content[FIELD_EVENT_ID]);
ev->level = wevt_field_get_uint8(&content[FIELD_LEVEL]);
ev->opcode = wevt_field_get_uint8(&content[FIELD_OPCODE]);
ev->keywords = wevt_field_get_uint64_hex(&content[FIELD_KEYWORDS]);
ev->version = wevt_field_get_uint8(&content[FIELD_VERSION]);
ev->task = wevt_field_get_uint16(&content[FIELD_TASK]);
ev->process_id = wevt_field_get_uint32(&content[FIELD_PROCESS_ID]);
ev->thread_id = wevt_field_get_uint32(&content[FIELD_THREAD_ID]);
ev->created_ns = wevt_field_get_filetime_to_ns(&content[FIELD_TIME_CREATED]);
static bool wevt_get_next_event_one(WEVT_LOG *log, WEVT_EVENT *ev) {
bool ret = false;
if(full) {
wevt_field_get_string_utf8(&content[FIELD_CHANNEL], &log->ops.channel);
wevt_field_get_string_utf8(&content[FIELD_COMPUTER_NAME], &log->ops.computer);
wevt_field_get_string_utf8(&content[FIELD_PROVIDER_NAME], &log->ops.provider);
wevt_field_get_string_utf8(&content[FIELD_EVENT_SOURCE_NAME], &log->ops.source);
wevt_get_uuid_by_type(&content[FIELD_PROVIDER_GUID], &ev->provider);
wevt_get_uuid_by_type(&content[FIELD_CORRELATION_ACTIVITY_ID], &ev->correlation_activity_id);
wevt_field_get_sid(&content[FIELD_USER_ID], &log->ops.user);
if(!wEvtRender(log, log->hRenderSystemContext, &log->ops.raw.system))
goto cleanup;
PROVIDER_META_HANDLE *h = log->publisher =
publisher_get(ev->provider, content[FIELD_PROVIDER_NAME].StringVal);
EVT_VARIANT *content = log->ops.raw.system.data;
wevt_get_level(log, ev, h);
wevt_get_task(log, ev, h);
wevt_get_opcode(log, ev, h);
wevt_get_keywords(log, ev, h);
ev->id = wevt_field_get_uint64(&content[EvtSystemEventRecordId]);
ev->event_id = wevt_field_get_uint16(&content[EvtSystemEventID]);
ev->level = wevt_field_get_uint8(&content[EvtSystemLevel]);
ev->opcode = wevt_field_get_uint8(&content[EvtSystemOpcode]);
ev->keywords = wevt_field_get_uint64_hex(&content[EvtSystemKeywords]);
ev->version = wevt_field_get_uint8(&content[EvtSystemVersion]);
ev->task = wevt_field_get_uint16(&content[EvtSystemTask]);
ev->qualifiers = wevt_field_get_uint16(&content[EvtSystemQualifiers]);
ev->process_id = wevt_field_get_uint32(&content[EvtSystemProcessID]);
ev->thread_id = wevt_field_get_uint32(&content[EvtSystemThreadID]);
ev->created_ns = wevt_field_get_filetime_to_ns(&content[EvtSystemTimeCreated]);
if(log->type & WEVT_QUERY_EXTENDED) {
wevt_field_get_string_utf8(&content[EvtSystemChannel], &log->ops.channel);
wevt_field_get_string_utf8(&content[EvtSystemComputer], &log->ops.computer);
wevt_field_get_string_utf8(&content[EvtSystemProviderName], &log->ops.provider);
wevt_get_uuid_by_type(&content[EvtSystemProviderGuid], &ev->provider);
wevt_get_uuid_by_type(&content[EvtSystemActivityID], &ev->activity_id);
wevt_get_uuid_by_type(&content[EvtSystemRelatedActivityID], &ev->related_activity_id);
wevt_field_get_sid(&content[EvtSystemUserID], &log->ops.account, &log->ops.domain, &log->ops.sid);
PROVIDER_META_HANDLE *p = log->provider =
provider_get(ev->provider, content[EvtSystemProviderName].StringVal);
ev->platform = provider_get_platform(p);
wevt_get_level(log, ev, p);
wevt_get_task(log, ev, p);
wevt_get_opcode(log, ev, p);
wevt_get_keyword(log, ev, p);
if(log->type & WEVT_QUERY_EVENT_DATA && wEvtRender(log, log->hRenderUserContext, &log->ops.raw.user)) {
#if (ON_FTS_PRELOAD_MESSAGE == 1)
EvtFormatMessage_Event_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.event);
#endif
#if (ON_FTS_PRELOAD_XML == 1)
EvtFormatMessage_Xml_utf8(&log->ops.unicode, log->provider, log->hEvent, &log->ops.xml);
#endif
#if (ON_FTS_PRELOAD_EVENT_DATA == 1)
for(size_t i = 0; i < log->ops.raw.user.count ;i++)
evt_variant_to_buffer(log->ops.event_data, &log->ops.raw.user.data[i], " ||| ");
#endif
}
}
ret = true;
@ -412,11 +405,11 @@ cleanup:
return ret;
}
bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
DWORD size = full ? BATCH_NEXT_EVENT : 1;
bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev) {
DWORD size = (log->type & WEVT_QUERY_EXTENDED) ? BATCH_NEXT_EVENT : 1;
DWORD max_failures = 10;
fatal_assert(log && log->hQuery && log->hRenderContext);
fatal_assert(log && log->hQuery && log->hRenderSystemContext);
while(max_failures > 0) {
if (log->batch.used >= log->batch.size) {
@ -433,7 +426,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
if(size == 1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtNext() failed, hQuery: 0x%lx, size: %zu, extended info: %s",
(uintptr_t)log->hQuery, (size_t)size, wevt_extended_status());
(uintptr_t)log->hQuery, (size_t)size, EvtGetExtendedStatus_utf8());
return false;
}
@ -455,7 +448,7 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
log->batch.hEvents[log->batch.used] = NULL;
log->batch.used++;
if(wevt_get_next_event_one(log, ev, full))
if(wevt_get_next_event_one(log, ev))
return true;
else {
log->query_stats.failed_count++;
@ -467,6 +460,69 @@ bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full) {
return false;
}
static void wevt_event_done(WEVT_LOG *log) {
if (log->provider) {
provider_release(log->provider);
log->provider = NULL;
}
if (log->hEvent) {
EvtClose(log->hEvent);
log->hEvent = NULL;
}
log->ops.channel.src = TXT_SOURCE_UNKNOWN;
log->ops.provider.src = TXT_SOURCE_UNKNOWN;
log->ops.computer.src = TXT_SOURCE_UNKNOWN;
log->ops.account.src = TXT_SOURCE_UNKNOWN;
log->ops.domain.src = TXT_SOURCE_UNKNOWN;
log->ops.sid.src = TXT_SOURCE_UNKNOWN;
log->ops.event.src = TXT_SOURCE_UNKNOWN;
log->ops.level.src = TXT_SOURCE_UNKNOWN;
log->ops.keywords.src = TXT_SOURCE_UNKNOWN;
log->ops.opcode.src = TXT_SOURCE_UNKNOWN;
log->ops.task.src = TXT_SOURCE_UNKNOWN;
log->ops.xml.src = TXT_SOURCE_UNKNOWN;
log->ops.channel.used = 0;
log->ops.provider.used = 0;
log->ops.computer.used = 0;
log->ops.account.used = 0;
log->ops.domain.used = 0;
log->ops.sid.used = 0;
log->ops.event.used = 0;
log->ops.level.used = 0;
log->ops.keywords.used = 0;
log->ops.opcode.used = 0;
log->ops.task.used = 0;
log->ops.xml.used = 0;
if(log->ops.event_data)
log->ops.event_data->len = 0;
}
// --------------------------------------------------------------------------------------------------------------------
// Query management
bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
wevt_query_done(log);
log->log_stats.queries_count++;
EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
if (!hQuery) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
query2utf8(query), EvtGetExtendedStatus_utf8());
log->log_stats.queries_failed++;
return false;
}
log->hQuery = hQuery;
return true;
}
void wevt_query_done(WEVT_LOG *log) {
// close the last working hEvent
wevt_event_done(log);
@ -490,19 +546,59 @@ void wevt_query_done(WEVT_LOG *log) {
log->query_stats.failed_count = 0;
}
// --------------------------------------------------------------------------------------------------------------------
// Log management
WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type) {
WEVT_LOG *log = callocz(1, sizeof(*log));
log->type = type;
// create the system render
log->hRenderSystemContext = EvtCreateRenderContext(0, NULL, EvtRenderContextSystem);
if (!log->hRenderSystemContext) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtCreateRenderContext() on system context failed, extended info: %s",
EvtGetExtendedStatus_utf8());
goto cleanup;
}
if(type & WEVT_QUERY_EVENT_DATA) {
log->hRenderUserContext = EvtCreateRenderContext(0, NULL, EvtRenderContextUser);
if (!log->hRenderUserContext) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"EvtCreateRenderContext failed, on user context failed, extended info: %s",
EvtGetExtendedStatus_utf8());
goto cleanup;
}
log->ops.event_data = buffer_create(4096, NULL);
}
return log;
cleanup:
wevt_closelog6(log);
return NULL;
}
void wevt_closelog6(WEVT_LOG *log) {
wevt_query_done(log);
if (log->hRenderContext)
EvtClose(log->hRenderContext);
if (log->hRenderSystemContext)
EvtClose(log->hRenderSystemContext);
wevt_variant_cleanup(&log->ops.content);
if (log->hRenderUserContext)
EvtClose(log->hRenderUserContext);
wevt_variant_cleanup(&log->ops.raw.system);
wevt_variant_cleanup(&log->ops.raw.user);
txt_unicode_cleanup(&log->ops.unicode);
txt_utf8_cleanup(&log->ops.channel);
txt_utf8_cleanup(&log->ops.provider);
txt_utf8_cleanup(&log->ops.source);
txt_utf8_cleanup(&log->ops.computer);
txt_utf8_cleanup(&log->ops.user);
txt_utf8_cleanup(&log->ops.account);
txt_utf8_cleanup(&log->ops.domain);
txt_utf8_cleanup(&log->ops.sid);
txt_utf8_cleanup(&log->ops.event);
txt_utf8_cleanup(&log->ops.level);
@ -510,9 +606,15 @@ void wevt_closelog6(WEVT_LOG *log) {
txt_utf8_cleanup(&log->ops.opcode);
txt_utf8_cleanup(&log->ops.task);
txt_utf8_cleanup(&log->ops.xml);
buffer_free(log->ops.event_data);
freez(log);
}
// --------------------------------------------------------------------------------------------------------------------
// Retention
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention) {
bool ret = false;
@ -525,15 +627,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
if (!log->hQuery) {
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, cannot get retention, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
else
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, cannot get retention, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
if (!wevt_get_next_event(log, &retention->first_event, false))
if (!wevt_get_next_event(log, &retention->first_event))
goto cleanup;
if (!retention->first_event.id) {
@ -548,15 +650,15 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
if (!log->hQuery) {
if (GetLastError() == ERROR_EVT_CHANNEL_NOT_FOUND)
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention failed, channel '%s' not found, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
else
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() for retention on channel '%s' failed, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
if (!wevt_get_next_event(log, &retention->last_event, false) || retention->last_event.id == 0) {
if (!wevt_get_next_event(log, &retention->last_event) || retention->last_event.id == 0) {
// no data in eventlog
retention->last_event = retention->first_event;
}
@ -582,24 +684,6 @@ cleanup:
return ret;
}
WEVT_LOG *wevt_openlog6(void) {
size_t RENDER_ITEMS_count = (sizeof(RENDER_ITEMS) / sizeof(const wchar_t *));
WEVT_LOG *log = callocz(1, sizeof(*log));
// create the system render
log->hRenderContext = EvtCreateRenderContext(RENDER_ITEMS_count, RENDER_ITEMS, EvtRenderContextValues);
if (!log->hRenderContext) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtCreateRenderContext failed, extended info: %s", wevt_extended_status());
freez(log);
log = NULL;
goto cleanup;
}
cleanup:
return log;
}
static uint64_t wevt_log_file_size(const wchar_t *channel) {
EVT_HANDLE hLog = NULL;
EVT_VARIANT evtVariant;
@ -610,14 +694,14 @@ static uint64_t wevt_log_file_size(const wchar_t *channel) {
hLog = EvtOpenLog(NULL, channel, EvtOpenChannelPath);
if (!hLog) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtOpenLog() on channel '%s' failed, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
// Get the file size of the log
if (!EvtGetLogInfo(hLog, EvtLogFileSize, sizeof(evtVariant), &evtVariant, &bufferUsed)) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetLogInfo() on channel '%s' failed, extended info: %s",
channel2utf8(channel), wevt_extended_status());
channel2utf8(channel), EvtGetExtendedStatus_utf8());
goto cleanup;
}
@ -630,20 +714,3 @@ cleanup:
return file_size;
}
bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction) {
wevt_query_done(log);
log->log_stats.queries_count++;
EVT_HANDLE hQuery = EvtQuery(NULL, channel, query, EvtQueryChannelPath | (direction & (EvtQueryReverseDirection | EvtQueryForwardDirection)) | EvtQueryTolerateQueryErrors);
if (!hQuery) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtQuery() failed, query: %s | extended info: %s",
query2utf8(query), wevt_extended_status());
log->log_stats.queries_failed++;
return false;
}
log->hQuery = hQuery;
return true;
}

99
src/collectors/windows-events.plugin/windows-events-query.h
View file

@ -4,6 +4,7 @@
#define NETDATA_WINDOWS_EVENTS_QUERY_H
#include "libnetdata/libnetdata.h"
#include "windows-events.h"
#define BATCH_NEXT_EVENT 500
@ -11,23 +12,27 @@ typedef struct wevt_event {
uint64_t id; // EventRecordId (unique and sequential per channel)
uint8_t version;
uint8_t level; // The severity of event
uint8_t opcode; // we receive this as 8bit, but publishers use 32bit
uint8_t opcode; // we receive this as 8bit, but providers use 32bit
uint16_t event_id; // This is the template that defines the message to be shown
uint16_t task;
uint16_t qualifiers;
uint32_t process_id;
uint32_t thread_id;
uint64_t keywords; // Categorization of the event
ND_UUID provider;
ND_UUID correlation_activity_id;
ND_UUID activity_id;
ND_UUID related_activity_id;
nsec_t created_ns;
WEVT_PROVIDER_PLATFORM platform;
} WEVT_EVENT;
#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
typedef struct {
EVT_VARIANT *data;
size_t size;
size_t used;
DWORD size;
DWORD used;
DWORD count;
} WEVT_VARIANT;
typedef struct {
@ -41,6 +46,16 @@ typedef struct {
struct provider_meta_handle;
typedef enum __attribute__((packed)) {
WEVT_QUERY_BASIC = (1 << 0),
WEVT_QUERY_EXTENDED = (1 << 1),
WEVT_QUERY_EVENT_DATA = (1 << 2),
} WEVT_QUERY_TYPE;
#define WEVT_QUERY_RETENTION WEVT_QUERY_BASIC
#define WEVT_QUERY_NORMAL (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED)
#define WEVT_QUERY_FTS (WEVT_QUERY_BASIC | WEVT_QUERY_EXTENDED | WEVT_QUERY_EVENT_DATA)
typedef struct wevt_log {
struct {
DWORD size;
@ -50,13 +65,19 @@ typedef struct wevt_log {
EVT_HANDLE hEvent;
EVT_HANDLE hQuery;
EVT_HANDLE hRenderContext;
struct provider_meta_handle *publisher;
EVT_HANDLE hRenderSystemContext;
EVT_HANDLE hRenderUserContext;
struct provider_meta_handle *provider;
WEVT_QUERY_TYPE type;
struct {
// temp buffer used for rendering event log messages
// never use directly
WEVT_VARIANT content;
struct {
// temp buffer used for rendering event log messages
// never use directly
WEVT_VARIANT system;
WEVT_VARIANT user;
} raw;
// temp buffer used for fetching and converting UNICODE and UTF-8
// every string operation overwrites it, multiple times per event log entry
@ -75,16 +96,19 @@ typedef struct wevt_log {
TXT_UTF8 channel;
TXT_UTF8 provider;
TXT_UTF8 source;
TXT_UTF8 computer;
TXT_UTF8 user;
TXT_UTF8 account;
TXT_UTF8 domain;
TXT_UTF8 sid;
TXT_UTF8 event;
TXT_UTF8 event; // the message to be shown to the user
TXT_UTF8 level;
TXT_UTF8 keywords;
TXT_UTF8 opcode;
TXT_UTF8 task;
TXT_UTF8 xml;
BUFFER *event_data;
} ops;
struct {
@ -102,7 +126,7 @@ typedef struct wevt_log {
} WEVT_LOG;
WEVT_LOG *wevt_openlog6(void);
WEVT_LOG *wevt_openlog6(WEVT_QUERY_TYPE type);
void wevt_closelog6(WEVT_LOG *log);
bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t *query, EVT_RETENTION *retention);
@ -110,12 +134,14 @@ bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, const wchar_t
bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction);
void wevt_query_done(WEVT_LOG *log);
bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev);
bool wevt_get_message_unicode(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
bool wevt_get_event_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
bool wevt_get_xml_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst);
void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator);
static inline void wevt_variant_cleanup(WEVT_VARIANT *v) {
freez(v->data);
@ -130,6 +156,10 @@ static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) {
v->data = mallocz(v->size);
}
static inline void wevt_variant_count_from_used(WEVT_VARIANT *v) {
v->count = v->used / sizeof(*v->data);
}
static inline uint8_t wevt_field_get_uint8(EVT_VARIANT *ev) {
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull)
return 0;
@ -180,16 +210,17 @@ static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) {
return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1);
}
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst) {
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
wevt_utf8_empty(dst);
wevt_utf8_empty(dst_account);
wevt_utf8_empty(dst_domain);
wevt_utf8_empty(dst_sid_str);
return false;
}
fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid);
return wevt_convert_user_id_to_name(ev->SidVal, dst);
return wevt_convert_user_id_to_name(ev->SidVal, dst_account, dst_domain, dst_sid_str);
}
static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) {
@ -222,42 +253,42 @@ static inline bool wevt_get_uuid_by_type(EVT_VARIANT *ev, ND_UUID *dst) {
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-severity-levels
static inline bool is_valid_publisher_level(uint64_t level, bool strict) {
static inline bool is_valid_provider_level(uint64_t level, bool strict) {
if(strict)
// when checking if the name is publisher independent
// when checking if the name is provider independent
return level >= 16 && level <= 255;
else
// when checking acceptable values in publisher manifests
// when checking acceptable values in provider manifests
return level <= 255;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
static inline bool is_valid_publisher_opcode(uint64_t opcode, bool strict) {
static inline bool is_valid_provider_opcode(uint64_t opcode, bool strict) {
if(strict)
// when checking if the name is publisher independent
// when checking if the name is provider independent
return opcode >= 10 && opcode <= 239;
else
// when checking acceptable values in publisher manifests
// when checking acceptable values in provider manifests
return opcode <= 255;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
static inline bool is_valid_publisher_task(uint64_t task, bool strict) {
static inline bool is_valid_provider_task(uint64_t task, bool strict) {
if(strict)
// when checking if the name is publisher independent
// when checking if the name is provider independent
return task > 0 && task <= 0xFFFF;
else
// when checking acceptable values in publisher manifests
// when checking acceptable values in provider manifests
return task <= 0xFFFF;
}
// https://learn.microsoft.com/en-us/windows/win32/wes/defining-keywords-used-to-classify-types-of-events
static inline bool is_valid_publisher_keywords(uint64_t keyword, bool strict) {
static inline bool is_valid_provider_keyword(uint64_t keyword, bool strict) {
if(strict)
// when checking if the name is publisher independent
// when checking if the name is provider independent
return keyword > 0 && keyword <= 0x0000FFFFFFFFFFFF;
else
// when checking acceptable values in publisher manifests
// when checking acceptable values in provider manifests
return true;
}

137
src/collectors/windows-events.plugin/windows-events-sid.c
View file

@ -9,8 +9,18 @@ typedef struct {
} SID_KEY;
typedef struct {
const char *user;
size_t user_len;
// IMPORTANT:
// This is malloc'd ! You have to manually set fields to zero.
const char *account;
const char *domain;
const char *full;
const char *sid_str;
uint32_t account_len;
uint32_t domain_len;
uint32_t full_len;
uint32_t sid_str_len;
// this needs to be last, because of its variable size
SID_KEY key;
@ -43,48 +53,46 @@ void sid_cache_init(void) {
simple_hashtable_init_SID(&sid_globals.hashtable, 100);
}
static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
if(found && found->user) {
txt_utf8_resize(dst, found->user_len + 1, false);
memcpy(dst->data, found->user, found->user_len + 1);
dst->used = found->user_len + 1;
return true;
}
txt_utf8_resize(dst, 1, false);
dst->data[0] = '\0';
dst->used = 1;
return false;
}
static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
static void lookup_user(SID_VALUE *sv) {
static __thread wchar_t account_unicode[256];
static __thread wchar_t domain_unicode[256];
static __thread char tmp[512 + 2];
DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
SID_NAME_USE sid_type;
txt_utf8_resize(dst, 1024, false);
if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
const char *user = account2utf8(account_unicode);
if (LookupAccountSidW(NULL, sv->key.sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
const char *account = account2utf8(account_unicode);
const char *domain = domain2utf8(domain_unicode);
dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
snprintfz(tmp, sizeof(tmp), "%s\\%s", domain, account);
sv->domain = strdupz(domain); sv->domain_len = strlen(sv->domain);
sv->account = strdupz(account); sv->account_len = strlen(sv->account);
sv->full = strdupz(tmp); sv->full_len = strlen(sv->full);
}
else {
wchar_t *sid_string = NULL;
if (ConvertSidToStringSidW(sid, &sid_string)) {
const char *user = account2utf8(sid_string);
dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
}
else
dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
sv->domain = NULL;
sv->account = NULL;
sv->full = NULL;
sv->domain_len = 0;
sv->account_len = 0;
sv->full_len = 0;
}
wchar_t *sid_string = NULL;
if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) {
sv->sid_str = strdupz(account2utf8(sid_string));
sv->sid_str_len = strlen(sv->sid_str);
}
else {
sv->sid_str = NULL;
sv->sid_str_len = 0;
}
}
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) {
if(!sid || !IsValidSid(sid))
return update_user(NULL, dst);
return NULL;
size_t size = GetLengthSid(sid);
@ -98,21 +106,76 @@ bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
spinlock_lock(&sid_globals.spinlock);
SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
spinlock_unlock(&sid_globals.spinlock);
if(found) return update_user(found, dst);
if(found) return found;
// allocate the SID_VALUE
found = mallocz(tmp_size);
memcpy(found, buf, tmp_size);
// lookup the user
lookup_user(sid, dst);
found->user = strdupz(dst->data);
found->user_len = dst->used - 1;
lookup_user(found);
// add it to the cache
spinlock_lock(&sid_globals.spinlock);
simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
spinlock_unlock(&sid_globals.spinlock);
return update_user(found, dst);
return found;
}
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) {
SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
if(found) {
if (found->account) {
txt_utf8_resize(dst_account, found->account_len + 1, false);
memcpy(dst_account->data, found->account, found->account_len + 1);
dst_account->used = found->account_len + 1;
}
else wevt_utf8_empty(dst_account);
if (found->domain) {
txt_utf8_resize(dst_domain, found->domain_len + 1, false);
memcpy(dst_domain->data, found->domain, found->domain_len + 1);
dst_domain->used = found->domain_len + 1;
}
else wevt_utf8_empty(dst_domain);
if (found->sid_str) {
txt_utf8_resize(dst_sid_str, found->sid_str_len + 1, false);
memcpy(dst_sid_str->data, found->sid_str, found->sid_str_len + 1);
dst_sid_str->used = found->sid_str_len + 1;
}
else wevt_utf8_empty(dst_sid_str);
return true;
}
wevt_utf8_empty(dst_account);
wevt_utf8_empty(dst_domain);
wevt_utf8_empty(dst_sid_str);
return false;
}
bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix) {
SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid);
size_t added = 0;
if(found) {
if (found->full) {
if (prefix && *prefix)
buffer_strcat(dst, prefix);
buffer_fast_strcat(dst, found->full, found->full_len);
added++;
}
if (found->sid_str) {
if (prefix && *prefix)
buffer_strcat(dst, prefix);
buffer_fast_strcat(dst, found->sid_str, found->sid_str_len);
added++;
}
}
return added > 0;
}

3
src/collectors/windows-events.plugin/windows-events-sid.h
View file

@ -6,7 +6,8 @@
#include "windows-events.h"
struct wevt_log;
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str);
bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix);
void sid_cache_init(void);
#endif //NETDATA_WINDOWS_EVENTS_SID_H

308
src/collectors/windows-events.plugin/windows-events-sources.c
View file

@ -137,41 +137,32 @@
// }
//};
ENUM_STR_MAP_DEFINE(WEVT_SOURCE_TYPE) = {
{ .id = WEVTS_ALL, .name = WEVT_SOURCE_ALL_NAME },
{ .id = WEVTS_ADMIN, .name = WEVT_SOURCE_ALL_ADMIN_NAME },
{ .id = WEVTS_OPERATIONAL, .name = WEVT_SOURCE_ALL_OPERATIONAL_NAME },
{ .id = WEVTS_ANALYTIC, .name = WEVT_SOURCE_ALL_ANALYTIC_NAME },
{ .id = WEVTS_DEBUG, .name = WEVT_SOURCE_ALL_DEBUG_NAME },
{ .id = WEVTS_WINDOWS, .name = WEVT_SOURCE_ALL_WINDOWS_NAME },
{ .id = WEVTS_ENABLED, .name = WEVT_SOURCE_ALL_ENABLED_NAME },
{ .id = WEVTS_DISABLED, .name = WEVT_SOURCE_ALL_DISABLED_NAME },
{ .id = WEVTS_FORWARDED, .name = WEVT_SOURCE_ALL_FORWARDED_NAME },
{ .id = WEVTS_CLASSIC, .name = WEVT_SOURCE_ALL_CLASSIC_NAME },
{ .id = WEVTS_BACKUP_MODE, .name = WEVT_SOURCE_ALL_BACKUP_MODE_NAME },
{ .id = WEVTS_OVERWRITE_MODE, .name = WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME },
{ .id = WEVTS_STOP_WHEN_FULL_MODE, .name = WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME },
{ .id = WEVTS_RETAIN_AND_BACKUP_MODE, .name = WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME },
// terminator
{ . id = 0, .name = NULL }
};
BITMAP_STR_DEFINE_FUNCTIONS(WEVT_SOURCE_TYPE, WEVTS_NONE, "");
DICTIONARY *wevt_sources = NULL;
DICTIONARY *used_hashes_registry = NULL;
static usec_t wevt_session = 0;
WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
return WEVTS_ALL;
if(strcmp(value, WEVT_SOURCE_ALL_ADMIN_NAME) == 0)
return WEVTS_ADMIN;
if(strcmp(value, WEVT_SOURCE_ALL_OPERATIONAL_NAME) == 0)
return WEVTS_OPERATIONAL;
if(strcmp(value, WEVT_SOURCE_ALL_ANALYTIC_NAME) == 0)
return WEVTS_ANALYTIC;
if(strcmp(value, WEVT_SOURCE_ALL_DEBUG_NAME) == 0)
return WEVTS_DEBUG;
if(strcmp(value, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME) == 0)
return WEVTS_DIAGNOSTIC;
if(strcmp(value, WEVT_SOURCE_ALL_TRACING_NAME) == 0)
return WEVTS_TRACING;
if(strcmp(value, WEVT_SOURCE_ALL_PERFORMANCE_NAME) == 0)
return WEVTS_PERFORMANCE;
if(strcmp(value, WEVT_SOURCE_ALL_WINDOWS_NAME) == 0)
return WEVTS_WINDOWS;
return WEVTS_NONE;
}
void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
LOGS_QUERY_SOURCE *src = value;
freez((void *)src->fullname);
@ -265,7 +256,14 @@ int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
// --------------------------------------------------------------------------------------------------------------------
typedef enum {
wevt_source_type_internal,
wevt_source_type_provider,
wevt_source_type_channel,
} wevt_source_type;
struct wevt_source {
wevt_source_type type;
usec_t first_ut;
usec_t last_ut;
size_t count;
@ -279,6 +277,15 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
const char *name = dictionary_acquired_item_name(item);
if(s->count == 1 && strncmp(name, WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX, sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) - 1) == 0)
// do not include "All-Of-X" when there is only 1 channel
return 0;
bool default_selected = (s->type == wevt_source_type_channel);
if(default_selected && (strcmp(name, "NetdataWEL") == 0 || strcmp(name, "Netdata/Access") == 0))
// do not select Netdata Access logs by default
default_selected = false;
buffer_json_add_array_item_object(wb);
{
char size_for_humans[128];
@ -300,6 +307,7 @@ static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry
buffer_json_member_add_string(wb, "name", name);
buffer_json_member_add_string(wb, "pill", size_for_humans);
buffer_json_member_add_string(wb, "info", info);
buffer_json_member_add_boolean(wb, "default_selected", default_selected);
}
buffer_json_object_close(wb); // options object
@ -337,61 +345,142 @@ void wevt_sources_to_json_array(BUFFER *wb) {
t.size = src->size;
t.entries = src->entries;
dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
src->source_type |= WEVTS_ALL;
t.type = wevt_source_type_internal;
for(size_t i = 0; WEVT_SOURCE_TYPE_names[i].name ;i++) {
if(src->source_type & WEVT_SOURCE_TYPE_names[i].id)
dictionary_set(dict, WEVT_SOURCE_TYPE_names[i].name, &t, sizeof(t));
}
if(src->source_type & WEVTS_ADMIN)
dictionary_set(dict, WEVT_SOURCE_ALL_ADMIN_NAME, &t, sizeof(t));
if(src->provider) {
t.type = wevt_source_type_provider;
dictionary_set(dict, string2str(src->provider), &t, sizeof(t));
}
if(src->source_type & WEVTS_OPERATIONAL)
dictionary_set(dict, WEVT_SOURCE_ALL_OPERATIONAL_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_ANALYTIC)
dictionary_set(dict, WEVT_SOURCE_ALL_ANALYTIC_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_DEBUG)
dictionary_set(dict, WEVT_SOURCE_ALL_DEBUG_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_DIAGNOSTIC)
dictionary_set(dict, WEVT_SOURCE_ALL_DIAGNOSTIC_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_TRACING)
dictionary_set(dict, WEVT_SOURCE_ALL_TRACING_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_PERFORMANCE)
dictionary_set(dict, WEVT_SOURCE_ALL_PERFORMANCE_NAME, &t, sizeof(t));
if(src->source_type & WEVTS_WINDOWS)
dictionary_set(dict, WEVT_SOURCE_ALL_WINDOWS_NAME, &t, sizeof(t));
if(src->source)
if(src->source) {
t.type = wevt_source_type_channel;
dictionary_set(dict, string2str(src->source), &t, sizeof(t));
}
}
dfe_done(jf);
dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
}
static bool check_and_remove_suffix(char *name, size_t len, const char *suffix) {
char s[strlen(suffix) + 2];
s[0] = '/';
memcpy(&s[1], suffix, sizeof(s) - 1);
size_t slen = sizeof(s) - 1;
if(slen + 1 >= len) return false;
char *match = &name[len - slen];
if(strcasecmp(match, s) == 0) {
*match = '\0';
return true;
static bool ndEvtGetChannelConfigProperty(EVT_HANDLE hChannelConfig, WEVT_VARIANT *pr, EVT_CHANNEL_CONFIG_PROPERTY_ID id) {
if (!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
DWORD status = GetLastError();
if (ERROR_INSUFFICIENT_BUFFER == status) {
wevt_variant_resize(pr, pr->used);
if(!EvtGetChannelConfigProperty(hChannelConfig, id, 0, pr->size, pr->data, &pr->used)) {
pr->used = 0;
pr->count = 0;
return false;
}
}
}
s[0] = '-';
if(strcasecmp(match, s) == 0) {
*match = '\0';
return true;
wevt_variant_count_from_used(pr);
return true;
}
WEVT_SOURCE_TYPE categorize_channel(const wchar_t *channel_path, const char **provider, WEVT_VARIANT *property) {
EVT_HANDLE hChannelConfig = NULL;
WEVT_SOURCE_TYPE result = WEVTS_ALL;
// Open the channel configuration
hChannelConfig = EvtOpenChannelConfig(NULL, channel_path, 0);
if (!hChannelConfig)
goto cleanup;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigType) &
property->count &&
property->data[0].Type == EvtVarTypeUInt32) {
switch (property->data[0].UInt32Val) {
case EvtChannelTypeAdmin:
result |= WEVTS_ADMIN;
break;
case EvtChannelTypeOperational:
result |= WEVTS_OPERATIONAL;
break;
case EvtChannelTypeAnalytic:
result |= WEVTS_ANALYTIC;
break;
case EvtChannelTypeDebug:
result |= WEVTS_DEBUG;
break;
default:
break;
}
}
return false;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigClassicEventlog) &&
property->count &&
property->data[0].Type == EvtVarTypeBoolean &&
property->data[0].BooleanVal)
result |= WEVTS_CLASSIC;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigOwningPublisher) &&
property->count &&
property->data[0].Type == EvtVarTypeString) {
*provider = provider2utf8(property->data[0].StringVal);
if(wcscasecmp(property->data[0].StringVal, L"Microsoft-Windows-EventCollector") == 0)
result |= WEVTS_FORWARDED;
}
else
*provider = NULL;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelConfigEnabled) &&
property->count &&
property->data[0].Type == EvtVarTypeBoolean) {
if(property->data[0].BooleanVal)
result |= WEVTS_ENABLED;
else
result |= WEVTS_DISABLED;
}
bool got_retention = false;
bool retained = false;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigRetention) &&
property->count &&
property->data[0].Type == EvtVarTypeBoolean) {
got_retention = true;
retained = property->data[0].BooleanVal;
}
bool got_auto_backup = false;
bool auto_backup = false;
if(ndEvtGetChannelConfigProperty(hChannelConfig, property, EvtChannelLoggingConfigAutoBackup) &&
property->count &&
property->data[0].Type == EvtVarTypeBoolean) {
got_auto_backup = true;
auto_backup = property->data[0].BooleanVal;
}
if(got_retention && got_auto_backup) {
if(!retained) {
if(auto_backup)
result |= WEVTS_BACKUP_MODE;
else
result |= WEVTS_OVERWRITE_MODE;
}
else {
if(auto_backup)
result |= WEVTS_STOP_WHEN_FULL_MODE;
else
result |= WEVTS_RETAIN_AND_BACKUP_MODE;
}
}
cleanup:
if (hChannelConfig)
EvtClose(hChannelConfig);
return result;
}
void wevt_sources_scan(void) {
@ -400,8 +489,9 @@ void wevt_sources_scan(void) {
EVT_HANDLE hChannelEnum = NULL;
if(spinlock_trylock(&spinlock)) {
const usec_t now_monotonic_ut = now_monotonic_usec();
const usec_t started_ut = now_monotonic_usec();
WEVT_VARIANT property = { 0 };
DWORD dwChannelBufferSize = 0;
DWORD dwChannelBufferUsed = 0;
DWORD status = ERROR_SUCCESS;
@ -414,7 +504,7 @@ void wevt_sources_scan(void) {
goto cleanup;
}
WEVT_LOG *log = wevt_openlog6();
WEVT_LOG *log = wevt_openlog6(WEVT_QUERY_RETENTION);
if(!log) goto cleanup;
while (true) {
@ -438,32 +528,26 @@ void wevt_sources_scan(void) {
if(!wevt_channel_retention(log, channel, NULL, &retention))
continue;
LOGS_QUERY_SOURCE *found = dictionary_get(wevt_sources, channel2utf8(channel));
if(found) {
// we just need to update its retention
found->last_scan_monotonic_ut = now_monotonic_usec();
found->msg_first_id = retention.first_event.id;
found->msg_last_id = retention.last_event.id;
found->msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC;
found->msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC;
found->size = retention.size_bytes;
continue;
}
const char *name = channel2utf8(channel);
const char *fullname = strdupz(name);
const char *provider;
WEVT_SOURCE_TYPE sources = WEVTS_ALL;
size_t len = strlen(fullname);
if(check_and_remove_suffix((char *)name, len, "Admin"))
sources |= WEVTS_ADMIN;
else if(check_and_remove_suffix((char *)name, len, "Operational"))
sources |= WEVTS_OPERATIONAL;
else if(check_and_remove_suffix((char *)name, len, "Analytic"))
sources |= WEVTS_ANALYTIC;
else if(check_and_remove_suffix((char *)name, len, "Debug") ||
check_and_remove_suffix((char *)name, len, "Verbose"))
sources |= WEVTS_DEBUG;
else if(check_and_remove_suffix((char *)name, len, "Diagnostic"))
sources |= WEVTS_DIAGNOSTIC;
else if(check_and_remove_suffix((char *)name, len, "Trace") ||
check_and_remove_suffix((char *)name, len, "Tracing"))
sources |= WEVTS_TRACING;
else if(check_and_remove_suffix((char *)name, len, "Performance") ||
check_and_remove_suffix((char *)name, len, "Perf"))
sources |= WEVTS_PERFORMANCE;
WEVT_SOURCE_TYPE sources = categorize_channel(channel, &provider, &property);
char *slash = strchr(name, '/');
if(slash)
*slash = '\0';
if(slash) *slash = '\0';
if(strcasecmp(name, "Application") == 0)
sources |= WEVTS_WINDOWS;
@ -485,9 +569,27 @@ void wevt_sources_scan(void) {
.msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
.size = retention.size_bytes,
.source_type = sources,
.source = string_strdupz(name),
.source = string_strdupz(fullname),
};
if(strncmp(fullname, "Netdata", 7) == 0)
// WEL based providers of Netdata are named NetdataX
provider = "Netdata";
if(provider && *provider) {
char buf[sizeof(WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX) + strlen(provider)]; // sizeof() includes terminator
snprintf(buf, sizeof(buf), WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "%s", provider);
if(trim_all(buf) != NULL) {
for (size_t i = 0; i < sizeof(buf) - 1; i++) {
// remove character that may interfere with our parsing
if (isspace((uint8_t) buf[i]) || buf[i] == '%' || buf[i] == '+' || buf[i] == '|' || buf[i] == ':')
buf[i] = '_';
}
src.provider = string_strdupz(buf);
}
}
dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
}
@ -519,13 +621,21 @@ void wevt_sources_scan(void) {
LOGS_QUERY_SOURCE *src;
dfe_start_write(wevt_sources, src)
{
if(src->last_scan_monotonic_ut < now_monotonic_ut)
if(src->last_scan_monotonic_ut < started_ut) {
src->msg_first_id = 0;
src->msg_last_id = 0;
src->msg_first_ut = 0;
src->msg_last_ut = 0;
src->size = 0;
dictionary_del(wevt_sources, src->fullname);
}
}
dfe_done(src);
dictionary_garbage_collect(wevt_sources);
spinlock_unlock(&spinlock);
wevt_variant_cleanup(&property);
}
cleanup:

55
src/collectors/windows-events.plugin/windows-events-sources.h
View file

@ -6,18 +6,42 @@
#include "libnetdata/libnetdata.h"
typedef enum {
WEVTS_NONE = 0,
WEVTS_ALL = (1 << 0),
WEVTS_ADMIN = (1 << 1),
WEVTS_OPERATIONAL = (1 << 2),
WEVTS_ANALYTIC = (1 << 3),
WEVTS_DEBUG = (1 << 4),
WEVTS_DIAGNOSTIC = (1 << 5),
WEVTS_TRACING = (1 << 6),
WEVTS_PERFORMANCE = (1 << 7),
WEVTS_WINDOWS = (1 << 8),
WEVTS_NONE = 0,
WEVTS_ALL = (1 << 0),
WEVTS_ADMIN = (1 << 1),
WEVTS_OPERATIONAL = (1 << 2),
WEVTS_ANALYTIC = (1 << 3),
WEVTS_DEBUG = (1 << 4),
WEVTS_WINDOWS = (1 << 5),
WEVTS_ENABLED = (1 << 6),
WEVTS_DISABLED = (1 << 7),
WEVTS_FORWARDED = (1 << 8),
WEVTS_CLASSIC = (1 << 9),
WEVTS_BACKUP_MODE = (1 << 10),
WEVTS_OVERWRITE_MODE = (1 << 11),
WEVTS_STOP_WHEN_FULL_MODE = (1 << 12),
WEVTS_RETAIN_AND_BACKUP_MODE = (1 << 13),
} WEVT_SOURCE_TYPE;
BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(WEVT_SOURCE_TYPE)
#define WEVT_SOURCE_ALL_NAME "All"
#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
#define WEVT_SOURCE_ALL_ENABLED_NAME "All-Enabled"
#define WEVT_SOURCE_ALL_DISABLED_NAME "All-Disabled"
#define WEVT_SOURCE_ALL_FORWARDED_NAME "All-Forwarded"
#define WEVT_SOURCE_ALL_CLASSIC_NAME "All-Classic"
#define WEVT_SOURCE_ALL_BACKUP_MODE_NAME "All-In-Backup-Mode"
#define WEVT_SOURCE_ALL_OVERWRITE_MODE_NAME "All-In-Overwrite-Mode"
#define WEVT_SOURCE_ALL_STOP_WHEN_FULL_MODE_NAME "All-In-StopWhenFull-Mode"
#define WEVT_SOURCE_ALL_RETAIN_AND_BACKUP_MODE_NAME "All-In-RetainAndBackup-Mode"
#define WEVT_SOURCE_ALL_OF_PROVIDER_PREFIX "All-Of-"
typedef struct {
const char *fullname;
size_t fullname_len;
@ -25,6 +49,7 @@ typedef struct {
const wchar_t *custom_query;
STRING *source;
STRING *provider;
WEVT_SOURCE_TYPE source_type;
usec_t msg_first_ut;
usec_t msg_last_ut;
@ -40,16 +65,6 @@ typedef struct {
extern DICTIONARY *wevt_sources;
extern DICTIONARY *used_hashes_registry;
#define WEVT_SOURCE_ALL_NAME "All"
#define WEVT_SOURCE_ALL_ADMIN_NAME "All-Admin"
#define WEVT_SOURCE_ALL_OPERATIONAL_NAME "All-Operational"
#define WEVT_SOURCE_ALL_ANALYTIC_NAME "All-Analytic"
#define WEVT_SOURCE_ALL_DEBUG_NAME "All-Debug"
#define WEVT_SOURCE_ALL_DIAGNOSTIC_NAME "All-Diagnostic"
#define WEVT_SOURCE_ALL_TRACING_NAME "All-Tracing"
#define WEVT_SOURCE_ALL_PERFORMANCE_NAME "All-Performance"
#define WEVT_SOURCE_ALL_WINDOWS_NAME "All-Windows"
void wevt_sources_init(void);
void wevt_sources_scan(void);
void buffer_json_wevt_versions(BUFFER *wb);

10
src/collectors/windows-events.plugin/windows-events-unicode.c
View file

@ -5,7 +5,7 @@
inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
if (src) {
// Convert from UTF-8 to wide char (UTF-16)
if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
if (utf8_to_utf16(dst, dst_size, src, -1) == 0)
wcsncpy(dst, L"[failed conv.]", dst_size - 1);
}
else
@ -41,7 +41,7 @@ char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) {
wchar_t *channel2unicode(const char *utf8str) {
static __thread wchar_t buffer[1024];
utf82unicode(buffer, sizeof(buffer) / sizeof(wchar_t), utf8str);
utf82unicode(buffer, _countof(buffer), utf8str);
return buffer;
}
@ -69,6 +69,12 @@ char *query2utf8(const wchar_t *query) {
return buffer;
}
char *provider2utf8(const wchar_t *provider) {
static __thread char buffer[256];
unicode2utf8(buffer, sizeof(buffer), provider);
return buffer;
}
bool wevt_str_wchar_to_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len_with_null) {
if(!src || !src_len_with_null)
goto cleanup;

3
src/collectors/windows-events.plugin/windows-events-unicode.h
View file

@ -9,7 +9,7 @@
typedef enum __attribute__((packed)) {
TXT_SOURCE_UNKNOWN = 0,
TXT_SOURCE_PUBLISHER,
TXT_SOURCE_PROVIDER,
TXT_SOURCE_FIELD_CACHE,
TXT_SOURCE_EVENT_LOG,
TXT_SOURCE_HARDCODED,
@ -177,6 +177,7 @@ char *channel2utf8(const wchar_t *channel);
wchar_t *channel2unicode(const char *utf8str);
char *query2utf8(const wchar_t *query);
char *provider2utf8(const wchar_t *provider);
char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len);

270
src/collectors/windows-events.plugin/windows-events.c
View file

@ -18,14 +18,17 @@ static bool plugin_should_exit = false;
#define WEVT_KEYS_INCLUDED_IN_FACETS \
"|" WEVT_FIELD_COMPUTER \
"|" WEVT_FIELD_PROVIDER \
"|" WEVT_FIELD_SOURCE \
"|" WEVT_FIELD_LEVEL \
"|" WEVT_FIELD_KEYWORDS \
"|" WEVT_FIELD_OPCODE \
"|" WEVT_FIELD_TASK \
"|" WEVT_FIELD_USER \
"|" WEVT_FIELD_ACCOUNT \
"|" WEVT_FIELD_DOMAIN \
"|" WEVT_FIELD_SID \
""
#define query_has_fts(lqs) ((lqs)->rq.query != NULL)
static inline WEVT_QUERY_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) {
if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
nd_log(NDLS_COLLECTORS, NDLP_INFO, "Function has been cancelled");
@ -66,9 +69,10 @@ FACET_ROW_SEVERITY wevt_levelid_to_facet_severity(FACETS *facets __maybe_unused,
struct wevt_bin_data {
bool rendered;
WEVT_EVENT ev;
WEVT_LOG *log;
EVT_HANDLE hEvent;
PROVIDER_META_HANDLE *publisher;
PROVIDER_META_HANDLE *provider;
};
static void wevt_cleanup_bin_data(void *data) {
@ -77,21 +81,30 @@ static void wevt_cleanup_bin_data(void *data) {
if(d->hEvent)
EvtClose(d->hEvent);
publisher_release(d->publisher);
provider_release(d->provider);
freez(d);
}
static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev __maybe_unused) {
static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets, WEVT_EVENT *ev) {
struct wevt_bin_data *d = mallocz(sizeof(struct wevt_bin_data));
#ifdef NETDATA_INTERNAL_CHECKS
internal_fatal(strcmp(log->ops.provider.data, provider_get_name(log->provider)) != 0,
"Provider name mismatch in data!");
internal_fatal(!UUIDeq(ev->provider, provider_get_uuid(log->provider)),
"Provider UUID mismatch in data!");
#endif
d->ev = *ev;
d->log = log;
d->rendered = false;
// take the bookmark
d->hEvent = log->hEvent; log->hEvent = NULL;
// dup the publisher
d->publisher = publisher_dup(log->publisher);
// dup the provider
d->provider = provider_dup(log->provider);
facets_row_bin_data_set(facets, wevt_cleanup_bin_data, d);
}
@ -99,12 +112,26 @@ static inline void wevt_facets_register_bin_data(WEVT_LOG *log, FACETS *facets,
static void wevt_lazy_loading_event_and_xml(struct wevt_bin_data *d, FACET_ROW *row __maybe_unused) {
if(d->rendered) return;
wevt_get_xml_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.xml);
wevt_get_event_utf8(d->log, d->publisher, d->hEvent, &d->log->ops.event);
#ifdef NETDATA_INTERNAL_CHECKS
const FACET_ROW_KEY_VALUE *provider_rkv = dictionary_get(row->dict, WEVT_FIELD_PROVIDER);
internal_fatal(!provider_rkv || strcmp(buffer_tostring(provider_rkv->wb), provider_get_name(d->provider)) != 0,
"Provider of row does not match the bin data associated with it");
uint64_t event_record_id = UINT64_MAX;
const FACET_ROW_KEY_VALUE *event_record_id_rkv = dictionary_get(row->dict, WEVT_FIELD_EVENTRECORDID);
if(event_record_id_rkv)
event_record_id = str2uint64_t(buffer_tostring(event_record_id_rkv->wb), NULL);
internal_fatal(event_record_id != d->ev.id,
"Event Record ID of row does not match the bin data associated with it");
#endif
// the message needs the xml
EvtFormatMessage_Xml_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.xml);
EvtFormatMessage_Event_utf8(&d->log->ops.unicode, d->provider, d->hEvent, &d->log->ops.event);
d->rendered = true;
}
static void wevt_render_xml(
static void wevt_lazy_load_xml(
FACETS *facets,
BUFFER *json_array,
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@ -121,7 +148,7 @@ static void wevt_render_xml(
buffer_json_add_array_item_string(json_array, d->log->ops.xml.data);
}
static void wevt_render_message(
static void wevt_lazy_load_message(
FACETS *facets,
BUFFER *json_array,
FACET_ROW_KEY_VALUE *rkv __maybe_unused,
@ -217,18 +244,22 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
facets_register_key_name(
facets, WEVT_FIELD_CHANNEL,
FACET_KEY_OPTION_FTS);
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_PROVIDER,
rq->default_facet | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_SOURCE,
facets, WEVT_FIELD_ACCOUNT,
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_USER,
facets, WEVT_FIELD_DOMAIN,
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_SID,
rq->default_facet | FACET_KEY_OPTION_FTS);
facets_register_key_name(
@ -236,6 +267,11 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
rq->default_facet |
FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_EVENTS_API,
rq->default_facet |
FACET_KEY_OPTION_FTS);
facets_register_key_name(
facets, WEVT_FIELD_LEVEL,
rq->default_facet | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_EXPANDED_FILTER);
@ -277,14 +313,32 @@ static void wevt_register_fields(LOGS_QUERY_STATUS *lqs) {
FACET_KEY_OPTION_NONE);
facets_register_dynamic_key_name(
facets, WEVT_FIELD_MESSAGE,
facets,
WEVT_FIELD_MESSAGE,
FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE,
wevt_render_message, NULL);
wevt_lazy_load_message,
NULL);
facets_register_dynamic_key_name(
facets, WEVT_FIELD_XML,
FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
wevt_render_xml, NULL);
facets,
WEVT_FIELD_XML,
FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_PRETTY_XML,
wevt_lazy_load_xml,
NULL);
if(query_has_fts(lqs)) {
facets_register_key_name(
facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN,
FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
facets_register_key_name(
facets, WEVT_FIELD_EVENT_XML_HIDDEN,
FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
facets_register_key_name(
facets, WEVT_FIELD_EVENT_DATA_HIDDEN,
FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_HIDDEN | FACET_KEY_OPTION_NEVER_FACET);
}
#ifdef NETDATA_INTERNAL_CHECKS
facets_register_key_name(
@ -315,8 +369,8 @@ static const char *source_to_str(TXT_UTF8 *txt) {
case TXT_SOURCE_EVENT_LOG:
return "event-log";
case TXT_SOURCE_PUBLISHER:
return "publisher";
case TXT_SOURCE_PROVIDER:
return "provider";
case TXT_SOURCE_FIELD_CACHE:
return "fields-cache";
@ -327,21 +381,80 @@ static const char *source_to_str(TXT_UTF8 *txt) {
}
#endif
static const char *events_api_to_str(WEVT_PROVIDER_PLATFORM platform) {
switch(platform) {
case WEVT_PLATFORM_WEL:
return "Windows Event Log";
case WEVT_PLATFORM_ETW:
return "Event Tracing for Windows";
case WEVT_PLATFORM_TL:
return "TraceLogging";
default:
return "Unknown";
}
}
static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUERY_SOURCE *src, usec_t *msg_ut __maybe_unused, WEVT_EVENT *ev) {
size_t len, bytes = log->ops.content.used;
static __thread char uuid_str[UUID_STR_LEN];
size_t len, bytes = log->ops.raw.system.used + log->ops.raw.user.used;
if(!UUIDiszero(ev->provider)) {
uuid_unparse_lower(ev->provider.uuid, uuid_str);
facets_add_key_value_length(
facets, WEVT_FIELD_PROVIDER_GUID, sizeof(WEVT_FIELD_PROVIDER_GUID) - 1,
uuid_str, sizeof(uuid_str) - 1);
}
if(!UUIDiszero(ev->activity_id)) {
uuid_unparse_lower(ev->activity_id.uuid, uuid_str);
facets_add_key_value_length(
facets, WEVT_FIELD_ACTIVITY_ID, sizeof(WEVT_FIELD_ACTIVITY_ID) - 1,
uuid_str, sizeof(uuid_str) - 1);
}
if(!UUIDiszero(ev->related_activity_id)) {
uuid_unparse_lower(ev->related_activity_id.uuid, uuid_str);
facets_add_key_value_length(
facets, WEVT_FIELD_RELATED_ACTIVITY_ID, sizeof(WEVT_FIELD_RELATED_ACTIVITY_ID) - 1,
uuid_str, sizeof(uuid_str) - 1);
}
if(ev->qualifiers) {
static __thread char qualifiers[UINT64_HEX_MAX_LENGTH];
len = print_uint64_hex(qualifiers, ev->qualifiers);
bytes += len;
facets_add_key_value_length(
facets, WEVT_FIELD_QUALIFIERS, sizeof(WEVT_FIELD_QUALIFIERS) - 1,
qualifiers, len);
}
{
static __thread char event_record_id_str[UINT64_MAX_LENGTH];
len = print_uint64(event_record_id_str, ev->id);
bytes += len;
facets_add_key_value_length(
facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
event_record_id_str, len);
}
if(ev->version) {
static __thread char version[UINT64_MAX_LENGTH];
len = print_uint64(version, ev->version);
bytes += len;
facets_add_key_value_length(
facets, WEVT_FIELD_VERSION, sizeof(WEVT_FIELD_VERSION) - 1,
version, len);
}
if(log->ops.provider.used > 1) {
bytes += log->ops.provider.used * 2; // unicode is double
facets_add_key_value_length(
facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
log->ops.provider.data, log->ops.provider.used - 1);
}
if(log->ops.source.used > 1) {
bytes += log->ops.source.used * 2;
facets_add_key_value_length(
facets, WEVT_FIELD_SOURCE, sizeof(WEVT_FIELD_SOURCE) - 1,
log->ops.source.data, log->ops.source.used - 1);
facets, WEVT_FIELD_PROVIDER, sizeof(WEVT_FIELD_PROVIDER) - 1,
log->ops.provider.data, log->ops.provider.used - 1);
}
if(log->ops.channel.used > 1) {
@ -357,15 +470,6 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
src->fullname, src->fullname_len);
}
{
static __thread char event_record_id_str[UINT64_MAX_LENGTH];
len = print_uint64(event_record_id_str, ev->id);
bytes += len;
facets_add_key_value_length(
facets, WEVT_FIELD_EVENTRECORDID, sizeof(WEVT_FIELD_EVENTRECORDID) - 1,
event_record_id_str, len);
}
if(log->ops.level.used > 1) {
bytes += log->ops.level.used * 2;
facets_add_key_value_length(
@ -401,11 +505,28 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
log->ops.task.data, log->ops.task.used - 1);
}
if(log->ops.user.used > 1) {
bytes += log->ops.user.used * 2;
if(log->ops.account.used > 1) {
bytes += log->ops.account.used * 2;
facets_add_key_value_length(
facets, WEVT_FIELD_USER, sizeof(WEVT_FIELD_USER) - 1,
log->ops.user.data, log->ops.user.used - 1);
facets,
WEVT_FIELD_ACCOUNT, sizeof(WEVT_FIELD_ACCOUNT) - 1,
log->ops.account.data, log->ops.account.used - 1);
}
if(log->ops.domain.used > 1) {
bytes += log->ops.domain.used * 2;
facets_add_key_value_length(
facets,
WEVT_FIELD_DOMAIN, sizeof(WEVT_FIELD_DOMAIN) - 1,
log->ops.domain.data, log->ops.domain.used - 1);
}
if(log->ops.sid.used > 1) {
bytes += log->ops.sid.used * 2;
facets_add_key_value_length(
facets,
WEVT_FIELD_SID, sizeof(WEVT_FIELD_SID) - 1,
log->ops.sid.data, log->ops.sid.used - 1);
}
{
@ -417,6 +538,12 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
event_id_str, len);
}
{
const char *s = events_api_to_str(ev->platform);
facets_add_key_value_length(
facets, WEVT_FIELD_EVENTS_API, sizeof(WEVT_FIELD_EVENTS_API) - 1, s, strlen(s));
}
if(ev->process_id) {
static __thread char process_id_str[UINT64_MAX_LENGTH];
len = print_uint64(process_id_str, ev->process_id);
@ -467,6 +594,30 @@ static inline size_t wevt_process_event(WEVT_LOG *log, FACETS *facets, LOGS_QUER
facets, WEVT_FIELD_TASK "ID", sizeof(WEVT_FIELD_TASK) + 2 - 1, str, len);
}
if(log->type & WEVT_QUERY_EVENT_DATA) {
// the query has full text-search
if(log->ops.event.used > 1) {
bytes += log->ops.event.used;
facets_add_key_value_length(
facets, WEVT_FIELD_EVENT_MESSAGE_HIDDEN, sizeof(WEVT_FIELD_EVENT_MESSAGE_HIDDEN) - 1,
log->ops.event.data, log->ops.event.used - 1);
}
if(log->ops.xml.used > 1) {
bytes += log->ops.xml.used;
facets_add_key_value_length(
facets, WEVT_FIELD_EVENT_XML_HIDDEN, sizeof(WEVT_FIELD_EVENT_XML_HIDDEN) - 1,
log->ops.xml.data, log->ops.xml.used - 1);
}
if(log->ops.event_data->len) {
bytes += log->ops.event_data->len;
facets_add_key_value_length(
facets, WEVT_FIELD_EVENT_DATA_HIDDEN, sizeof(WEVT_FIELD_EVENT_DATA_HIDDEN) - 1,
buffer_tostring(log->ops.event_data), buffer_strlen(log->ops.event_data));
}
}
wevt_facets_register_bin_data(log, facets, ev);
#ifdef NETDATA_INTERNAL_CHECKS
@ -536,7 +687,7 @@ static WEVT_QUERY_STATUS wevt_query_backward(
facets_rows_begin(facets);
WEVT_EVENT e;
while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
if(unlikely(!msg_ut)) {
@ -650,7 +801,7 @@ static WEVT_QUERY_STATUS wevt_query_forward(
facets_rows_begin(facets);
WEVT_EVENT e;
while (status == WEVT_OK && wevt_get_next_event(log, &e, true)) {
while (status == WEVT_OK && wevt_get_next_event(log, &e)) {
usec_t msg_ut = e.created_ns / NSEC_PER_USEC;
if(unlikely(!msg_ut)) {
@ -754,8 +905,20 @@ static WEVT_QUERY_STATUS wevt_query_one_channel(
}
static bool source_is_mine(LOGS_QUERY_SOURCE *src, LOGS_QUERY_STATUS *lqs) {
if((lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) || (src->source_type & lqs->rq.source_type) ||
(lqs->rq.sources && simple_pattern_matches(lqs->rq.sources, string2str(src->source)))) {
if(
// no source is requested
(lqs->rq.source_type == WEVTS_NONE && !lqs->rq.sources) ||
// matches our internal source types
(src->source_type & lqs->rq.source_type) ||
// matches the source name
(lqs->rq.sources && src->source && simple_pattern_matches(lqs->rq.sources, string2str(src->source))) ||
// matches the provider (providers start with a special prefix to avoid mix and match)
(lqs->rq.sources && src->provider && simple_pattern_matches(lqs->rq.sources, string2str(src->provider)))
) {
if(!src->msg_last_ut)
// the file is not scanned yet, or the timestamps have not been updated,
@ -837,7 +1000,7 @@ static int wevt_master_query(BUFFER *wb __maybe_unused, LOGS_QUERY_STATUS *lqs _
usec_t ended_ut = started_ut;
usec_t duration_ut, max_duration_ut = 0;
WEVT_LOG *log = wevt_openlog6();
WEVT_LOG *log = wevt_openlog6(query_has_fts(lqs) ? WEVT_QUERY_FTS : WEVT_QUERY_NORMAL);
if(!log) {
// release the files
for(size_t f = 0; f < files_used ;f++)
@ -1135,7 +1298,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
// initialization
wevt_sources_init();
publisher_cache_init();
provider_cache_init();
sid_cache_init();
field_cache_init();
@ -1209,6 +1372,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
const usec_t step_ut = 100 * USEC_PER_MS;
usec_t send_newline_ut = 0;
usec_t since_last_scan_ut = WINDOWS_EVENTS_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
usec_t since_last_providers_release_ut = 0;
const bool tty = isatty(fileno(stdout)) == 1;
heartbeat_t hb;
@ -1220,7 +1384,13 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
since_last_scan_ut = 0;
}
if(since_last_providers_release_ut > WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT) {
providers_release_unused_handles();
since_last_providers_release_ut = 0;
}
usec_t dt_ut = heartbeat_next(&hb, step_ut);
since_last_providers_release_ut += dt_ut;
since_last_scan_ut += dt_ut;
send_newline_ut += dt_ut;

30
src/collectors/windows-events.plugin/windows-events.h
View file

@ -128,10 +128,15 @@ typedef enum {
#include "windows-events-unicode.h"
#include "windows-events-sid.h"
#include "windows-events-xml.h"
#include "windows-events-publishers.h"
#include "windows-events-providers.h"
#include "windows-events-fields-cache.h"
#include "windows-events-query.h"
// enable or disable preloading on full-text-search
#define ON_FTS_PRELOAD_MESSAGE 1
#define ON_FTS_PRELOAD_XML 0
#define ON_FTS_PRELOAD_EVENT_DATA 1
#define WEVT_FUNCTION_DESCRIPTION "View, search and analyze the Microsoft Windows Events log."
#define WEVT_FUNCTION_NAME "windows-events"
@ -139,26 +144,40 @@ typedef enum {
#define WINDOWS_EVENTS_DEFAULT_TIMEOUT 600
#define WINDOWS_EVENTS_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
#define WINDOWS_EVENTS_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
#define FUNCTION_PROGRESS_EVERY_ROWS (2000)
#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1000)
#define ANCHOR_DELTA_UT (10 * USEC_PER_SEC)
// run providers release every 5 mins
#define WINDOWS_EVENTS_RELEASE_PROVIDERS_HANDLES_EVERY_UT (5 * 60 * USEC_PER_SEC)
// release idle handles that are older than 5 mins
#define WINDOWS_EVENTS_RELEASE_IDLE_PROVIDER_HANDLES_TIME_UT (5 * 60 * USEC_PER_SEC)
#define WEVT_FIELD_COMPUTER "Computer"
#define WEVT_FIELD_CHANNEL "Channel"
#define WEVT_FIELD_PROVIDER "Provider"
#define WEVT_FIELD_SOURCE "Source"
#define WEVT_FIELD_PROVIDER_GUID "ProviderGUID"
#define WEVT_FIELD_EVENTRECORDID "EventRecordID"
#define WEVT_FIELD_VERSION "Version"
#define WEVT_FIELD_QUALIFIERS "Qualifiers"
#define WEVT_FIELD_EVENTID "EventID"
#define WEVT_FIELD_LEVEL "Level"
#define WEVT_FIELD_KEYWORDS "Keywords"
#define WEVT_FIELD_OPCODE "Opcode"
#define WEVT_FIELD_USER "User"
#define WEVT_FIELD_ACCOUNT "UserAccount"
#define WEVT_FIELD_DOMAIN "UserDomain"
#define WEVT_FIELD_SID "UserSID"
#define WEVT_FIELD_TASK "Task"
#define WEVT_FIELD_PROCESSID "ProcessID"
#define WEVT_FIELD_THREADID "ThreadID"
#define WEVT_FIELD_ACTIVITY_ID "ActivityID"
#define WEVT_FIELD_RELATED_ACTIVITY_ID "RelatedActivityID"
#define WEVT_FIELD_XML "XML"
#define WEVT_FIELD_MESSAGE "Message"
#define WEVT_FIELD_EVENTS_API "EventsAPI"
#define WEVT_FIELD_EVENT_DATA_HIDDEN "__HIDDEN__EVENT__DATA__"
#define WEVT_FIELD_EVENT_MESSAGE_HIDDEN "__HIDDEN__MESSAGE__DATA__"
#define WEVT_FIELD_EVENT_XML_HIDDEN "__HIDDEN__XML__DATA__"
// functions needed by LQS
@ -237,7 +256,8 @@ struct lqs_extension {
#define LQS_SOURCE_TYPE WEVT_SOURCE_TYPE
#define LQS_SOURCE_TYPE_ALL WEVTS_ALL
#define LQS_SOURCE_TYPE_NONE WEVTS_NONE
#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) wevt_internal_source_type(value)
#define LQS_PARAMETER_SOURCE_NAME "Event Channels" // this is how it is shown to users
#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) WEVT_SOURCE_TYPE_2id_one(value)
#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) wevt_sources_to_json_array(wb)
#include "libnetdata/facets/logs_query_status.h"

2
src/daemon/buildinfo.c
View file

@ -1331,7 +1331,7 @@ char *get_value_from_key(char *buffer, char *key) {
return s;
}
void get_install_type(char **install_type, char **prebuilt_arch, char **prebuilt_dist) {
void get_install_type(char **install_type, char **prebuilt_arch __maybe_unused, char **prebuilt_dist __maybe_unused) {
#ifndef OS_WINDOWS
char *install_type_filename;

40
src/daemon/main.c
View file

@ -847,34 +847,60 @@ static void log_init(void) {
nd_log_set_priority_level(config_get(CONFIG_SECTION_LOGS, "level", netdata_log_level));
char filename[FILENAME_MAX + 1];
char* os_default_method = NULL;
#if defined(OS_LINUX)
os_default_method = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */ ? "journal" : NULL;
#elif defined(OS_WINDOWS)
#if defined(HAVE_ETW)
os_default_method = "etw";
#elif defined(HAVE_WEL)
os_default_method = "wel";
#endif
#endif
#if defined(OS_WINDOWS)
// on windows, debug log goes to windows events
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
#else
snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir);
#endif
nd_log_set_user_settings(NDLS_DEBUG, config_get(CONFIG_SECTION_LOGS, "debug", filename));
bool with_journal = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */;
if(with_journal)
snprintfz(filename, FILENAME_MAX, "journal");
if(os_default_method)
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/daemon.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_DAEMON, config_get(CONFIG_SECTION_LOGS, "daemon", filename));
if(with_journal)
snprintfz(filename, FILENAME_MAX, "journal");
if(os_default_method)
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/collector.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_COLLECTORS, config_get(CONFIG_SECTION_LOGS, "collector", filename));
#if defined(OS_WINDOWS)
// on windows, access log goes to windows events
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
#else
snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
#endif
nd_log_set_user_settings(NDLS_ACCESS, config_get(CONFIG_SECTION_LOGS, "access", filename));
if(with_journal)
snprintfz(filename, FILENAME_MAX, "journal");
if(os_default_method)
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
else
snprintfz(filename, FILENAME_MAX, "%s/health.log", netdata_configured_log_dir);
nd_log_set_user_settings(NDLS_HEALTH, config_get(CONFIG_SECTION_LOGS, "health", filename));
aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO);
if (aclklog_enabled) {
#if defined(OS_WINDOWS)
// on windows, aclk log goes to windows events
snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
#else
snprintfz(filename, FILENAME_MAX, "%s/aclk.log", netdata_configured_log_dir);
#endif
nd_log_set_user_settings(NDLS_ACLK, config_get(CONFIG_SECTION_CLOUD, "conversation log file", filename));
}

4
src/daemon/winsvc.cc
View file

@ -219,7 +219,11 @@ static bool update_path() {
int main(int argc, char *argv[])
{
#if defined(OS_WINDOWS) && defined(RUN_UNDER_CLION)
bool tty = true;
#else
bool tty = isatty(fileno(stdin)) == 1;
#endif
if (!update_path()) {
return 1;

4
src/libnetdata/buffer/buffer.h
View file

@ -521,13 +521,13 @@ static inline size_t print_int64(char *dst, int64_t value) {
#define UINT64_MAX_LENGTH (24) // 21 should be enough
static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
buffer_need_bytes(wb, 50);
buffer_need_bytes(wb, UINT64_MAX_LENGTH);
wb->len += print_uint64(&wb->buffer[wb->len], value);
buffer_overflow_check(wb);
}
static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
buffer_need_bytes(wb, 50);
buffer_need_bytes(wb, UINT64_MAX_LENGTH);
wb->len += print_int64(&wb->buffer[wb->len], value);
buffer_overflow_check(wb);
}

64
src/libnetdata/facets/facets.c
View file

@ -2363,8 +2363,8 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w
if(with_keys) {
FACET_KEY *k;
foreach_key_in_facets(facets, k){
if (!k->values.enabled)
foreach_key_in_facets(facets, k) {
if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
buffer_json_add_array_item_string(wb, facets_key_id(k));
@ -2591,7 +2591,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
CLEAN_BUFFER *tb = buffer_create(0, NULL);
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
if(!k->values.enabled)
if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
facets_sort_and_reorder_values(k);
@ -2677,37 +2677,40 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
if(k->options & FACET_KEY_OPTION_HIDDEN)
continue;
if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
simple_pattern_matches(facets->visible_keys, k->name))
options |= RRDF_FIELD_OPTS_VISIBLE;
RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE;
RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE;
if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) ||
((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) ||
simple_pattern_matches(facets->visible_keys, k->name))
options |= RRDF_FIELD_OPTS_VISIBLE;
if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
if (k->options & FACET_KEY_OPTION_PRETTY_XML)
transform = RRDF_FIELD_TRANSFORM_XML;
if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
const char *key_id = facets_key_id(k);
if (k->options & FACET_KEY_OPTION_PRETTY_XML)
transform = RRDF_FIELD_TRANSFORM_XML;
buffer_rrdf_table_add_field(
wb, field_id++,
key_id, k->name ? k->name : key_id,
RRDF_FIELD_TYPE_STRING,
visual, transform, 0, NULL, NAN,
RRDF_FIELD_SORT_FIXED,
NULL,
RRDF_FIELD_SUMMARY_COUNT,
(k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
options, FACET_VALUE_UNSET);
}
const char *key_id = facets_key_id(k);
buffer_rrdf_table_add_field(
wb, field_id++,
key_id, k->name ? k->name : key_id,
RRDF_FIELD_TYPE_STRING,
visual, transform, 0, NULL, NAN,
RRDF_FIELD_SORT_FIXED,
NULL,
RRDF_FIELD_SUMMARY_COUNT,
(k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
options, FACET_VALUE_UNSET);
}
foreach_key_in_facets_done(k);
}
buffer_json_object_close(wb); // columns
@ -2744,6 +2747,9 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
if(k->options & FACET_KEY_OPTION_HIDDEN)
continue;
FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
if(unlikely(k->dynamic.cb)) {
@ -2786,7 +2792,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
{
FACET_KEY *k;
foreach_key_in_facets(facets, k) {
if (!k->values.enabled)
if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)
continue;
if(unlikely(!first_histogram_hash))

1
src/libnetdata/facets/facets.h
View file

@ -37,6 +37,7 @@ typedef enum __attribute__((packed)) {
FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 10), // when registering the transformation, do it only at the view, not on all data
FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 11), // the presentation should have this filter expanded by default
FACET_KEY_OPTION_PRETTY_XML = (1 << 12), // instruct the UI to parse this as an XML document
FACET_KEY_OPTION_HIDDEN = (1 << 13), // do not include this field in the response
} FACET_KEY_OPTIONS;
typedef enum __attribute__((packed)) {

1
src/libnetdata/facets/logs_query_status.h
View file

@ -17,7 +17,6 @@
#define LQS_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"
#define LQS_PARAMETER_DATA_ONLY "data_only"
#define LQS_PARAMETER_SOURCE "__logs_sources" // this must never conflict with user fields
#define LQS_PARAMETER_SOURCE_NAME "Logs Sources" // this is how it is shown to users
#define LQS_PARAMETER_INFO "info"
#define LQS_PARAMETER_SLICE "slice"
#define LQS_PARAMETER_DELTA "delta"

4
src/libnetdata/libnetdata.h
View file

@ -459,14 +459,14 @@ extern const char *netdata_configured_host_prefix;
#include "datetime/rfc3339.h"
#include "datetime/rfc7231.h"
#include "completion/completion.h"
#include "log/log.h"
#include "libnetdata/log/nd_log.h"
#include "spawn_server/spawn_server.h"
#include "spawn_server/spawn_popen.h"
#include "simple_pattern/simple_pattern.h"
#include "socket/security.h"
#include "socket/socket.h"
#include "config/appconfig.h"
#include "log/journal.h"
#include "log/systemd-journal-helpers.h"
#include "buffered_reader/buffered_reader.h"
#include "procfile/procfile.h"
#include "string/string.h"

250
src/libnetdata/log/README.md
View file

@ -17,14 +17,15 @@ For each log source, Netdata supports the following output methods:
- **off**, to disable this log source
- **journal**, to send the logs to systemd-journal.
- **etw**, to send the logs to Event Tracing for Windows (ETW).
- **wel**, to send the logs to the Windows Event Log (WEL).
- **syslog**, to send the logs to syslog.
- **system**, to send the output to `stderr` or `stdout` depending on the log source.
- **stdout**, to write the logs to Netdata's `stdout`.
- **stderr**, to write the logs to Netdata's `stderr`.
- **filename**, to send the logs to a file.
For `daemon` and `collector` the default is `journal` when systemd-journal is available.
To decide if systemd-journal is available, Netdata checks:
On Linux, when systemd-journal is available, the default is `journal` for `daemon` and `collector` and `filename` for the rest. To decide if systemd-journal is available, Netdata checks:
1. `stderr` is connected to systemd-journald
2. `/run/systemd/journal/socket` exists
@ -32,13 +33,16 @@ To decide if systemd-journal is available, Netdata checks:
If any of the above is detected, Netdata will select `journal` for `daemon` and `collector` sources.
All other sources default to a file.
On Windows, the default is `etw` and if that is not available it falls back to `wel`. The availability of `etw` is decided at compile time.
## Log formats
| Format | Description |
|---------|--------------------------------------------------------------------------------------------------------|
| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
| etw | Event Tracing for Windows specific format. Structured logging in Event Viewer. |
| wel | Windows Event Log specific format. Basic field-based logging in Event Viewer. |
| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
| logfmt | logs data as a series of key/value pairs. The default when logging to any output other than `journal`. |
| json | logs data in JSON format. |
@ -57,6 +61,9 @@ Each time Netdata logs, it assigns a priority to the log. It can be one of this
| info | the default log level about information the user should know. |
| debug | these are more verbose logs that can be ignored. |
For `etw` these are mapped to `Verbose`, `Informational`, `Warning`, `Error` and `Critical`.
For `wel` these are mapped to `Informational`, `Warning`, `Error`.
## Logs Configuration
In `netdata.conf`, there are the following settings:
@ -108,66 +115,69 @@ Sending a `SIGHUP` to Netdata, will instruct it to re-open all its log files.
<details>
<summary>All fields exposed by Netdata</summary>
| journal | logfmt | json | Description |
|:--------------------------------------:|:------------------------------:|:------------------------------:|:---------------------------------------------------------------------------------------------------------:|
| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `time` | the timestamp of the event |
| `SYSLOG_IDENTIFIER` | `comm` | `comm` | the program logging the event |
| `ND_LOG_SOURCE` | `source` | `source` | one of the [log sources](#log-sources) |
| `PRIORITY`<br/>numeric | `level`<br/>text | `level`<br/>numeric | one of the [log levels](#log-levels) |
| `ERRNO` | `errno` | `errno` | the numeric value of `errno` |
| `INVOCATION_ID` | - | - | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
| `CODE_LINE` | - | - | the line number of of the source code logging this event |
| `CODE_FILE` | - | - | the filename of the source code logging this event |
| `CODE_FUNCTION` | - | - | the function name of the source code logging this event |
| `TID` | `tid` | `tid` | the thread id of the thread logging this event |
| `THREAD_TAG` | `thread` | `thread` | the name of the thread logging this event |
| `MESSAGE_ID` | `msg_id` | `msg_id` | see [message IDs](#message-ids) |
| `ND_MODULE` | `module` | `module` | the Netdata module logging this event |
| `ND_NIDL_NODE` | `node` | `node` | the hostname of the node the event is related to |
| `ND_NIDL_INSTANCE` | `instance` | `instance` | the instance of the node the event is related to |
| `ND_NIDL_CONTEXT` | `context` | `context` | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
| `ND_NIDL_DIMENSION` | `dimension` | `dimension` | the dimension the event is related to |
| `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport |
| `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from |
| `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from |
| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` |
| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` |
| `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child |
| `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport |
| `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination |
| `ND_DST_PORT` | `dst_port` | `dst_port` | when the event happened during an outbound request, this is the port the request destination |
| `ND_DST_CAPABILITIES` | `dst_capabilities` | `dst_capabilities` | when the request goes to a parent, this is the communication capabilities of the parent |
| `ND_REQUEST_METHOD` | `req_method` | `req_method` | when the event happened during an inbound request, this is the method the request was received |
| `ND_RESPONSE_CODE` | `code` | `code` | when responding to a request, this this the response code |
| `ND_CONNECTION_ID` | `conn` | `conn` | when there is a connection id for an inbound connection, this is the connection id |
| `ND_TRANSACTION_ID` | `transaction` | `transaction` | the transaction id (UUID) of all API requests |
| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `sent_bytes` | the bytes we sent to API responses |
| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `size_bytes` | the uncompressed bytes of the API responses |
| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `prep_ut` | the time needed to prepare a response |
| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `sent_ut` | the time needed to send a response |
| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `total_ut` | the total time needed to complete a response |
| `ND_ALERT_ID` | `alert_id` | `alert_id` | the alert id this event is related to |
| `ND_ALERT_EVENT_ID` | `alert_event_id` | `alert_event_id` | a sequential number of the alert transition (per host) |
| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `alert_unique_id` | a sequential number of the alert transition (per alert) |
| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `alert_transition_id` | the unique UUID of this alert transition |
| `ND_ALERT_CONFIG` | `alert_config` | `alert_config` | the alert configuration hash (UUID) |
| `ND_ALERT_NAME` | `alert` | `alert` | the alert name |
| `ND_ALERT_CLASS` | `alert_class` | `alert_class` | the alert classification |
| `ND_ALERT_COMPONENT` | `alert_component` | `alert_component` | the alert component |
| `ND_ALERT_TYPE` | `alert_type` | `alert_type` | the alert type |
| `ND_ALERT_EXEC` | `alert_exec` | `alert_exec` | the alert notification program |
| `ND_ALERT_RECIPIENT` | `alert_recipient` | `alert_recipient` | the alert recipient(s) |
| `ND_ALERT_VALUE` | `alert_value` | `alert_value` | the current alert value |
| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
| `ND_ALERT_STATUS` | `alert_status` | `alert_status` | the current alert status |
| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
| `ND_ALERT_UNITS` | `alert_units` | `alert_units` | the units of the alert |
| `ND_ALERT_SUMMARY` | `alert_summary` | `alert_summary` | the summary text of the alert |
| `ND_ALERT_INFO` | `alert_info` | `alert_info` | the info text of the alert |
| `ND_ALERT_DURATION` | `alert_duration` | `alert_duration` | the duration the alert was in its previous state |
| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `alert_notification_timestamp` | the timestamp the notification delivery is scheduled |
| `ND_REQUEST` | `request` | `request` | the full request during which the event happened |
| `MESSAGE` | `msg` | `msg` | the event message |
| `journal` | `logfmt` and `json` | `etw` | `wel` | Description |
|:--------------------------------------:|:------------------------------:|:-----------------------------:|:-----:|:----------------------------------------------------------------------------------------------------------|
| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `Timestamp` | 1 | the timestamp of the event |
| `SYSLOG_IDENTIFIER` | `comm` | `Program` | 2 | the program logging the event |
| `ND_LOG_SOURCE` | `source` | `NetdataLogSource` | 3 | one of the [log sources](#log-sources) |
| `PRIORITY`<br/>numeric | `level`<br/>text | `Level`<br/>text | 4 | one of the [log levels](#log-levels) |
| `ERRNO` | `errno` | `UnixErrno` | 5 | the numeric value of `errno` |
| `INVOCATION_ID` | - | `InvocationID` | 7 | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
| `CODE_LINE` | - | `CodeLine` | 8 | the line number of of the source code logging this event |
| `CODE_FILE` | - | `CodeFile` | 9 | the filename of the source code logging this event |
| `CODE_FUNCTION` | - | `CodeFunction` | 10 | the function name of the source code logging this event |
| `TID` | `tid` | `ThreadID` | 11 | the thread id of the thread logging this event |
| `THREAD_TAG` | `thread` | `ThreadName` | 12 | the name of the thread logging this event |
| `MESSAGE_ID` | `msg_id` | `MessageID` | 13 | see [message IDs](#message-ids) |
| `ND_MODULE` | `module` | `Module` | 14 | the Netdata module logging this event |
| `ND_NIDL_NODE` | `node` | `Node` | 15 | the hostname of the node the event is related to |
| `ND_NIDL_INSTANCE` | `instance` | `Instance` | 16 | the instance of the node the event is related to |
| `ND_NIDL_CONTEXT` | `context` | `Context` | 17 | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
| `ND_NIDL_DIMENSION` | `dimension` | `Dimension` | 18 | the dimension the event is related to |
| `ND_SRC_TRANSPORT` | `src_transport` | `SourceTransport` | 19 | when the event happened during a request, this is the request transport |
| `ND_SRC_IP` | `src_ip` | `SourceIP` | 24 | when the event happened during an inbound request, this is the IP the request came from |
| `ND_SRC_PORT` | `src_port` | `SourcePort` | 25 | when the event happened during an inbound request, this is the port the request came from |
| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `SourceForwardedHost` | 26 | the contents of the HTTP header `X-Forwarded-Host` |
| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `SourceForwardedFor` | 27 | the contents of the HTTP header `X-Forwarded-For` |
| `ND_SRC_CAPABILITIES` | `src_capabilities` | `SourceCapabilities` | 28 | when the request came from a child, this is the communication capabilities of the child |
| `ND_DST_TRANSPORT` | `dst_transport` | `DestinationTransport` | 29 | when the event happened during an outbound request, this is the outbound request transport |
| `ND_DST_IP` | `dst_ip` | `DestinationIP` | 30 | when the event happened during an outbound request, this is the IP the request destination |
| `ND_DST_PORT` | `dst_port` | `DestinationPort` | 31 | when the event happened during an outbound request, this is the port the request destination |
| `ND_DST_CAPABILITIES` | `dst_capabilities` | `DestinationCapabilities` | 32 | when the request goes to a parent, this is the communication capabilities of the parent |
| `ND_REQUEST_METHOD` | `req_method` | `RequestMethod` | 33 | when the event happened during an inbound request, this is the method the request was received |
| `ND_RESPONSE_CODE` | `code` | `ResponseCode` | 34 | when responding to a request, this this the response code |
| `ND_CONNECTION_ID` | `conn` | `ConnectionID` | 35 | when there is a connection id for an inbound connection, this is the connection id |
| `ND_TRANSACTION_ID` | `transaction` | `TransactionID` | 36 | the transaction id (UUID) of all API requests |
| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `ResponseSentBytes` | 37 | the bytes we sent to API responses |
| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `ResponseSizeBytes` | 38 | the uncompressed bytes of the API responses |
| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `ResponsePreparationTimeUsec` | 39 | the time needed to prepare a response |
| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `ResponseSentTimeUsec` | 40 | the time needed to send a response |
| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `ResponseTotalTimeUsec` | 41 | the total time needed to complete a response |
| `ND_ALERT_ID` | `alert_id` | `AlertID` | 42 | the alert id this event is related to |
| `ND_ALERT_EVENT_ID` | `alert_event_id` | `AlertEventID` | 44 | a sequential number of the alert transition (per host) |
| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `AlertUniqueID` | 43 | a sequential number of the alert transition (per alert) |
| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `AlertTransitionID` | 45 | the unique UUID of this alert transition |
| `ND_ALERT_CONFIG` | `alert_config` | `AlertConfig` | 46 | the alert configuration hash (UUID) |
| `ND_ALERT_NAME` | `alert` | `AlertName` | 47 | the alert name |
| `ND_ALERT_CLASS` | `alert_class` | `AlertClass` | 48 | the alert classification |
| `ND_ALERT_COMPONENT` | `alert_component` | `AlertComponent` | 49 | the alert component |
| `ND_ALERT_TYPE` | `alert_type` | `AlertType` | 50 | the alert type |
| `ND_ALERT_EXEC` | `alert_exec` | `AlertExec` | 51 | the alert notification program |
| `ND_ALERT_RECIPIENT` | `alert_recipient` | `AlertRecipient` | 52 | the alert recipient(s) |
| `ND_ALERT_VALUE` | `alert_value` | `AlertValue` | 54 | the current alert value |
| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `AlertOldValue` | 55 | the previous alert value |
| `ND_ALERT_STATUS` | `alert_status` | `AlertStatus` | 56 | the current alert status |
| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `AlertOldStatus` | 57 | the previous alert status |
| `ND_ALERT_UNITS` | `alert_units` | `AlertUnits` | 59 | the units of the alert |
| `ND_ALERT_SUMMARY` | `alert_summary` | `AlertSummary` | 60 | the summary text of the alert |
| `ND_ALERT_INFO` | `alert_info` | `AlertInfo` | 61 | the info text of the alert |
| `ND_ALERT_DURATION` | `alert_duration` | `AlertDuration` | 53 | the duration the alert was in its previous state |
| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `AlertNotificationTimeUsec` | 62 | the timestamp the notification delivery is scheduled |
| `ND_REQUEST` | `request` | `Request` | 63 | the full request during which the event happened |
| `MESSAGE` | `msg` | `Message` | 64 | the event message |
For `wel` (Windows Event Logs), all logs have an array of 64 fields strings, and their index number provides their meaning.
For `etw` (Event Tracing for Windows), Netdata logs in a structured way, and field names are available.
</details>
@ -212,3 +222,117 @@ journalctl -u netdata --namespace=netdata
# All netdata logs, the newest entries are displayed first
journalctl -u netdata --namespace=netdata -r
```
## Using Event Tracing for Windows (ETW)
ETW requires the publisher `Netdata` to be registered. Our Windows installer does this automatically.
Registering the publisher is done via a manifest (`%SystemRoot%\System32\wevt_netdata_manifest.xml`)
and its messages resources DLL (`%SystemRoot%\System32\wevt_netdata.dll`).
If needed, the publisher can be registered and unregistered manually using these commands:
```bat
REM register the Netdata publisher
wevtutil im "%SystemRoot%\System32\wevt_netdata_manifest.xml" "/mf:%SystemRoot%\System32\wevt_netdata.dll" "/rf:%SystemRoot%\System32\wevt_netdata.dll"
REM unregister the Netdata publisher
wevtutil um "%SystemRoot%\System32\wevt_netdata_manifest.xml"
```
The structure of the logs are as follows:
- Publisher `Netdata`
- Channel `Netdata/Daemon`: general messages about the Netdata service
- Channel `Netdata/Collector`: general messages about Netdata external plugins
- Channel `Netdata/Health`: alert transitions and general messages generated by Netdata's health engine
- Channel `Netdata/Access`: all accesses to Netdata APIs
- Channel `Netdata/Aclk`: for cloud connectivity tracing (disabled by default)
Retention can be configured per Channel via the Event Viewer. Netdata does not set a default, so the system default is used.
> **IMPORTANT**<br/>
> Event Tracing for Windows (ETW) does not allow logging the percentage character `%`.
> The `%` followed by a number, is recursively used for fields expansion and ETW has not
> provided any way to escape the character for preventing further expansion.<br/>
> <br/>
> To work around this limitation, Netdata replaces all `%` which are followed by a number, with `℅`
> (the Unicode character `care of`). Visually, they look similar, but when copying IPv6 addresses
> or URLs from the logs, you have to be careful to manually replace `℅` with `%` before using them.
## Using Windows Event Logs (WEL)
WEL has a different logs structure and unfortunately WEL and ETW need to use different names if they are to be used
concurrently.
For WEL, Netdata logs as follows:
- Channel `NetdataWEL` (unfortunately `Netdata` cannot be used, it conflicts with the ETW Publisher name)
- Publisher `NetdataDaemon`: general messages about the Netdata service
- Publisher `NetdataCollector`: general messages about Netdata external plugins
- Publisher `NetdataHealth`: alert transitions and general messages generated by Netdata's health engine
- Publisher `NetdataAccess`: all accesses to Netdata APIs
- Publisher `NetdataAclk`: for cloud connectivity tracing (disabled by default)
Publishers must have unique names system-wide, so we had to prefix them with `Netdata`.
Retention can be configured per Publisher via the Event Viewer or the Registry.
Netdata sets by default 20MiB for all of them, except `NetdataAclk` (5MiB) and `NetdataAccess` (35MiB),
for a total of 100MiB.
For WEL some registry entries are needed. Netdata automatically takes care of them when it starts.
WEL does not have the problem ETW has with the percent character `%`, so Netdata logs it as-is.
## Differences between ETW and WEL
There are key differences between ETW and WEL.
### Publishers and Providers
**Publishers** are collections of ETW Providers. A Publisher is implied by a manifest file,
each of which is considered a Publisher, and each manifest file can define multiple **Providers** in it.
Other than that there is no entity related to **Publishers** in the system.
**Publishers** are not defined for WEL.
**Providers** are the applications or modules logging. Provider names must be unique across the system,
for ETW and WEL together.
To define a **Provider**:
- ETW requires a **Publisher** manifest coupled with resources DLLs and must be registered
via `wevtutil` (handled by the Netdata Windows installer automatically).
- WEL requires some registry entries and a message resources DLL (handled by Netdata automatically on startup).
The Provider appears as `Source` in the Event Viewer, for both WEL and ETW.
### Channels
- **Channels** for WEL are collections of WEL Providers, (each WEL Provider is a single Stream of logs).
- **Channels** for ETW slice the logs of each Provider into multiple Streams.
WEL Channels cannot have the same name as ETW Providers. This is why Netdata's ETW provider is
called `Netdata`, and WEL channel is called `NetdataWEL`.
Despite the fact that ETW **Publishers** and WEL **Channels** are both collections of Providers,
they are not similar. In ETW a Publisher is a collection on the publisher's Providers, but in WEL
a Channel may include independent WEL Providers (e.g. the "Applications" Channel). Additionally,
WEL Channels cannot include ETW Providers.
### Retention
Retention is always defined per Stream.
- Retention in ETW is defined per ETW Channel (ETW Provider Stream).
- Retention in WEL is defined per WEL Provider (each WEL Provider is a single Stream).
### Messages Formatting
- ETW supports recursive fields expansion, and therefore `%N` in fields is expanded recursively
(or replaced with an error message if expansion fails). Netdata replaces `%N` with `℅N` to stop
recursive expansion (since `%N` cannot be logged otherwise).
- WEL performs a single field expansion, and therefore the `%` character in fields is never expanded.
### Usability
- ETW names all the fields and allows multiple datatypes per field, enabling log consumers to know
what each field means and its datatype.
- WEL uses a simple string table for fields, and consumers need to map these string fields based on
their index.

2555
src/libnetdata/log/log.c
View file

File diff suppressed because it is too large Load diff

84
src/libnetdata/log/nd_log-annotators.c Normal file
View file

@ -0,0 +1,84 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
const char *timestamp_usec_annotator(struct log_field *lf) {
usec_t ut = log_field_to_uint64(lf);
if(!ut)
return NULL;
static __thread char datetime[RFC3339_MAX_LENGTH];
rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
return datetime;
}
const char *errno_annotator(struct log_field *lf) {
int64_t errnum = log_field_to_int64(lf);
if(errnum == 0)
return NULL;
static __thread char buf[256];
size_t len = print_uint64(buf, errnum);
buf[len++] = ',';
buf[len++] = ' ';
char *msg_to = &buf[len];
size_t msg_size = sizeof(buf) - len;
const char *s = errno2str((int)errnum, msg_to, msg_size);
if(s != msg_to)
strncpyz(msg_to, s, msg_size - 1);
return buf;
}
#if defined(OS_WINDOWS)
const char *winerror_annotator(struct log_field *lf) {
DWORD errnum = log_field_to_uint64(lf);
if (errnum == 0)
return NULL;
static __thread char buf[256];
size_t len = print_uint64(buf, errnum);
buf[len++] = ',';
buf[len++] = ' ';
char *msg_to = &buf[len];
size_t msg_size = sizeof(buf) - len;
wchar_t wbuf[1024];
DWORD size = FormatMessageW(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL,
errnum,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
wbuf,
(DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1),
NULL
);
if (size > 0) {
// Remove \r\n at the end
while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n'))
wbuf[--size] = L'\0';
// Convert wide string to UTF-8
int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, msg_to, (int)msg_size, NULL, NULL);
if (utf8_size == 0)
snprintf(msg_to, msg_size - 1, "unknown error code");
msg_to[msg_size - 1] = '\0';
}
else
snprintf(msg_to, msg_size - 1, "unknown error code");
return buf;
}
#endif
const char *priority_annotator(struct log_field *lf) {
uint64_t pri = log_field_to_uint64(lf);
return nd_log_id2priority(pri);
}

147
src/libnetdata/log/nd_log-common.h Normal file
View file

@ -0,0 +1,147 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_ND_LOG_COMMON_H
#define NETDATA_ND_LOG_COMMON_H
#include <syslog.h>
typedef enum __attribute__((__packed__)) {
NDLS_UNSET = 0, // internal use only
NDLS_ACCESS, // access.log
NDLS_ACLK, // aclk.log
NDLS_COLLECTORS, // collector.log
NDLS_DAEMON, // error.log
NDLS_HEALTH, // health.log
NDLS_DEBUG, // debug.log
// terminator
_NDLS_MAX,
} ND_LOG_SOURCES;
typedef enum __attribute__((__packed__)) {
NDLP_EMERG = LOG_EMERG, // from syslog.h
NDLP_ALERT = LOG_ALERT, // from syslog.h
NDLP_CRIT = LOG_CRIT, // from syslog.h
NDLP_ERR = LOG_ERR, // from syslog.h
NDLP_WARNING = LOG_WARNING, // from syslog.h
NDLP_NOTICE = LOG_NOTICE, // from syslog.h
NDLP_INFO = LOG_INFO, // from syslog.h
NDLP_DEBUG = LOG_DEBUG, // from syslog.h
// terminator
_NDLP_MAX,
} ND_LOG_FIELD_PRIORITY;
typedef enum __attribute__((__packed__)) {
// KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
// so that it easy to audit for missing fields
// NEVER RENUMBER THIS LIST
// The Windows Events Log has them at fixed positions
NDF_STOP = 0,
NDF_TIMESTAMP_REALTIME_USEC = 1, // the timestamp of the log message - added automatically
NDF_SYSLOG_IDENTIFIER = 2, // the syslog identifier of the application - added automatically
NDF_LOG_SOURCE = 3, // DAEMON, COLLECTORS, HEALTH, MSGID_ACCESS, ACLK - set at the log call
NDF_PRIORITY = 4, // the syslog priority (severity) - set at the log call
NDF_ERRNO = 5, // the ERRNO at the time of the log call - added automatically
NDF_WINERROR = 6, // Windows GetLastError()
NDF_INVOCATION_ID = 7, // the INVOCATION_ID of Netdata - added automatically
NDF_LINE = 8, // the source code file line number - added automatically
NDF_FILE = 9, // the source code filename - added automatically
NDF_FUNC = 10, // the source code function - added automatically
NDF_TID = 11, // the thread ID of the thread logging - added automatically
NDF_THREAD_TAG = 12, // the thread tag of the thread logging - added automatically
NDF_MESSAGE_ID = 13, // for specific events
NDF_MODULE = 14, // for internal plugin module, all other get the NDF_THREAD_TAG
NDF_NIDL_NODE = 15, // the node / rrdhost currently being worked
NDF_NIDL_INSTANCE = 16, // the instance / rrdset currently being worked
NDF_NIDL_CONTEXT = 17, // the context of the instance currently being worked
NDF_NIDL_DIMENSION = 18, // the dimension / rrddim currently being worked
// web server, aclk and stream receiver
NDF_SRC_TRANSPORT = 19, // the transport we received the request, one of: http, https, pluginsd
// Netdata Cloud Related
NDF_ACCOUNT_ID = 20,
NDF_USER_NAME = 21,
NDF_USER_ROLE = 22,
NDF_USER_ACCESS = 23,
// web server and stream receiver
NDF_SRC_IP = 24, // the streaming / web server source IP
NDF_SRC_PORT = 25, // the streaming / web server source Port
NDF_SRC_FORWARDED_HOST = 26,
NDF_SRC_FORWARDED_FOR = 27,
NDF_SRC_CAPABILITIES = 28, // the stream receiver capabilities
// stream sender (established links)
NDF_DST_TRANSPORT = 29, // the transport we send the request, one of: http, https
NDF_DST_IP = 30, // the destination streaming IP
NDF_DST_PORT = 31, // the destination streaming Port
NDF_DST_CAPABILITIES = 32, // the destination streaming capabilities
// web server, aclk and stream receiver
NDF_REQUEST_METHOD = 33, // for http like requests, the http request method
NDF_RESPONSE_CODE = 34, // for http like requests, the http response code, otherwise a status string
// web server (all), aclk (queries)
NDF_CONNECTION_ID = 35, // the web server connection ID
NDF_TRANSACTION_ID = 36, // the web server and API transaction ID
NDF_RESPONSE_SENT_BYTES = 37, // for http like requests, the response bytes
NDF_RESPONSE_SIZE_BYTES = 38, // for http like requests, the uncompressed response size
NDF_RESPONSE_PREPARATION_TIME_USEC = 39, // for http like requests, the preparation time
NDF_RESPONSE_SENT_TIME_USEC = 40, // for http like requests, the time to send the response back
NDF_RESPONSE_TOTAL_TIME_USEC = 41, // for http like requests, the total time to complete the response
// health alerts
NDF_ALERT_ID = 42,
NDF_ALERT_UNIQUE_ID = 43,
NDF_ALERT_EVENT_ID = 44,
NDF_ALERT_TRANSITION_ID = 45,
NDF_ALERT_CONFIG_HASH = 46,
NDF_ALERT_NAME = 47,
NDF_ALERT_CLASS = 48,
NDF_ALERT_COMPONENT = 49,
NDF_ALERT_TYPE = 50,
NDF_ALERT_EXEC = 51,
NDF_ALERT_RECIPIENT = 52,
NDF_ALERT_DURATION = 53,
NDF_ALERT_VALUE = 54,
NDF_ALERT_VALUE_OLD = 55,
NDF_ALERT_STATUS = 56,
NDF_ALERT_STATUS_OLD = 57,
NDF_ALERT_SOURCE = 58,
NDF_ALERT_UNITS = 59,
NDF_ALERT_SUMMARY = 60,
NDF_ALERT_INFO = 61,
NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62,
// NDF_ALERT_FLAGS,
// put new items here
// leave the request URL and the message last
NDF_REQUEST = 63, // the request we are currently working on
NDF_MESSAGE = 64, // the log message, if any
// terminator
_NDF_MAX,
} ND_LOG_FIELD_ID;
typedef enum __attribute__((__packed__)) {
NDFT_UNSET = 0,
NDFT_TXT,
NDFT_STR,
NDFT_BFR,
NDFT_U64,
NDFT_I64,
NDFT_DBL,
NDFT_UUID,
NDFT_CALLBACK,
// terminator
_NDFT_MAX,
} ND_LOG_STACK_FIELD_TYPE;
#endif //NETDATA_ND_LOG_COMMON_H

207
src/libnetdata/log/nd_log-config.c Normal file
View file

@ -0,0 +1,207 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
char buf[FILENAME_MAX + 100];
if(setting && *setting)
strncpyz(buf, setting, sizeof(buf) - 1);
else
buf[0] = '\0';
struct nd_log_source *ls = &nd_log.sources[source];
char *output = strrchr(buf, '@');
if(!output)
// all of it is the output
output = buf;
else {
// we found an '@', the next char is the output
*output = '\0';
output++;
// parse the other params
char *remaining = buf;
while(remaining) {
char *value = strsep_skip_consecutive_separators(&remaining, ",");
if (!value || !*value) continue;
char *name = strsep_skip_consecutive_separators(&value, "=");
if (!name || !*name) continue;
if(strcmp(name, "logfmt") == 0)
ls->format = NDLF_LOGFMT;
else if(strcmp(name, "json") == 0)
ls->format = NDLF_JSON;
else if(strcmp(name, "journal") == 0)
ls->format = NDLF_JOURNAL;
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
else if(strcmp(name, ETW_NAME) == 0)
ls->format = NDLF_ETW;
#endif
#if defined(HAVE_WEL)
else if(strcmp(name, WEL_NAME) == 0)
ls->format = NDLF_WEL;
#endif
#endif
else if(strcmp(name, "level") == 0 && value && *value)
ls->min_priority = nd_log_priority2id(value);
else if(strcmp(name, "protection") == 0 && value && *value) {
if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
ls->limits = ND_LOG_LIMITS_UNLIMITED;
ls->limits.counter = 0;
ls->limits.prevented = 0;
}
else {
ls->limits = ND_LOG_LIMITS_DEFAULT;
char *slash = strchr(value, '/');
if(slash) {
*slash = '\0';
slash++;
ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
int period;
if(!duration_parse_seconds(slash, &period)) {
nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash);
period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
}
ls->limits.throttle_period = period;
}
else {
ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
}
}
}
else
nd_log(NDLS_DAEMON, NDLP_ERR,
"Error while parsing configuration of log source '%s'. "
"In config '%s', '%s' is not understood.",
nd_log_id2source(source), setting, name);
}
}
if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
ls->method = NDLM_DISABLED;
ls->filename = "/dev/null";
}
else if(strcmp(output, "journal") == 0) {
ls->method = NDLM_JOURNAL;
ls->filename = NULL;
}
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
else if(strcmp(output, ETW_NAME) == 0) {
ls->method = NDLM_ETW;
ls->filename = NULL;
}
#endif
#if defined(HAVE_WEL)
else if(strcmp(output, WEL_NAME) == 0) {
ls->method = NDLM_WEL;
ls->filename = NULL;
}
#endif
#endif
else if(strcmp(output, "syslog") == 0) {
ls->method = NDLM_SYSLOG;
ls->filename = NULL;
}
else if(strcmp(output, "/dev/null") == 0) {
ls->method = NDLM_DEVNULL;
ls->filename = "/dev/null";
}
else if(strcmp(output, "system") == 0) {
if(ls->fd == STDERR_FILENO) {
ls->method = NDLM_STDERR;
ls->filename = NULL;
ls->fd = STDERR_FILENO;
}
else {
ls->method = NDLM_STDOUT;
ls->filename = NULL;
ls->fd = STDOUT_FILENO;
}
}
else if(strcmp(output, "stderr") == 0) {
ls->method = NDLM_STDERR;
ls->filename = NULL;
ls->fd = STDERR_FILENO;
}
else if(strcmp(output, "stdout") == 0) {
ls->method = NDLM_STDOUT;
ls->filename = NULL;
ls->fd = STDOUT_FILENO;
}
else {
ls->method = NDLM_FILE;
ls->filename = strdupz(output);
}
#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
ls->min_priority = NDLP_DEBUG;
#endif
if(source == NDLS_COLLECTORS) {
// set the method for the collector processes we will spawn
ND_LOG_METHOD method = NDLM_STDERR;
ND_LOG_FORMAT format = NDLF_LOGFMT;
ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ls->method)) {
method = ls->method;
format = ls->format;
}
nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
}
}
void nd_log_set_priority_level(const char *setting) {
if(!setting || !*setting)
setting = "info";
ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
priority = NDLP_DEBUG;
#endif
for (size_t i = 0; i < _NDLS_MAX; i++) {
if (i != NDLS_DEBUG)
nd_log.sources[i].min_priority = priority;
}
// the right one
nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
}
void nd_log_set_facility(const char *facility) {
if(!facility || !*facility)
facility = "daemon";
nd_log.syslog.facility = nd_log_facility2id(facility);
nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
}
void nd_log_set_flood_protection(size_t logs, time_t period) {
nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
nd_log.sources[NDLS_DAEMON].limits.throttle_period =
nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
char buf[100];
snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
}

127
src/libnetdata/log/nd_log-field-formatters.c Normal file
View file

@ -0,0 +1,127 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
int64_t log_field_to_int64(struct log_field *lf) {
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
CLEAN_BUFFER *tmp = NULL;
const char *s = NULL;
switch(lf->entry.type) {
default:
case NDFT_UUID:
case NDFT_UNSET:
return 0;
case NDFT_TXT:
s = lf->entry.txt;
break;
case NDFT_STR:
s = string2str(lf->entry.str);
break;
case NDFT_BFR:
s = buffer_tostring(lf->entry.bfr);
break;
case NDFT_CALLBACK:
tmp = buffer_create(0, NULL);
if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
s = buffer_tostring(tmp);
else
s = NULL;
break;
case NDFT_U64:
return (int64_t)lf->entry.u64;
case NDFT_I64:
return (int64_t)lf->entry.i64;
case NDFT_DBL:
return (int64_t)lf->entry.dbl;
}
if(s && *s)
return str2ll(s, NULL);
return 0;
}
uint64_t log_field_to_uint64(struct log_field *lf) {
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
CLEAN_BUFFER *tmp = NULL;
const char *s = NULL;
switch(lf->entry.type) {
default:
case NDFT_UUID:
case NDFT_UNSET:
return 0;
case NDFT_TXT:
s = lf->entry.txt;
break;
case NDFT_STR:
s = string2str(lf->entry.str);
break;
case NDFT_BFR:
s = buffer_tostring(lf->entry.bfr);
break;
case NDFT_CALLBACK:
tmp = buffer_create(0, NULL);
if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
s = buffer_tostring(tmp);
else
s = NULL;
break;
case NDFT_U64:
return lf->entry.u64;
case NDFT_I64:
return lf->entry.i64;
case NDFT_DBL:
return (uint64_t) lf->entry.dbl;
}
if(s && *s)
return str2uint64_t(s, NULL);
return 0;
}

78
src/libnetdata/log/nd_log-format-json.c Normal file
View file

@ -0,0 +1,78 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
CLEAN_BUFFER *tmp = NULL;
for (size_t i = 0; i < fields_max; i++) {
if (!fields[i].entry.set || !fields[i].logfmt)
continue;
const char *key = fields[i].logfmt;
const char *s = NULL;
switch(fields[i].entry.type) {
case NDFT_TXT:
s = fields[i].entry.txt;
break;
case NDFT_STR:
s = string2str(fields[i].entry.str);
break;
case NDFT_BFR:
s = buffer_tostring(fields[i].entry.bfr);
break;
case NDFT_U64:
buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
break;
case NDFT_I64:
buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
break;
case NDFT_DBL:
buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
break;
case NDFT_UUID:
if(!uuid_is_null(*fields[i].entry.uuid)) {
char u[UUID_COMPACT_STR_LEN];
uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
buffer_json_member_add_string(wb, key, u);
}
break;
case NDFT_CALLBACK: {
if(!tmp)
tmp = buffer_create(1024, NULL);
else
buffer_flush(tmp);
if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
s = buffer_tostring(tmp);
else
s = NULL;
}
break;
default:
s = "UNHANDLED";
break;
}
if(s && *s)
buffer_json_member_add_string(wb, key, s);
}
buffer_json_finalize(wb);
}

151
src/libnetdata/log/nd_log-format-logfmt.c Normal file
View file

@ -0,0 +1,151 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
static bool needs_quotes_for_logfmt(const char *s)
{
static bool safe_for_logfmt[256] = {
[' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
};
if(!*s)
return true;
while(*s) {
if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s])
return true;
s++;
}
return false;
}
static void string_to_logfmt(BUFFER *wb, const char *s)
{
bool spaces = needs_quotes_for_logfmt(s);
if(spaces)
buffer_fast_strcat(wb, "\"", 1);
buffer_json_strcat(wb, s);
if(spaces)
buffer_fast_strcat(wb, "\"", 1);
}
void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) {
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
CLEAN_BUFFER *tmp = NULL;
for (size_t i = 0; i < fields_max; i++) {
if (!fields[i].entry.set || !fields[i].logfmt)
continue;
const char *key = fields[i].logfmt;
if(fields[i].annotator) {
const char *s = fields[i].annotator(&fields[i]);
if(!s) continue;
if(buffer_strlen(wb))
buffer_fast_strcat(wb, " ", 1);
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
string_to_logfmt(wb, s);
}
else {
if(buffer_strlen(wb))
buffer_fast_strcat(wb, " ", 1);
switch(fields[i].entry.type) {
case NDFT_TXT:
if(*fields[i].entry.txt) {
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
string_to_logfmt(wb, fields[i].entry.txt);
}
break;
case NDFT_STR:
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
string_to_logfmt(wb, string2str(fields[i].entry.str));
break;
case NDFT_BFR:
if(buffer_strlen(fields[i].entry.bfr)) {
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
}
break;
case NDFT_U64:
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
buffer_print_uint64(wb, fields[i].entry.u64);
break;
case NDFT_I64:
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
buffer_print_int64(wb, fields[i].entry.i64);
break;
case NDFT_DBL:
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
buffer_print_netdata_double(wb, fields[i].entry.dbl);
break;
case NDFT_UUID:
if(!uuid_is_null(*fields[i].entry.uuid)) {
char u[UUID_COMPACT_STR_LEN];
uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
buffer_fast_strcat(wb, u, sizeof(u) - 1);
}
break;
case NDFT_CALLBACK: {
if(!tmp)
tmp = buffer_create(1024, NULL);
else
buffer_flush(tmp);
if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
buffer_strcat(wb, key);
buffer_fast_strcat(wb, "=", 1);
string_to_logfmt(wb, buffer_tostring(tmp));
}
}
break;
default:
buffer_strcat(wb, "UNHANDLED");
break;
}
}
}
}

301
src/libnetdata/log/nd_log-init.c Normal file
View file

@ -0,0 +1,301 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
// --------------------------------------------------------------------------------------------------------------------
__attribute__((constructor)) void initialize_invocation_id(void) {
// check for a NETDATA_INVOCATION_ID
if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
// not found, check for systemd set INVOCATION_ID
if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
// not found, generate a new one
uuid_generate_random(nd_log.invocation_id);
}
}
char uuid[UUID_COMPACT_STR_LEN];
uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);
}
// --------------------------------------------------------------------------------------------------------------------
void nd_log_initialize_for_external_plugins(const char *name) {
// if we don't run under Netdata, log to stderr,
// otherwise, use the logging method Netdata wants us to use.
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
nd_setenv("NETDATA_LOG_METHOD", ETW_NAME, 0);
nd_setenv("NETDATA_LOG_FORMAT", ETW_NAME, 0);
#elif defined(HAVE_WEL)
nd_setenv("NETDATA_LOG_METHOD", WEL_NAME, 0);
nd_setenv("NETDATA_LOG_FORMAT", WEL_NAME, 0);
#else
nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
#endif
#else
nd_setenv("NETDATA_LOG_METHOD", "stderr", 0);
nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
#endif
nd_log.overwrite_process_source = NDLS_COLLECTORS;
program_name = name;
for(size_t i = 0; i < _NDLS_MAX ;i++) {
nd_log.sources[i].method = STDERR_FILENO;
nd_log.sources[i].fd = -1;
nd_log.sources[i].fp = NULL;
}
nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
time_t period = 1200;
size_t logs = 200;
const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
if(s && *s >= '0' && *s <= '9') {
period = str2l(s);
if(period < 0) period = 0;
}
s = getenv("NETDATA_ERRORS_PER_PERIOD");
if(s && *s >= '0' && *s <= '9')
logs = str2u(s);
nd_log_set_flood_protection(logs, period);
if(!netdata_configured_host_prefix) {
s = getenv("NETDATA_HOST_PREFIX");
if(s && *s)
netdata_configured_host_prefix = (char *)s;
}
ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
if(is_stderr_connected_to_journal()) {
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
method = NDLM_JOURNAL;
}
else {
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
method = NDLM_STDERR;
}
}
switch(method) {
case NDLM_JOURNAL:
if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
!nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
method = NDLM_STDERR;
}
break;
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
case NDLM_ETW:
if(!nd_log_init_etw()) {
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Events Tracing for Windows (ETW). Using stderr.");
method = NDLM_STDERR;
}
break;
#endif
#if defined(HAVE_WEL)
case NDLM_WEL:
if(!nd_log_init_wel()) {
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Windows Event Log (WEL). Using stderr.");
method = NDLM_STDERR;
}
break;
#endif
#endif
case NDLM_SYSLOG:
nd_log_init_syslog();
break;
default:
method = NDLM_STDERR;
break;
}
for(size_t i = 0; i < _NDLS_MAX ;i++) {
nd_log.sources[i].method = method;
nd_log.sources[i].format = format;
nd_log.sources[i].fd = -1;
nd_log.sources[i].fp = NULL;
}
// nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
}
// --------------------------------------------------------------------------------------------------------------------
void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
if(e->method == NDLM_DEFAULT)
nd_log_set_user_settings(source, e->filename);
if((e->method == NDLM_FILE && !e->filename) ||
(e->method == NDLM_DEVNULL && e->fd == -1))
e->method = NDLM_DISABLED;
if(e->fp)
fflush(e->fp);
switch(e->method) {
case NDLM_SYSLOG:
nd_log_init_syslog();
break;
case NDLM_JOURNAL:
nd_log_journal_direct_init(NULL);
nd_log_journal_systemd_init();
break;
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
case NDLM_ETW:
nd_log_init_etw();
break;
#endif
#if defined(HAVE_WEL)
case NDLM_WEL:
nd_log_init_wel();
break;
#endif
#endif
case NDLM_STDOUT:
e->fp = stdout;
e->fd = STDOUT_FILENO;
break;
case NDLM_DISABLED:
break;
case NDLM_DEFAULT:
case NDLM_STDERR:
e->method = NDLM_STDERR;
e->fp = stderr;
e->fd = STDERR_FILENO;
break;
case NDLM_DEVNULL:
case NDLM_FILE: {
int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
if(fd == -1) {
if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
e->fd = STDERR_FILENO;
e->method = NDLM_STDERR;
netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
}
else
netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
}
else {
if (!nd_log_replace_existing_fd(e, fd)) {
if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
if(e->fd == STDOUT_FILENO)
e->method = NDLM_STDOUT;
else if(e->fd == STDERR_FILENO)
e->method = NDLM_STDERR;
// we have dup2() fd, so we can close the one we opened
if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
close(fd);
}
else
e->fd = fd;
}
}
// at this point we have e->fd set properly
if(e->fd == STDOUT_FILENO)
e->fp = stdout;
else if(e->fd == STDERR_FILENO)
e->fp = stderr;
if(!e->fp) {
e->fp = fdopen(e->fd, "a");
if (!e->fp) {
netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
close(e->fd);
e->fp = stderr;
e->fd = STDERR_FILENO;
}
}
else {
if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
}
}
break;
}
}
// --------------------------------------------------------------------------------------------------------------------
void nd_log_stdin_init(int fd, const char *filename) {
int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
if(f == -1)
return;
if(f != fd) {
dup2(f, fd);
close(f);
}
}
void nd_log_initialize(void) {
nd_log_stdin_init(STDIN_FILENO, "/dev/null");
for(size_t i = 0 ; i < _NDLS_MAX ; i++)
nd_log_open(&nd_log.sources[i], i);
}
void nd_log_reopen_log_files(bool log) {
if(log)
netdata_log_info("Reopening all log files.");
nd_log.std_output.initialized = false;
nd_log.std_error.initialized = false;
nd_log.journal_direct.initialized = false;
nd_log.journal.initialized = false;
nd_log_initialize();
if(log)
netdata_log_info("Log files re-opened.");
}
void nd_log_reopen_log_files_for_spawn_server(void) {
gettid_uncached();
if(nd_log.syslog.initialized) {
closelog();
nd_log.syslog.initialized = false;
nd_log_init_syslog();
}
if(nd_log.journal_direct.initialized) {
close(nd_log.journal_direct.fd);
nd_log.journal_direct.fd = -1;
nd_log.journal_direct.initialized = false;
nd_log_journal_direct_init(NULL);
}
nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED;
nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED;
nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED;
nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED;
nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED;
nd_log_reopen_log_files(false);
}

821
src/libnetdata/log/nd_log-internals.c Normal file
View file

@ -0,0 +1,821 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
// --------------------------------------------------------------------------------------------------------------------
// workaround strerror_r()
#if defined(STRERROR_R_CHAR_P)
// GLIBC version of strerror_r
static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
#elif defined(HAVE_STRERROR_R)
// POSIX version of strerror_r
static const char *strerror_result(int a, const char *b) { (void)a; return b; }
#elif defined(HAVE_C__GENERIC)
// what a trick!
// http://stackoverflow.com/questions/479207/function-overloading-in-c
static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
#define strerror_result(a, b) _Generic((a), \
int: strerror_result_int, \
char *: strerror_result_string \
)(a, b)
#else
#error "cannot detect the format of function strerror_r()"
#endif
const char *errno2str(int errnum, char *buf, size_t size) {
return strerror_result(strerror_r(errnum, buf, size), buf);
}
// --------------------------------------------------------------------------------------------------------------------
// logging method
static struct {
ND_LOG_METHOD method;
const char *name;
} nd_log_methods[] = {
{ .method = NDLM_DISABLED, .name = "none" },
{ .method = NDLM_DEVNULL, .name = "/dev/null" },
{ .method = NDLM_DEFAULT, .name = "default" },
{ .method = NDLM_JOURNAL, .name = "journal" },
{ .method = NDLM_SYSLOG, .name = "syslog" },
{ .method = NDLM_STDOUT, .name = "stdout" },
{ .method = NDLM_STDERR, .name = "stderr" },
{ .method = NDLM_FILE, .name = "file" },
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
{ .method = NDLM_ETW, .name = ETW_NAME },
#endif
#if defined(HAVE_WEL)
{ .method = NDLM_WEL, .name = WEL_NAME },
#endif
#endif
};
ND_LOG_METHOD nd_log_method2id(const char *method) {
if(!method || !*method)
return NDLM_DEFAULT;
size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
for(size_t i = 0; i < entries ;i++) {
if(strcmp(nd_log_methods[i].name, method) == 0)
return nd_log_methods[i].method;
}
return NDLM_FILE;
}
const char *nd_log_id2method(ND_LOG_METHOD method) {
size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
for(size_t i = 0; i < entries ;i++) {
if(method == nd_log_methods[i].method)
return nd_log_methods[i].name;
}
return "unknown";
}
const char *nd_log_method_for_external_plugins(const char *s) {
if(s && *s) {
ND_LOG_METHOD method = nd_log_method2id(s);
if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
return nd_log_id2method(method);
}
return nd_log_id2method(NDLM_STDERR);
}
// --------------------------------------------------------------------------------------------------------------------
// facilities
//
// sys/syslog.h (Linux)
// sys/sys/syslog.h (FreeBSD)
// bsd/sys/syslog.h (darwin-xnu)
static struct {
int facility;
const char *name;
} nd_log_facilities[] = {
{ LOG_AUTH, "auth" },
{ LOG_AUTHPRIV, "authpriv" },
{ LOG_CRON, "cron" },
{ LOG_DAEMON, "daemon" },
{ LOG_FTP, "ftp" },
{ LOG_KERN, "kern" },
{ LOG_LPR, "lpr" },
{ LOG_MAIL, "mail" },
{ LOG_NEWS, "news" },
{ LOG_SYSLOG, "syslog" },
{ LOG_USER, "user" },
{ LOG_UUCP, "uucp" },
{ LOG_LOCAL0, "local0" },
{ LOG_LOCAL1, "local1" },
{ LOG_LOCAL2, "local2" },
{ LOG_LOCAL3, "local3" },
{ LOG_LOCAL4, "local4" },
{ LOG_LOCAL5, "local5" },
{ LOG_LOCAL6, "local6" },
{ LOG_LOCAL7, "local7" },
#ifdef __FreeBSD__
{ LOG_CONSOLE, "console" },
{ LOG_NTP, "ntp" },
// FreeBSD does not consider 'security' as deprecated.
{ LOG_SECURITY, "security" },
#else
// For all other O/S 'security' is mapped to 'auth'.
{ LOG_AUTH, "security" },
#endif
#ifdef __APPLE__
{ LOG_INSTALL, "install" },
{ LOG_NETINFO, "netinfo" },
{ LOG_RAS, "ras" },
{ LOG_REMOTEAUTH, "remoteauth" },
{ LOG_LAUNCHD, "launchd" },
#endif
};
int nd_log_facility2id(const char *facility) {
size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
for(size_t i = 0; i < entries ;i++) {
if(strcmp(nd_log_facilities[i].name, facility) == 0)
return nd_log_facilities[i].facility;
}
return LOG_DAEMON;
}
const char *nd_log_id2facility(int facility) {
size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
for(size_t i = 0; i < entries ;i++) {
if(nd_log_facilities[i].facility == facility)
return nd_log_facilities[i].name;
}
return "daemon";
}
// --------------------------------------------------------------------------------------------------------------------
// priorities
static struct {
ND_LOG_FIELD_PRIORITY priority;
const char *name;
} nd_log_priorities[] = {
{ .priority = NDLP_EMERG, .name = "emergency" },
{ .priority = NDLP_EMERG, .name = "emerg" },
{ .priority = NDLP_ALERT, .name = "alert" },
{ .priority = NDLP_CRIT, .name = "critical" },
{ .priority = NDLP_CRIT, .name = "crit" },
{ .priority = NDLP_ERR, .name = "error" },
{ .priority = NDLP_ERR, .name = "err" },
{ .priority = NDLP_WARNING, .name = "warning" },
{ .priority = NDLP_WARNING, .name = "warn" },
{ .priority = NDLP_NOTICE, .name = "notice" },
{ .priority = NDLP_INFO, .name = NDLP_INFO_STR },
{ .priority = NDLP_DEBUG, .name = "debug" },
};
int nd_log_priority2id(const char *priority) {
size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
for(size_t i = 0; i < entries ;i++) {
if(strcmp(nd_log_priorities[i].name, priority) == 0)
return nd_log_priorities[i].priority;
}
return NDLP_INFO;
}
const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
for(size_t i = 0; i < entries ;i++) {
if(priority == nd_log_priorities[i].priority)
return nd_log_priorities[i].name;
}
return NDLP_INFO_STR;
}
// --------------------------------------------------------------------------------------------------------------------
// log sources
const char *nd_log_sources[] = {
[NDLS_UNSET] = "UNSET",
[NDLS_ACCESS] = "access",
[NDLS_ACLK] = "aclk",
[NDLS_COLLECTORS] = "collector",
[NDLS_DAEMON] = "daemon",
[NDLS_HEALTH] = "health",
[NDLS_DEBUG] = "debug",
};
size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
for(size_t i = 0; i < entries ;i++) {
if(strcmp(nd_log_sources[i], source) == 0)
return i;
}
return def;
}
const char *nd_log_id2source(ND_LOG_SOURCES source) {
size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
if(source < entries)
return nd_log_sources[source];
return nd_log_sources[NDLS_COLLECTORS];
}
// --------------------------------------------------------------------------------------------------------------------
// log output formats
static struct {
ND_LOG_FORMAT format;
const char *name;
} nd_log_formats[] = {
{ .format = NDLF_JOURNAL, .name = "journal" },
{ .format = NDLF_LOGFMT, .name = "logfmt" },
{ .format = NDLF_JSON, .name = "json" },
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
{ .format = NDLF_ETW, .name = ETW_NAME },
#endif
#if defined(HAVE_WEL)
{ .format = NDLF_WEL, .name = WEL_NAME },
#endif
#endif
};
ND_LOG_FORMAT nd_log_format2id(const char *format) {
if(!format || !*format)
return NDLF_LOGFMT;
size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
for(size_t i = 0; i < entries ;i++) {
if(strcmp(nd_log_formats[i].name, format) == 0)
return nd_log_formats[i].format;
}
return NDLF_LOGFMT;
}
const char *nd_log_id2format(ND_LOG_FORMAT format) {
size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
for(size_t i = 0; i < entries ;i++) {
if(format == nd_log_formats[i].format)
return nd_log_formats[i].name;
}
return "logfmt";
}
// --------------------------------------------------------------------------------------------------------------------
struct nd_log nd_log = {
.overwrite_process_source = 0,
.journal = {
.initialized = false,
},
.journal_direct = {
.initialized = false,
.fd = -1,
},
.syslog = {
.initialized = false,
.facility = LOG_DAEMON,
},
#if defined(OS_WINDOWS)
.eventlog = {
.initialized = false,
},
#endif
.std_output = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.initialized = false,
},
.std_error = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.initialized = false,
},
.sources = {
[NDLS_UNSET] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DISABLED,
.format = NDLF_JOURNAL,
.filename = NULL,
.fd = -1,
.fp = NULL,
.min_priority = NDLP_EMERG,
.limits = ND_LOG_LIMITS_UNLIMITED,
},
[NDLS_ACCESS] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DEFAULT,
.format = NDLF_LOGFMT,
.filename = LOG_DIR "/access.log",
.fd = -1,
.fp = NULL,
.min_priority = NDLP_DEBUG,
.limits = ND_LOG_LIMITS_UNLIMITED,
},
[NDLS_ACLK] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_FILE,
.format = NDLF_LOGFMT,
.filename = LOG_DIR "/aclk.log",
.fd = -1,
.fp = NULL,
.min_priority = NDLP_DEBUG,
.limits = ND_LOG_LIMITS_UNLIMITED,
},
[NDLS_COLLECTORS] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DEFAULT,
.format = NDLF_LOGFMT,
.filename = LOG_DIR "/collector.log",
.fd = STDERR_FILENO,
.fp = NULL,
.min_priority = NDLP_INFO,
.limits = ND_LOG_LIMITS_DEFAULT,
},
[NDLS_DEBUG] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DISABLED,
.format = NDLF_LOGFMT,
.filename = LOG_DIR "/debug.log",
.fd = STDOUT_FILENO,
.fp = NULL,
.min_priority = NDLP_DEBUG,
.limits = ND_LOG_LIMITS_UNLIMITED,
},
[NDLS_DAEMON] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DEFAULT,
.filename = LOG_DIR "/daemon.log",
.format = NDLF_LOGFMT,
.fd = -1,
.fp = NULL,
.min_priority = NDLP_INFO,
.limits = ND_LOG_LIMITS_DEFAULT,
},
[NDLS_HEALTH] = {
.spinlock = NETDATA_SPINLOCK_INITIALIZER,
.method = NDLM_DEFAULT,
.format = NDLF_LOGFMT,
.filename = LOG_DIR "/health.log",
.fd = -1,
.fp = NULL,
.min_priority = NDLP_DEBUG,
.limits = ND_LOG_LIMITS_UNLIMITED,
},
},
};
// --------------------------------------------------------------------------------------------------------------------
__thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
__thread size_t thread_log_stack_next = 0;
__thread struct log_field thread_log_fields[_NDF_MAX] = {
// THE ORDER HERE IS IRRELEVANT (but keep them sorted by their number)
[NDF_STOP] = { // processing will not stop on this - so it is ok to be first
.journal = NULL,
.logfmt = NULL,
.eventlog = NULL,
.annotator = NULL,
},
[NDF_TIMESTAMP_REALTIME_USEC] = {
.journal = NULL,
.eventlog = "Timestamp",
.logfmt = "time",
.annotator = timestamp_usec_annotator,
},
[NDF_SYSLOG_IDENTIFIER] = {
.journal = "SYSLOG_IDENTIFIER", // standard journald field
.eventlog = "Program",
.logfmt = "comm",
},
[NDF_LOG_SOURCE] = {
.journal = "ND_LOG_SOURCE",
.eventlog = "NetdataLogSource",
.logfmt = "source",
},
[NDF_PRIORITY] = {
.journal = "PRIORITY", // standard journald field
.eventlog = "Level",
.logfmt = "level",
.annotator = priority_annotator,
},
[NDF_ERRNO] = {
.journal = "ERRNO", // standard journald field
.eventlog = "UnixErrno",
.logfmt = "errno",
.annotator = errno_annotator,
},
[NDF_WINERROR] = {
#if defined(OS_WINDOWS)
.journal = "WINERROR",
.eventlog = "WindowsLastError",
.logfmt = "winerror",
.annotator = winerror_annotator,
#endif
},
[NDF_INVOCATION_ID] = {
.journal = "INVOCATION_ID", // standard journald field
.eventlog = "InvocationID",
.logfmt = NULL,
},
[NDF_LINE] = {
.journal = "CODE_LINE", // standard journald field
.eventlog = "CodeLine",
.logfmt = NULL,
},
[NDF_FILE] = {
.journal = "CODE_FILE", // standard journald field
.eventlog = "CodeFile",
.logfmt = NULL,
},
[NDF_FUNC] = {
.journal = "CODE_FUNC", // standard journald field
.eventlog = "CodeFunction",
.logfmt = NULL,
},
[NDF_TID] = {
.journal = "TID", // standard journald field
.eventlog = "ThreadID",
.logfmt = "tid",
},
[NDF_THREAD_TAG] = {
.journal = "THREAD_TAG",
.eventlog = "ThreadName",
.logfmt = "thread",
},
[NDF_MESSAGE_ID] = {
.journal = "MESSAGE_ID",
.eventlog = "MessageID",
.logfmt = "msg_id",
},
[NDF_MODULE] = {
.journal = "ND_MODULE",
.eventlog = "Module",
.logfmt = "module",
},
[NDF_NIDL_NODE] = {
.journal = "ND_NIDL_NODE",
.eventlog = "Node",
.logfmt = "node",
},
[NDF_NIDL_INSTANCE] = {
.journal = "ND_NIDL_INSTANCE",
.eventlog = "Instance",
.logfmt = "instance",
},
[NDF_NIDL_CONTEXT] = {
.journal = "ND_NIDL_CONTEXT",
.eventlog = "Context",
.logfmt = "context",
},
[NDF_NIDL_DIMENSION] = {
.journal = "ND_NIDL_DIMENSION",
.eventlog = "Dimension",
.logfmt = "dimension",
},
[NDF_SRC_TRANSPORT] = {
.journal = "ND_SRC_TRANSPORT",
.eventlog = "SourceTransport",
.logfmt = "src_transport",
},
[NDF_ACCOUNT_ID] = {
.journal = "ND_ACCOUNT_ID",
.eventlog = "AccountID",
.logfmt = "account",
},
[NDF_USER_NAME] = {
.journal = "ND_USER_NAME",
.eventlog = "UserName",
.logfmt = "user",
},
[NDF_USER_ROLE] = {
.journal = "ND_USER_ROLE",
.eventlog = "UserRole",
.logfmt = "role",
},
[NDF_USER_ACCESS] = {
.journal = "ND_USER_PERMISSIONS",
.eventlog = "UserPermissions",
.logfmt = "permissions",
},
[NDF_SRC_IP] = {
.journal = "ND_SRC_IP",
.eventlog = "SourceIP",
.logfmt = "src_ip",
},
[NDF_SRC_FORWARDED_HOST] = {
.journal = "ND_SRC_FORWARDED_HOST",
.eventlog = "SourceForwardedHost",
.logfmt = "src_forwarded_host",
},
[NDF_SRC_FORWARDED_FOR] = {
.journal = "ND_SRC_FORWARDED_FOR",
.eventlog = "SourceForwardedFor",
.logfmt = "src_forwarded_for",
},
[NDF_SRC_PORT] = {
.journal = "ND_SRC_PORT",
.eventlog = "SourcePort",
.logfmt = "src_port",
},
[NDF_SRC_CAPABILITIES] = {
.journal = "ND_SRC_CAPABILITIES",
.eventlog = "SourceCapabilities",
.logfmt = "src_capabilities",
},
[NDF_DST_TRANSPORT] = {
.journal = "ND_DST_TRANSPORT",
.eventlog = "DestinationTransport",
.logfmt = "dst_transport",
},
[NDF_DST_IP] = {
.journal = "ND_DST_IP",
.eventlog = "DestinationIP",
.logfmt = "dst_ip",
},
[NDF_DST_PORT] = {
.journal = "ND_DST_PORT",
.eventlog = "DestinationPort",
.logfmt = "dst_port",
},
[NDF_DST_CAPABILITIES] = {
.journal = "ND_DST_CAPABILITIES",
.eventlog = "DestinationCapabilities",
.logfmt = "dst_capabilities",
},
[NDF_REQUEST_METHOD] = {
.journal = "ND_REQUEST_METHOD",
.eventlog = "RequestMethod",
.logfmt = "req_method",
},
[NDF_RESPONSE_CODE] = {
.journal = "ND_RESPONSE_CODE",
.eventlog = "ResponseCode",
.logfmt = "code",
},
[NDF_CONNECTION_ID] = {
.journal = "ND_CONNECTION_ID",
.eventlog = "ConnectionID",
.logfmt = "conn",
},
[NDF_TRANSACTION_ID] = {
.journal = "ND_TRANSACTION_ID",
.eventlog = "TransactionID",
.logfmt = "transaction",
},
[NDF_RESPONSE_SENT_BYTES] = {
.journal = "ND_RESPONSE_SENT_BYTES",
.eventlog = "ResponseSentBytes",
.logfmt = "sent_bytes",
},
[NDF_RESPONSE_SIZE_BYTES] = {
.journal = "ND_RESPONSE_SIZE_BYTES",
.eventlog = "ResponseSizeBytes",
.logfmt = "size_bytes",
},
[NDF_RESPONSE_PREPARATION_TIME_USEC] = {
.journal = "ND_RESPONSE_PREP_TIME_USEC",
.eventlog = "ResponsePreparationTimeUsec",
.logfmt = "prep_ut",
},
[NDF_RESPONSE_SENT_TIME_USEC] = {
.journal = "ND_RESPONSE_SENT_TIME_USEC",
.eventlog = "ResponseSentTimeUsec",
.logfmt = "sent_ut",
},
[NDF_RESPONSE_TOTAL_TIME_USEC] = {
.journal = "ND_RESPONSE_TOTAL_TIME_USEC",
.eventlog = "ResponseTotalTimeUsec",
.logfmt = "total_ut",
},
[NDF_ALERT_ID] = {
.journal = "ND_ALERT_ID",
.eventlog = "AlertID",
.logfmt = "alert_id",
},
[NDF_ALERT_UNIQUE_ID] = {
.journal = "ND_ALERT_UNIQUE_ID",
.eventlog = "AlertUniqueID",
.logfmt = "alert_unique_id",
},
[NDF_ALERT_TRANSITION_ID] = {
.journal = "ND_ALERT_TRANSITION_ID",
.eventlog = "AlertTransitionID",
.logfmt = "alert_transition_id",
},
[NDF_ALERT_EVENT_ID] = {
.journal = "ND_ALERT_EVENT_ID",
.eventlog = "AlertEventID",
.logfmt = "alert_event_id",
},
[NDF_ALERT_CONFIG_HASH] = {
.journal = "ND_ALERT_CONFIG",
.eventlog = "AlertConfig",
.logfmt = "alert_config",
},
[NDF_ALERT_NAME] = {
.journal = "ND_ALERT_NAME",
.eventlog = "AlertName",
.logfmt = "alert",
},
[NDF_ALERT_CLASS] = {
.journal = "ND_ALERT_CLASS",
.eventlog = "AlertClass",
.logfmt = "alert_class",
},
[NDF_ALERT_COMPONENT] = {
.journal = "ND_ALERT_COMPONENT",
.eventlog = "AlertComponent",
.logfmt = "alert_component",
},
[NDF_ALERT_TYPE] = {
.journal = "ND_ALERT_TYPE",
.eventlog = "AlertType",
.logfmt = "alert_type",
},
[NDF_ALERT_EXEC] = {
.journal = "ND_ALERT_EXEC",
.eventlog = "AlertExec",
.logfmt = "alert_exec",
},
[NDF_ALERT_RECIPIENT] = {
.journal = "ND_ALERT_RECIPIENT",
.eventlog = "AlertRecipient",
.logfmt = "alert_recipient",
},
[NDF_ALERT_VALUE] = {
.journal = "ND_ALERT_VALUE",
.eventlog = "AlertValue",
.logfmt = "alert_value",
},
[NDF_ALERT_VALUE_OLD] = {
.journal = "ND_ALERT_VALUE_OLD",
.eventlog = "AlertOldValue",
.logfmt = "alert_value_old",
},
[NDF_ALERT_STATUS] = {
.journal = "ND_ALERT_STATUS",
.eventlog = "AlertStatus",
.logfmt = "alert_status",
},
[NDF_ALERT_STATUS_OLD] = {
.journal = "ND_ALERT_STATUS_OLD",
.eventlog = "AlertOldStatus",
.logfmt = "alert_value_old",
},
[NDF_ALERT_UNITS] = {
.journal = "ND_ALERT_UNITS",
.eventlog = "AlertUnits",
.logfmt = "alert_units",
},
[NDF_ALERT_SUMMARY] = {
.journal = "ND_ALERT_SUMMARY",
.eventlog = "AlertSummary",
.logfmt = "alert_summary",
},
[NDF_ALERT_INFO] = {
.journal = "ND_ALERT_INFO",
.eventlog = "AlertInfo",
.logfmt = "alert_info",
},
[NDF_ALERT_DURATION] = {
.journal = "ND_ALERT_DURATION",
.eventlog = "AlertDuration",
.logfmt = "alert_duration",
},
[NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
.journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
.eventlog = "AlertNotificationTime",
.logfmt = "alert_notification_timestamp",
.annotator = timestamp_usec_annotator,
},
// put new items here
// leave the request URL and the message last
[NDF_REQUEST] = {
.journal = "ND_REQUEST",
.eventlog = "Request",
.logfmt = "request",
},
[NDF_MESSAGE] = {
.journal = "MESSAGE",
.eventlog = "Message",
.logfmt = "msg",
},
};
// --------------------------------------------------------------------------------------------------------------------
void log_stack_pop(void *ptr) {
if(!ptr) return;
struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
fatal("You cannot pop in the middle of the stack, or an item not in the stack");
return;
}
thread_log_stack_next--;
}
void log_stack_push(struct log_stack_entry *lgs) {
if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
thread_log_stack_base[thread_log_stack_next++] = lgs;
}
// --------------------------------------------------------------------------------------------------------------------
ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len) {
for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
return i;
}
return NDF_STOP;
}
// --------------------------------------------------------------------------------------------------------------------
int nd_log_health_fd(void) {
if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
return nd_log.sources[NDLS_HEALTH].fd;
return STDERR_FILENO;
}
int nd_log_collectors_fd(void) {
if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
return nd_log.sources[NDLS_COLLECTORS].fd;
return STDERR_FILENO;
}
// --------------------------------------------------------------------------------------------------------------------
void log_date(char *buffer, size_t len, time_t now) {
if(unlikely(!buffer || !len))
return;
time_t t = now;
struct tm *tmp, tmbuf;
tmp = localtime_r(&t, &tmbuf);
if (unlikely(!tmp)) {
buffer[0] = '\0';
return;
}
if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
buffer[0] = '\0';
buffer[len - 1] = '\0';
}
// --------------------------------------------------------------------------------------------------------------------
bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
if(new_fd == -1 || e->fd == -1 ||
(e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
(e->fd == STDERR_FILENO && nd_log.std_error.initialized))
return false;
if(new_fd != e->fd) {
int t = dup2(new_fd, e->fd);
bool ret = true;
if (t == -1) {
netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
ret = false;
}
else
close(new_fd);
if(e->fd == STDOUT_FILENO)
nd_log.std_output.initialized = true;
else if(e->fd == STDERR_FILENO)
nd_log.std_error.initialized = true;
return ret;
}
return false;
}

251
src/libnetdata/log/nd_log-internals.h Normal file
View file

@ -0,0 +1,251 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_ND_LOG_INTERNALS_H
#define NETDATA_ND_LOG_INTERNALS_H
#include "../libnetdata.h"
#if defined(OS_WINDOWS)
#include <windows.h>
#endif
#ifdef __FreeBSD__
#include <sys/endian.h>
#endif
#ifdef __APPLE__
#include <machine/endian.h>
#endif
#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
#include <execinfo.h>
#endif
#ifdef HAVE_SYSTEMD
#include <systemd/sd-journal.h>
#endif
const char *errno2str(int errnum, char *buf, size_t size);
// --------------------------------------------------------------------------------------------------------------------
// ND_LOG_METHOD
typedef enum __attribute__((__packed__)) {
NDLM_DISABLED = 0,
NDLM_DEVNULL,
NDLM_DEFAULT,
NDLM_JOURNAL,
NDLM_SYSLOG,
NDLM_STDOUT,
NDLM_STDERR,
NDLM_FILE,
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
NDLM_ETW,
#endif
#if defined(HAVE_WEL)
NDLM_WEL,
#endif
#endif
} ND_LOG_METHOD;
// all the log methods are finally mapped to these
#if defined(HAVE_ETW)
#define ETW_CONDITION(ndlo) ((ndlo) == NDLM_ETW)
#else
#define ETW_CONDITION(ndlo) (false)
#endif
#if defined(HAVE_WEL)
#define WEL_CONDITION(ndlo) ((ndlo) == NDLM_WEL)
#else
#define WEL_CONDITION(ndlo) (false)
#endif
#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
#define IS_FINAL_LOG_METHOD(ndlo) ((ndlo) == NDLM_FILE || (ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo))
ND_LOG_METHOD nd_log_method2id(const char *method);
const char *nd_log_id2method(ND_LOG_METHOD method);
// --------------------------------------------------------------------------------------------------------------------
// ND_LOG_FORMAT
typedef enum __attribute__((__packed__)) {
NDLF_JOURNAL,
NDLF_LOGFMT,
NDLF_JSON,
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
NDLF_ETW, // Event Tracing for Windows
#endif
#if defined(HAVE_WEL)
NDLF_WEL, // Windows Event Log
#endif
#endif
} ND_LOG_FORMAT;
#define ETW_NAME "etw"
#define WEL_NAME "wel"
const char *nd_log_id2format(ND_LOG_FORMAT format);
ND_LOG_FORMAT nd_log_format2id(const char *format);
size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def);
const char *nd_log_id2source(ND_LOG_SOURCES source);
const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
int nd_log_priority2id(const char *priority);
const char *nd_log_id2facility(int facility);
int nd_log_facility2id(const char *facility);
#include "nd_log_limit.h"
struct nd_log_source {
SPINLOCK spinlock;
ND_LOG_METHOD method;
ND_LOG_FORMAT format;
const char *filename;
int fd;
FILE *fp;
ND_LOG_FIELD_PRIORITY min_priority;
const char *pending_msg;
struct nd_log_limit limits;
#if defined(OS_WINDOWS)
ND_LOG_SOURCES source;
HANDLE hEventLog;
USHORT channelID;
UCHAR Opcode;
USHORT Task;
ULONGLONG Keyword;
#endif
};
struct nd_log {
nd_uuid_t invocation_id;
ND_LOG_SOURCES overwrite_process_source;
struct nd_log_source sources[_NDLS_MAX];
struct {
bool initialized;
} journal;
struct {
bool initialized;
int fd;
char filename[FILENAME_MAX + 1];
} journal_direct;
struct {
bool initialized;
int facility;
} syslog;
struct {
bool etw; // when set use etw, otherwise wel
bool initialized;
} eventlog;
struct {
SPINLOCK spinlock;
bool initialized;
} std_output;
struct {
SPINLOCK spinlock;
bool initialized;
} std_error;
};
// --------------------------------------------------------------------------------------------------------------------
struct log_field;
typedef const char *(*annotator_t)(struct log_field *lf);
struct log_field {
const char *journal;
const char *logfmt;
const char *eventlog;
annotator_t annotator;
struct log_stack_entry entry;
};
#define THREAD_LOG_STACK_MAX 50
#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
extern __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
extern __thread size_t thread_log_stack_next;
extern __thread struct log_field thread_log_fields[_NDF_MAX];
// --------------------------------------------------------------------------------------------------------------------
extern struct nd_log nd_log;
bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd);
void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source);
void nd_log_stdin_init(int fd, const char *filename);
// --------------------------------------------------------------------------------------------------------------------
// annotators
struct log_field;
const char *errno_annotator(struct log_field *lf);
const char *priority_annotator(struct log_field *lf);
const char *timestamp_usec_annotator(struct log_field *lf);
#if defined(OS_WINDOWS)
const char *winerror_annotator(struct log_field *lf);
#endif
// --------------------------------------------------------------------------------------------------------------------
// field formatters
uint64_t log_field_to_uint64(struct log_field *lf);
int64_t log_field_to_int64(struct log_field *lf);
// --------------------------------------------------------------------------------------------------------------------
// common text formatters
void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max);
void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max);
// --------------------------------------------------------------------------------------------------------------------
// output to syslog
void nd_log_init_syslog(void);
void nd_log_reset_syslog(void);
bool nd_logger_syslog(int priority, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
// --------------------------------------------------------------------------------------------------------------------
// output to systemd-journal
bool nd_log_journal_systemd_init(void);
bool nd_log_journal_direct_init(const char *path);
bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max);
bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max);
// --------------------------------------------------------------------------------------------------------------------
// output to file
bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max);
// --------------------------------------------------------------------------------------------------------------------
// output to windows events log
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
bool nd_log_init_etw(void);
bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
#endif
#if defined(HAVE_WEL)
bool nd_log_init_wel(void);
bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max);
#endif
#endif
#endif //NETDATA_ND_LOG_INTERNALS_H

41
src/libnetdata/log/nd_log-to-file.c Normal file
View file

@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
void chown_open_file(int fd, uid_t uid, gid_t gid) {
if(fd == -1) return;
struct stat buf;
if(fstat(fd, &buf) == -1) {
netdata_log_error("Cannot fstat() fd %d", fd);
return;
}
if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
if(fchown(fd, uid, gid) == -1)
netdata_log_error("Cannot fchown() fd %d.", fd);
}
}
void nd_log_chown_log_files(uid_t uid, gid_t gid) {
for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
chown_open_file(nd_log.sources[i].fd, uid, gid);
}
}
bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
BUFFER *wb = buffer_create(1024, NULL);
if(format == NDLF_JSON)
nd_logger_json(wb, fields, fields_max);
else
nd_logger_logfmt(wb, fields, fields_max);
int r = fprintf(fp, "%s\n", buffer_tostring(wb));
fflush(fp);
buffer_free(wb);
return r > 0;
}

20
src/libnetdata/log/nd_log-to-syslog.c Normal file
View file

@ -0,0 +1,20 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
void nd_log_init_syslog(void) {
if(nd_log.syslog.initialized)
return;
openlog(program_name, LOG_PID, nd_log.syslog.facility);
nd_log.syslog.initialized = true;
}
bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) {
CLEAN_BUFFER *wb = buffer_create(1024, NULL);
nd_logger_logfmt(wb, fields, fields_max);
syslog(priority, "%s", buffer_tostring(wb));
return true;
}

273
src/libnetdata/log/nd_log-to-systemd-journal.c Normal file
View file

@ -0,0 +1,273 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
bool nd_log_journal_systemd_init(void) {
#ifdef HAVE_SYSTEMD
nd_log.journal.initialized = true;
#else
nd_log.journal.initialized = false;
#endif
return nd_log.journal.initialized;
}
bool nd_log_journal_socket_available(void) {
if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
char filename[FILENAME_MAX + 1];
snprintfz(filename, sizeof(filename), "%s%s",
netdata_configured_host_prefix, "/run/systemd/journal/socket");
if(is_path_unix_socket(filename))
return true;
}
return is_path_unix_socket("/run/systemd/journal/socket");
}
static void nd_log_journal_direct_set_env(void) {
if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
}
bool nd_log_journal_direct_init(const char *path) {
if(nd_log.journal_direct.initialized) {
nd_log_journal_direct_set_env();
return true;
}
int fd;
char filename[FILENAME_MAX + 1];
if(!is_path_unix_socket(path)) {
journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
journal_construct_path(filename, sizeof(filename), NULL, "netdata");
if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
journal_construct_path(filename, sizeof(filename), NULL, NULL);
if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
return false;
}
}
}
}
else {
snprintfz(filename, sizeof(filename), "%s", path);
fd = journal_direct_fd(filename);
}
if(fd < 0)
return false;
nd_log.journal_direct.fd = fd;
nd_log.journal_direct.initialized = true;
strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
nd_log_journal_direct_set_env();
return true;
}
bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) {
#ifdef HAVE_SYSTEMD
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
struct iovec iov[fields_max];
int iov_count = 0;
memset(iov, 0, sizeof(iov));
CLEAN_BUFFER *tmp = NULL;
for (size_t i = 0; i < fields_max; i++) {
if (!fields[i].entry.set || !fields[i].journal)
continue;
const char *key = fields[i].journal;
char *value = NULL;
int rc = 0;
switch (fields[i].entry.type) {
case NDFT_TXT:
if(*fields[i].entry.txt)
rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt);
break;
case NDFT_STR:
rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
break;
case NDFT_BFR:
if(buffer_strlen(fields[i].entry.bfr))
rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
break;
case NDFT_U64:
rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
break;
case NDFT_I64:
rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
break;
case NDFT_DBL:
rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
break;
case NDFT_UUID:
if(!uuid_is_null(*fields[i].entry.uuid)) {
char u[UUID_COMPACT_STR_LEN];
uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
rc = asprintf(&value, "%s=%s", key, u);
}
break;
case NDFT_CALLBACK: {
if(!tmp)
tmp = buffer_create(1024, NULL);
else
buffer_flush(tmp);
if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
}
break;
default:
rc = asprintf(&value, "%s=%s", key, "UNHANDLED");
break;
}
if (rc != -1 && value) {
iov[iov_count].iov_base = value;
iov[iov_count].iov_len = strlen(value);
iov_count++;
}
}
int r = sd_journal_sendv(iov, iov_count);
// Clean up allocated memory
for (int i = 0; i < iov_count; i++) {
if (iov[i].iov_base != NULL) {
free(iov[i].iov_base);
}
}
return r == 0;
#else
return false;
#endif
}
bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
if(!nd_log.journal_direct.initialized)
return false;
// --- FIELD_PARSER_VERSIONS ---
//
// IMPORTANT:
// THERE ARE 6 VERSIONS OF THIS CODE
//
// 1. journal (direct socket API),
// 2. journal (libsystemd API),
// 3. logfmt,
// 4. json,
// 5. convert to uint64
// 6. convert to int64
//
// UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
CLEAN_BUFFER *wb = buffer_create(4096, NULL);
CLEAN_BUFFER *tmp = NULL;
for (size_t i = 0; i < fields_max; i++) {
if (!fields[i].entry.set || !fields[i].journal)
continue;
const char *key = fields[i].journal;
const char *s = NULL;
switch(fields[i].entry.type) {
case NDFT_TXT:
s = fields[i].entry.txt;
break;
case NDFT_STR:
s = string2str(fields[i].entry.str);
break;
case NDFT_BFR:
s = buffer_tostring(fields[i].entry.bfr);
break;
case NDFT_U64:
buffer_strcat(wb, key);
buffer_putc(wb, '=');
buffer_print_uint64(wb, fields[i].entry.u64);
buffer_putc(wb, '\n');
break;
case NDFT_I64:
buffer_strcat(wb, key);
buffer_putc(wb, '=');
buffer_print_int64(wb, fields[i].entry.i64);
buffer_putc(wb, '\n');
break;
case NDFT_DBL:
buffer_strcat(wb, key);
buffer_putc(wb, '=');
buffer_print_netdata_double(wb, fields[i].entry.dbl);
buffer_putc(wb, '\n');
break;
case NDFT_UUID:
if(!uuid_is_null(*fields[i].entry.uuid)) {
char u[UUID_COMPACT_STR_LEN];
uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
buffer_strcat(wb, key);
buffer_putc(wb, '=');
buffer_fast_strcat(wb, u, sizeof(u) - 1);
buffer_putc(wb, '\n');
}
break;
case NDFT_CALLBACK: {
if(!tmp)
tmp = buffer_create(1024, NULL);
else
buffer_flush(tmp);
if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
s = buffer_tostring(tmp);
else
s = NULL;
}
break;
default:
s = "UNHANDLED";
break;
}
if(s && *s) {
buffer_strcat(wb, key);
if(!strchr(s, '\n')) {
buffer_putc(wb, '=');
buffer_strcat(wb, s);
buffer_putc(wb, '\n');
}
else {
buffer_putc(wb, '\n');
size_t size = strlen(s);
uint64_t le_size = htole64(size);
buffer_memcat(wb, &le_size, sizeof(le_size));
buffer_memcat(wb, s, size);
buffer_putc(wb, '\n');
}
}
}
return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
}

188
src/libnetdata/log/nd_log-to-windows-common.h Normal file
View file

@ -0,0 +1,188 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
#define NETDATA_ND_LOG_TO_WINDOWS_COMMON_H
// Helper macro to create wide string literals
#define WIDEN2(x) L ## x
#define WIDEN(x) WIDEN2(x)
#define NETDATA_ETW_PROVIDER_GUID_STR "{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
#define NETDATA_ETW_PROVIDER_GUID_STR_W WIDEN(NETDATA_ETW_PROVIDER_GUID)
#define NETDATA_CHANNEL_NAME "Netdata"
#define NETDATA_CHANNEL_NAME_W WIDEN(NETDATA_CHANNEL_NAME)
#define NETDATA_WEL_CHANNEL_NAME "NetdataWEL"
#define NETDATA_WEL_CHANNEL_NAME_W WIDEN(NETDATA_WEL_CHANNEL_NAME)
#define NETDATA_ETW_CHANNEL_NAME "Netdata"
#define NETDATA_ETW_CHANNEL_NAME_W WIDEN(NETDATA_ETW_CHANNEL_NAME)
#define NETDATA_ETW_PROVIDER_NAME "Netdata"
#define NETDATA_ETW_PROVIDER_NAME_W WIDEN(NETDATA_ETW_PROVIDER_NAME)
#define NETDATA_WEL_PROVIDER_PREFIX "Netdata"
#define NETDATA_WEL_PROVIDER_PREFIX_W WIDEN(NETDATA_WEL_PROVIDER_PREFIX)
#define NETDATA_WEL_PROVIDER_ACCESS NETDATA_WEL_PROVIDER_PREFIX "Access"
#define NETDATA_WEL_PROVIDER_ACCESS_W WIDEN(NETDATA_WEL_PROVIDER_ACCESS)
#define NETDATA_WEL_PROVIDER_ACLK NETDATA_WEL_PROVIDER_PREFIX "Aclk"
#define NETDATA_WEL_PROVIDER_ACLK_W WIDEN(NETDATA_WEL_PROVIDER_ACLK)
#define NETDATA_WEL_PROVIDER_COLLECTORS NETDATA_WEL_PROVIDER_PREFIX "Collectors"
#define NETDATA_WEL_PROVIDER_COLLECTORS_W WIDEN(NETDATA_WEL_PROVIDER_COLLECTORS)
#define NETDATA_WEL_PROVIDER_DAEMON NETDATA_WEL_PROVIDER_PREFIX "Daemon"
#define NETDATA_WEL_PROVIDER_DAEMON_W WIDEN(NETDATA_WEL_PROVIDER_DAEMON)
#define NETDATA_WEL_PROVIDER_HEALTH NETDATA_WEL_PROVIDER_PREFIX "Health"
#define NETDATA_WEL_PROVIDER_HEALTH_W WIDEN(NETDATA_WEL_PROVIDER_HEALTH)
#define NETDATA_ETW_SUBCHANNEL_ACCESS "Access"
#define NETDATA_ETW_SUBCHANNEL_ACCESS_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACCESS)
#define NETDATA_ETW_SUBCHANNEL_ACLK "Aclk"
#define NETDATA_ETW_SUBCHANNEL_ACLK_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACLK)
#define NETDATA_ETW_SUBCHANNEL_COLLECTORS "Collectors"
#define NETDATA_ETW_SUBCHANNEL_COLLECTORS_W WIDEN(NETDATA_ETW_SUBCHANNEL_COLLECTORS)
#define NETDATA_ETW_SUBCHANNEL_DAEMON "Daemon"
#define NETDATA_ETW_SUBCHANNEL_DAEMON_W WIDEN(NETDATA_ETW_SUBCHANNEL_DAEMON)
#define NETDATA_ETW_SUBCHANNEL_HEALTH "Health"
#define NETDATA_ETW_SUBCHANNEL_HEALTH_W WIDEN(NETDATA_ETW_SUBCHANNEL_HEALTH)
// Define shift values
#define EVENT_ID_SEV_SHIFT 30
#define EVENT_ID_C_SHIFT 29
#define EVENT_ID_R_SHIFT 28
#define EVENT_ID_FACILITY_SHIFT 16
#define EVENT_ID_CODE_SHIFT 0
#define EVENT_ID_PRIORITY_SHIFT 0 // Shift 0 bits
#define EVENT_ID_SOURCE_SHIFT 4 // Shift 4 bits
// Define masks
#define EVENT_ID_SEV_MASK 0xC0000000 // Bits 31-30
#define EVENT_ID_C_MASK 0x20000000 // Bit 29
#define EVENT_ID_R_MASK 0x10000000 // Bit 28
#define EVENT_ID_FACILITY_MASK 0x0FFF0000 // Bits 27-16
#define EVENT_ID_CODE_MASK 0x0000FFFF // Bits 15-0
#define EVENT_ID_PRIORITY_MASK 0x000F // Bits 0-3
#define EVENT_ID_SOURCE_MASK 0x00F0 // Bits 4-7
typedef enum __attribute__((packed)) {
MSGID_MESSAGE_ONLY = 1,
MSGID_MESSAGE_ERRNO,
MSGID_REQUEST_ONLY,
MSGID_ALERT_TRANSITION,
MSGID_ACCESS,
MSGID_ACCESS_FORWARDER,
MSGID_ACCESS_USER,
MSGID_ACCESS_FORWARDER_USER,
MSGID_ACCESS_MESSAGE,
MSGID_ACCESS_MESSAGE_REQUEST,
MSGID_ACCESS_MESSAGE_USER,
// terminator
_MSGID_MAX,
} MESSAGE_ID;
static inline uint32_t get_event_type_from_priority(ND_LOG_FIELD_PRIORITY priority) {
switch (priority) {
case NDLP_EMERG:
case NDLP_ALERT:
case NDLP_CRIT:
case NDLP_ERR:
return EVENTLOG_ERROR_TYPE;
case NDLP_WARNING:
return EVENTLOG_WARNING_TYPE;
case NDLP_NOTICE:
case NDLP_INFO:
case NDLP_DEBUG:
default:
return EVENTLOG_INFORMATION_TYPE;
}
}
static inline uint8_t get_severity_from_priority(ND_LOG_FIELD_PRIORITY priority) {
switch (priority) {
case NDLP_EMERG:
case NDLP_ALERT:
case NDLP_CRIT:
case NDLP_ERR:
return STATUS_SEVERITY_ERROR;
case NDLP_WARNING:
return STATUS_SEVERITY_WARNING;
case NDLP_NOTICE:
case NDLP_INFO:
case NDLP_DEBUG:
default:
return STATUS_SEVERITY_INFORMATIONAL;
}
}
static inline uint8_t get_level_from_priority(ND_LOG_FIELD_PRIORITY priority) {
switch (priority) {
// return 0 = log an event regardless of any filtering applied
case NDLP_EMERG:
case NDLP_ALERT:
case NDLP_CRIT:
return 1;
case NDLP_ERR:
return 2;
case NDLP_WARNING:
return 3;
case NDLP_NOTICE:
case NDLP_INFO:
return 4;
case NDLP_DEBUG:
default:
return 5;
}
}
static inline const char *get_level_from_priority_str(ND_LOG_FIELD_PRIORITY priority) {
switch (priority) {
// return "win:LogAlways" to log an event regardless of any filtering applied
case NDLP_EMERG:
case NDLP_ALERT:
case NDLP_CRIT:
return "win:Critical";
case NDLP_ERR:
return "win:Error";
case NDLP_WARNING:
return "win:Warning";
case NDLP_NOTICE:
case NDLP_INFO:
return "win:Informational";
case NDLP_DEBUG:
default:
return "win:Verbose";
}
}
static inline uint16_t construct_event_code(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
return (source << 12 | priority << 8 | messageID << 0);
}
#endif //NETDATA_ND_LOG_TO_WINDOWS_COMMON_H

560
src/libnetdata/log/nd_log-to-windows-events.c Normal file
View file

@ -0,0 +1,560 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log-internals.h"
#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
#include <windows.h>
#include <winevt.h>
#include <evntprov.h>
#include <wchar.h>
#include <guiddef.h>
#include <wctype.h>
// --------------------------------------------------------------------------------------------------------------------
// construct an event id
// load message resources generated header
#include "wevt_netdata.h"
// include the common definitions with the message resources and manifest generator
#include "nd_log-to-windows-common.h"
#if defined(HAVE_ETW)
// we need the manifest, only in ETW mode
// eliminate compiler warnings and load manifest generated header
#undef EXTERN_C
#define EXTERN_C
#undef __declspec
#define __declspec(x)
#include "wevt_netdata_manifest.h"
static REGHANDLE regHandle;
#endif
// Function to construct EventID
static DWORD complete_event_id(DWORD facility, DWORD severity, DWORD event_code) {
DWORD event_id = 0;
// Set Severity
event_id |= ((DWORD)(severity) << EVENT_ID_SEV_SHIFT) & EVENT_ID_SEV_MASK;
// Set Customer Code Flag (C)
event_id |= (0x0 << EVENT_ID_C_SHIFT) & EVENT_ID_C_MASK;
// Set Reserved Bit (R) - typically 0
event_id |= (0x0 << EVENT_ID_R_SHIFT) & EVENT_ID_R_MASK;
// Set Facility
event_id |= ((DWORD)(facility) << EVENT_ID_FACILITY_SHIFT) & EVENT_ID_FACILITY_MASK;
// Set Code
event_id |= ((DWORD)(event_code) << EVENT_ID_CODE_SHIFT) & EVENT_ID_CODE_MASK;
return event_id;
}
DWORD construct_event_id(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) {
DWORD event_code = construct_event_code(source, priority, messageID);
return complete_event_id(FACILITY_NETDATA, get_severity_from_priority(priority), event_code);
}
static bool check_event_id(ND_LOG_SOURCES source __maybe_unused, ND_LOG_FIELD_PRIORITY priority __maybe_unused, MESSAGE_ID messageID __maybe_unused, DWORD event_code __maybe_unused) {
#ifdef NETDATA_INTERNAL_CHECKS
DWORD generated = construct_event_id(source, priority, messageID);
if(generated != event_code) {
// this is just used for a break point, to see the values in hex
char current[UINT64_HEX_MAX_LENGTH];
print_uint64_hex(current, generated);
char wanted[UINT64_HEX_MAX_LENGTH];
print_uint64_hex(wanted, event_code);
const char *got = current;
const char *good = wanted;
internal_fatal(true, "EventIDs mismatch, expected %s, got %s", good, got);
}
#endif
return true;
}
// --------------------------------------------------------------------------------------------------------------------
// initialization
// Define provider names per source (only when not using ETW)
static const wchar_t *wel_provider_per_source[_NDLS_MAX] = {
[NDLS_UNSET] = NULL, // not used, linked to NDLS_DAEMON
[NDLS_ACCESS] = NETDATA_WEL_PROVIDER_ACCESS_W, //
[NDLS_ACLK] = NETDATA_WEL_PROVIDER_ACLK_W, //
[NDLS_COLLECTORS] = NETDATA_WEL_PROVIDER_COLLECTORS_W,//
[NDLS_DAEMON] = NETDATA_WEL_PROVIDER_DAEMON_W, //
[NDLS_HEALTH] = NETDATA_WEL_PROVIDER_HEALTH_W, //
[NDLS_DEBUG] = NULL, // used, linked to NDLS_DAEMON
};
bool wel_replace_program_with_wevt_netdata_dll(wchar_t *str, size_t size) {
const wchar_t *replacement = L"\\wevt_netdata.dll";
// Find the last occurrence of '\\' to isolate the filename
wchar_t *lastBackslash = wcsrchr(str, L'\\');
if (lastBackslash != NULL) {
// Calculate new length after replacement
size_t newLen = (lastBackslash - str) + wcslen(replacement);
// Ensure new length does not exceed buffer size
if (newLen >= size)
return false; // Not enough space in the buffer
// Terminate the string at the last backslash
*lastBackslash = L'\0';
// Append the replacement filename
wcsncat(str, replacement, size - wcslen(str) - 1);
// Check if the new file exists
if (GetFileAttributesW(str) != INVALID_FILE_ATTRIBUTES)
return true; // The file exists
else
return false; // The file does not exist
}
return false; // No backslash found (likely invalid input)
}
static bool wel_add_to_registry(const wchar_t *channel, const wchar_t *provider, DWORD defaultMaxSize) {
// Build the registry path: SYSTEM\CurrentControlSet\Services\EventLog\<LogName>\<SourceName>
wchar_t key[MAX_PATH];
if(!provider)
swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls", channel);
else
swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls\\%ls", channel, provider);
HKEY hRegKey;
DWORD disposition;
LONG result = RegCreateKeyExW(HKEY_LOCAL_MACHINE, key,
0, NULL, REG_OPTION_NON_VOLATILE, KEY_SET_VALUE, NULL, &hRegKey, &disposition);
if (result != ERROR_SUCCESS)
return false; // Could not create the registry key
// Check if MaxSize is already set
DWORD maxSize = 0;
DWORD size = sizeof(maxSize);
if (RegQueryValueExW(hRegKey, L"MaxSize", NULL, NULL, (LPBYTE)&maxSize, &size) != ERROR_SUCCESS) {
// MaxSize is not set, set it to the default value
RegSetValueExW(hRegKey, L"MaxSize", 0, REG_DWORD, (const BYTE*)&defaultMaxSize, sizeof(defaultMaxSize));
}
wchar_t modulePath[MAX_PATH];
if (GetModuleFileNameW(NULL, modulePath, MAX_PATH) == 0) {
RegCloseKey(hRegKey);
return false;
}
if(wel_replace_program_with_wevt_netdata_dll(modulePath, _countof(modulePath))) {
RegSetValueExW(hRegKey, L"EventMessageFile", 0, REG_EXPAND_SZ,
(LPBYTE)modulePath, (wcslen(modulePath) + 1) * sizeof(wchar_t));
DWORD types_supported = EVENTLOG_SUCCESS | EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE;
RegSetValueExW(hRegKey, L"TypesSupported", 0, REG_DWORD, (LPBYTE)&types_supported, sizeof(DWORD));
}
RegCloseKey(hRegKey);
return true;
}
#if defined(HAVE_ETW)
static void etw_set_source_meta(struct nd_log_source *source, USHORT channelID, const EVENT_DESCRIPTOR *ed) {
// It turns out that the keyword varies per only per channel!
// so, to log with the right keyword, Task, Opcode we copy the ids from the header
// the messages compiler (mc.exe) generated from the manifest.
source->channelID = channelID;
source->Opcode = ed->Opcode;
source->Task = ed->Task;
source->Keyword = ed->Keyword;
}
static bool etw_register_provider(void) {
// Register the ETW provider
if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, NULL, NULL, &regHandle) != ERROR_SUCCESS)
return false;
etw_set_source_meta(&nd_log.sources[NDLS_DAEMON], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_COLLECTORS], CHANNEL_COLLECTORS, &ED_COLLECTORS_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_ACCESS], CHANNEL_ACCESS, &ED_ACCESS_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_HEALTH], CHANNEL_HEALTH, &ED_HEALTH_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_ACLK], CHANNEL_ACLK, &ED_ACLK_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_UNSET], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
etw_set_source_meta(&nd_log.sources[NDLS_DEBUG], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);
return true;
}
#endif
bool nd_log_init_windows(void) {
if(nd_log.eventlog.initialized)
return true;
// validate we have the right keys
if(
!check_event_id(NDLS_COLLECTORS, NDLP_INFO, MSGID_MESSAGE_ONLY, MC_COLLECTORS_INFO_MESSAGE_ONLY) ||
!check_event_id(NDLS_DAEMON, NDLP_ERR, MSGID_MESSAGE_ONLY, MC_DAEMON_ERR_MESSAGE_ONLY) ||
!check_event_id(NDLS_ACCESS, NDLP_WARNING, MSGID_ACCESS_USER, MC_ACCESS_WARN_ACCESS_USER) ||
!check_event_id(NDLS_HEALTH, NDLP_CRIT, MSGID_ALERT_TRANSITION, MC_HEALTH_CRIT_ALERT_TRANSITION) ||
!check_event_id(NDLS_DEBUG, NDLP_ALERT, MSGID_ACCESS_FORWARDER_USER, MC_DEBUG_ALERT_ACCESS_FORWARDER_USER))
return false;
#if defined(HAVE_ETW)
if(nd_log.eventlog.etw && !etw_register_provider())
return false;
#endif
// if(!nd_log.eventlog.etw && !wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, NULL, 50 * 1024 * 1024))
// return false;
// Loop through each source and add it to the registry
for(size_t i = 0; i < _NDLS_MAX; i++) {
nd_log.sources[i].source = i;
const wchar_t *sub_channel = wel_provider_per_source[i];
if(!sub_channel)
// we will map these to NDLS_DAEMON
continue;
DWORD defaultMaxSize = 0;
switch (i) {
case NDLS_ACLK:
defaultMaxSize = 5 * 1024 * 1024;
break;
case NDLS_HEALTH:
defaultMaxSize = 35 * 1024 * 1024;
break;
default:
case NDLS_ACCESS:
case NDLS_COLLECTORS:
case NDLS_DAEMON:
defaultMaxSize = 20 * 1024 * 1024;
break;
}
if(!nd_log.eventlog.etw) {
if(!wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, sub_channel, defaultMaxSize))
return false;
// when not using a manifest, each source is a provider
nd_log.sources[i].hEventLog = RegisterEventSourceW(NULL, sub_channel);
if (!nd_log.sources[i].hEventLog)
return false;
}
}
if(!nd_log.eventlog.etw) {
// Map the unset ones to NDLS_DAEMON
for (size_t i = 0; i < _NDLS_MAX; i++) {
if (!nd_log.sources[i].hEventLog)
nd_log.sources[i].hEventLog = nd_log.sources[NDLS_DAEMON].hEventLog;
}
}
nd_log.eventlog.initialized = true;
return true;
}
bool nd_log_init_etw(void) {
nd_log.eventlog.etw = true;
return nd_log_init_windows();
}
bool nd_log_init_wel(void) {
nd_log.eventlog.etw = false;
return nd_log_init_windows();
}
// --------------------------------------------------------------------------------------------------------------------
// we pass all our fields to the windows events logs
// numbered the same way we have them in memory.
//
// to avoid runtime memory allocations, we use a static allocations with ready to use buffers
// which are immediately available for logging.
#define SMALL_WIDE_BUFFERS_SIZE 256
#define MEDIUM_WIDE_BUFFERS_SIZE 2048
#define BIG_WIDE_BUFFERS_SIZE 16384
static wchar_t small_wide_buffers[_NDF_MAX][SMALL_WIDE_BUFFERS_SIZE];
static wchar_t medium_wide_buffers[2][MEDIUM_WIDE_BUFFERS_SIZE];
static wchar_t big_wide_buffers[2][BIG_WIDE_BUFFERS_SIZE];
static struct {
size_t size;
wchar_t *buf;
} fields_buffers[_NDF_MAX] = { 0 };
#if defined(HAVE_ETW)
static EVENT_DATA_DESCRIPTOR etw_eventData[_NDF_MAX - 1];
#endif
static LPCWSTR wel_messages[_NDF_MAX - 1];
__attribute__((constructor)) void wevents_initialize_buffers(void) {
for(size_t i = 0; i < _NDF_MAX ;i++) {
fields_buffers[i].buf = small_wide_buffers[i];
fields_buffers[i].size = SMALL_WIDE_BUFFERS_SIZE;
}
fields_buffers[NDF_NIDL_INSTANCE].buf = medium_wide_buffers[0];
fields_buffers[NDF_NIDL_INSTANCE].size = MEDIUM_WIDE_BUFFERS_SIZE;
fields_buffers[NDF_REQUEST].buf = big_wide_buffers[0];
fields_buffers[NDF_REQUEST].size = BIG_WIDE_BUFFERS_SIZE;
fields_buffers[NDF_MESSAGE].buf = big_wide_buffers[1];
fields_buffers[NDF_MESSAGE].size = BIG_WIDE_BUFFERS_SIZE;
for(size_t i = 1; i < _NDF_MAX ;i++)
wel_messages[i - 1] = fields_buffers[i].buf;
}
// --------------------------------------------------------------------------------------------------------------------
#define is_field_set(fields, fields_max, field) ((field) < (fields_max) && (fields)[field].entry.set)
static const char *get_field_value_unsafe(struct log_field *fields, ND_LOG_FIELD_ID i, size_t fields_max, BUFFER **tmp) {
if(!is_field_set(fields, fields_max, i) || !fields[i].eventlog)
return "";
static char number_str[MAX(MAX(UINT64_MAX_LENGTH, DOUBLE_MAX_LENGTH), UUID_STR_LEN)];
const char *s = NULL;
if (fields[i].annotator)
s = fields[i].annotator(&fields[i]);
else
switch (fields[i].entry.type) {
case NDFT_TXT:
s = fields[i].entry.txt;
break;
case NDFT_STR:
s = string2str(fields[i].entry.str);
break;
case NDFT_BFR:
s = buffer_tostring(fields[i].entry.bfr);
break;
case NDFT_U64:
print_uint64(number_str, fields[i].entry.u64);
s = number_str;
break;
case NDFT_I64:
print_int64(number_str, fields[i].entry.i64);
s = number_str;
break;
case NDFT_DBL:
print_netdata_double(number_str, fields[i].entry.dbl);
s = number_str;
break;
case NDFT_UUID:
if (!uuid_is_null(*fields[i].entry.uuid)) {
uuid_unparse_lower(*fields[i].entry.uuid, number_str);
s = number_str;
}
break;
case NDFT_CALLBACK:
if (!*tmp)
*tmp = buffer_create(1024, NULL);
else
buffer_flush(*tmp);
if (fields[i].entry.cb.formatter(*tmp, fields[i].entry.cb.formatter_data))
s = buffer_tostring(*tmp);
else
s = NULL;
break;
default:
s = "UNHANDLED";
break;
}
if(!s || !*s) return "";
return s;
}
static void etw_replace_percent_with_unicode(wchar_t *s, size_t size) {
size_t original_len = wcslen(s);
// Traverse the string, replacing '%' with the Unicode fullwidth percent sign
for (size_t i = 0; i < original_len && i < size - 1; i++) {
if (s[i] == L'%' && iswdigit(s[i + 1])) {
// s[i] = 0xFF05; // Replace '%' with fullwidth percent sign '％'
// s[i] = 0x29BC; // ⦼
s[i] = 0x2105; // ℅
}
}
// Ensure null termination if needed
s[size - 1] = L'\0';
}
static void wevt_generate_all_fields_unsafe(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
for (size_t i = 0; i < fields_max; i++) {
fields_buffers[i].buf[0] = L'\0';
if (!fields[i].entry.set || !fields[i].eventlog)
continue;
const char *s = get_field_value_unsafe(fields, i, fields_max, tmp);
if (s && *s) {
utf8_to_utf16(fields_buffers[i].buf, (int) fields_buffers[i].size, s, -1);
if(nd_log.eventlog.etw)
// UNBELIEVABLE! they do recursive parameter expansion in ETW...
etw_replace_percent_with_unicode(fields_buffers[i].buf, fields_buffers[i].size);
}
}
}
static bool has_user_role_permissions(struct log_field *fields, size_t fields_max, BUFFER **tmp) {
const char *t;
t = get_field_value_unsafe(fields, NDF_USER_NAME, fields_max, tmp);
if (*t) return true;
t = get_field_value_unsafe(fields, NDF_USER_ROLE, fields_max, tmp);
if (*t && strcmp(t, "none") != 0) return true;
t = get_field_value_unsafe(fields, NDF_USER_ACCESS, fields_max, tmp);
if (*t && strcmp(t, "0x0") != 0) return true;
return false;
}
static bool nd_logger_windows(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
if (!nd_log.eventlog.initialized)
return false;
ND_LOG_FIELD_PRIORITY priority = NDLP_INFO;
if (fields[NDF_PRIORITY].entry.set)
priority = (ND_LOG_FIELD_PRIORITY) fields[NDF_PRIORITY].entry.u64;
DWORD wType = get_event_type_from_priority(priority);
(void) wType;
CLEAN_BUFFER *tmp = NULL;
static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
spinlock_lock(&spinlock);
wevt_generate_all_fields_unsafe(fields, fields_max, &tmp);
MESSAGE_ID messageID;
switch (source->source) {
default:
case NDLS_DEBUG:
case NDLS_DAEMON:
case NDLS_COLLECTORS:
messageID = MSGID_MESSAGE_ONLY;
break;
case NDLS_HEALTH:
messageID = MSGID_ALERT_TRANSITION;
break;
case NDLS_ACCESS:
if (is_field_set(fields, fields_max, NDF_MESSAGE)) {
messageID = MSGID_ACCESS_MESSAGE;
if (has_user_role_permissions(fields, fields_max, &tmp))
messageID = MSGID_ACCESS_MESSAGE_USER;
else if (*get_field_value_unsafe(fields, NDF_REQUEST, fields_max, &tmp))
messageID = MSGID_ACCESS_MESSAGE_REQUEST;
} else if (is_field_set(fields, fields_max, NDF_RESPONSE_CODE)) {
messageID = MSGID_ACCESS;
if (*get_field_value_unsafe(fields, NDF_SRC_FORWARDED_FOR, fields_max, &tmp))
messageID = MSGID_ACCESS_FORWARDER;
if (has_user_role_permissions(fields, fields_max, &tmp)) {
if (messageID == MSGID_ACCESS)
messageID = MSGID_ACCESS_USER;
else
messageID = MSGID_ACCESS_FORWARDER_USER;
}
} else
messageID = MSGID_REQUEST_ONLY;
break;
case NDLS_ACLK:
messageID = MSGID_MESSAGE_ONLY;
break;
}
if (messageID == MSGID_MESSAGE_ONLY && (
*get_field_value_unsafe(fields, NDF_ERRNO, fields_max, &tmp) ||
*get_field_value_unsafe(fields, NDF_WINERROR, fields_max, &tmp))) {
messageID = MSGID_MESSAGE_ERRNO;
}
DWORD eventID = construct_event_id(source->source, priority, messageID);
// wType
//
// without a manifest => this determines the Level of the event
// with a manifest => Level from the manifest is used (wType ignored)
// [however it is good to have, in case the manifest is not accessible somehow]
//
// wCategory
//
// without a manifest => numeric Task values appear
// with a manifest => Task from the manifest is used (wCategory ignored)
BOOL rc;
#if defined(HAVE_ETW)
if (nd_log.eventlog.etw) {
// metadata based logging - ETW
for (size_t i = 1; i < _NDF_MAX; i++)
EventDataDescCreate(&etw_eventData[i - 1], fields_buffers[i].buf,
(wcslen(fields_buffers[i].buf) + 1) * sizeof(WCHAR));
EVENT_DESCRIPTOR EventDesc = {
.Id = eventID & EVENT_ID_CODE_MASK, // ETW needs the raw event id
.Version = 0,
.Channel = source->channelID,
.Level = get_level_from_priority(priority),
.Opcode = source->Opcode,
.Task = source->Task,
.Keyword = source->Keyword,
};
rc = ERROR_SUCCESS == EventWrite(regHandle, &EventDesc, _NDF_MAX - 1, etw_eventData);
}
else
#endif
{
// eventID based logging - WEL
rc = ReportEventW(source->hEventLog, wType, 0, eventID, NULL, _NDF_MAX - 1, 0, wel_messages, NULL);
}
spinlock_unlock(&spinlock);
return rc == TRUE;
}
#if defined(HAVE_ETW)
bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
return nd_logger_windows(source, fields, fields_max);
}
#endif
#if defined(HAVE_WEL)
bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max) {
return nd_logger_windows(source, fields, fields_max);
}
#endif
#endif

465
src/libnetdata/log/nd_log.c Normal file
View file

@ -0,0 +1,465 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the
// source code that makes the calls, allowing our loggers to log the lines of source code that actually log
#define SD_JOURNAL_SUPPRESS_LOCATION
#include "../libnetdata.h"
#include "nd_log-internals.h"
const char *program_name = "";
uint64_t debug_flags = 0;
int aclklog_enabled = 0;
// --------------------------------------------------------------------------------------------------------------------
void errno_clear(void) {
errno = 0;
#if defined(OS_WINDOWS)
SetLastError(ERROR_SUCCESS);
#endif
}
// --------------------------------------------------------------------------------------------------------------------
// logger router
static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
*spinlock = NULL;
ND_LOG_METHOD output = nd_log.sources[source].method;
switch(output) {
case NDLM_JOURNAL:
if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
output = NDLM_FILE;
*fpp = stderr;
*spinlock = &nd_log.std_error.spinlock;
}
else {
*fpp = NULL;
*spinlock = NULL;
}
break;
#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL))
#if defined(HAVE_ETW)
case NDLM_ETW:
#endif
#if defined(HAVE_WEL)
case NDLM_WEL:
#endif
if(unlikely(!nd_log.eventlog.initialized)) {
output = NDLM_FILE;
*fpp = stderr;
*spinlock = &nd_log.std_error.spinlock;
}
else {
*fpp = NULL;
*spinlock = NULL;
}
break;
#endif
case NDLM_SYSLOG:
if(unlikely(!nd_log.syslog.initialized)) {
output = NDLM_FILE;
*spinlock = &nd_log.std_error.spinlock;
*fpp = stderr;
}
else {
*spinlock = NULL;
*fpp = NULL;
}
break;
case NDLM_FILE:
if(!nd_log.sources[source].fp) {
*fpp = stderr;
*spinlock = &nd_log.std_error.spinlock;
}
else {
*fpp = nd_log.sources[source].fp;
*spinlock = &nd_log.sources[source].spinlock;
}
break;
case NDLM_STDOUT:
output = NDLM_FILE;
*fpp = stdout;
*spinlock = &nd_log.std_output.spinlock;
break;
default:
case NDLM_DEFAULT:
case NDLM_STDERR:
output = NDLM_FILE;
*fpp = stderr;
*spinlock = &nd_log.std_error.spinlock;
break;
case NDLM_DISABLED:
case NDLM_DEVNULL:
output = NDLM_DISABLED;
*fpp = NULL;
*spinlock = NULL;
break;
}
return output;
}
// --------------------------------------------------------------------------------------------------------------------
// high level logger
static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
ND_LOG_METHOD output, struct nd_log_source *source,
struct log_field *fields, size_t fields_max) {
if(spinlock)
spinlock_lock(spinlock);
// check the limits
if(limit && nd_log_limit_reached(source))
goto cleanup;
if(output == NDLM_JOURNAL) {
if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
// we can't log to journal, let's log to stderr
if(spinlock)
spinlock_unlock(spinlock);
output = NDLM_FILE;
spinlock = &nd_log.std_error.spinlock;
fp = stderr;
if(spinlock)
spinlock_lock(spinlock);
}
}
#if defined(OS_WINDOWS)
#if defined(HAVE_ETW)
if(output == NDLM_ETW) {
if(!nd_logger_etw(source, fields, fields_max)) {
// we can't log to windows events, let's log to stderr
if(spinlock)
spinlock_unlock(spinlock);
output = NDLM_FILE;
spinlock = &nd_log.std_error.spinlock;
fp = stderr;
if(spinlock)
spinlock_lock(spinlock);
}
}
#endif
#if defined(HAVE_WEL)
if(output == NDLM_WEL) {
if(!nd_logger_wel(source, fields, fields_max)) {
// we can't log to windows events, let's log to stderr
if(spinlock)
spinlock_unlock(spinlock);
output = NDLM_FILE;
spinlock = &nd_log.std_error.spinlock;
fp = stderr;
if(spinlock)
spinlock_lock(spinlock);
}
}
#endif
#endif
if(output == NDLM_SYSLOG)
nd_logger_syslog(priority, source->format, fields, fields_max);
if(output == NDLM_FILE)
nd_logger_file(fp, source->format, fields, fields_max);
cleanup:
if(spinlock)
spinlock_unlock(spinlock);
}
static void nd_logger_unset_all_thread_fields(void) {
size_t fields_max = THREAD_FIELDS_MAX;
for(size_t i = 0; i < fields_max ; i++)
thread_log_fields[i].entry.set = false;
}
static void nd_logger_merge_log_stack_to_thread_fields(void) {
for(size_t c = 0; c < thread_log_stack_next ;c++) {
struct log_stack_entry *lgs = thread_log_stack_base[c];
for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
continue;
struct log_stack_entry *e = &lgs[i];
ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
// do not add empty / unset fields
if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
(type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
(type == NDFT_STR && !e->str) ||
(type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
(type == NDFT_CALLBACK && !e->cb.formatter) ||
type == NDFT_UNSET)
continue;
thread_log_fields[lgs[i].id].entry = *e;
}
}
}
static void nd_logger(const char *file, const char *function, const unsigned long line,
ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit,
int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) {
SPINLOCK *spinlock;
FILE *fp;
ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
if(!IS_FINAL_LOG_METHOD(output))
return;
// mark all fields as unset
nd_logger_unset_all_thread_fields();
// flatten the log stack into the fields
nd_logger_merge_log_stack_to_thread_fields();
// set the common fields that are automatically set by the logging subsystem
if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
else {
ND_LOG_SOURCES src = source;
if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
if(src != source && src < _NDLS_MAX) {
source = src;
output = nd_logger_select_output(source, &fp, &spinlock);
if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
return;
}
}
if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
}
if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
}
if(likely(!thread_log_fields[NDF_TID].entry.set))
thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached());
if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
const char *thread_tag = nd_thread_tag();
thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
// TODO: fix the ND_MODULE in logging by setting proper module name in threads
// if(!thread_log_fields[NDF_MODULE].entry.set)
// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
}
if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set)
thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror);
CLEAN_BUFFER *wb = NULL;
if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
wb = buffer_create(1024, NULL);
buffer_vsprintf(wb, fmt, ap);
thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
}
nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
thread_log_fields, THREAD_FIELDS_MAX);
if(nd_log.sources[source].pending_msg) {
// log a pending message
nd_logger_unset_all_thread_fields();
thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
.set = true,
.type = NDFT_U64,
.u64 = now_realtime_usec(),
};
thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
.set = true,
.type = NDFT_TXT,
.txt = nd_log_id2source(source),
};
thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
.set = true,
.type = NDFT_TXT,
.txt = program_name,
};
thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
.set = true,
.type = NDFT_TXT,
.txt = nd_log.sources[source].pending_msg,
};
nd_logger_log_fields(spinlock, fp, false, priority, output,
&nd_log.sources[source],
thread_log_fields, THREAD_FIELDS_MAX);
freez((void *)nd_log.sources[source].pending_msg);
nd_log.sources[source].pending_msg = NULL;
}
errno_clear();
}
static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
if(source >= _NDLS_MAX)
source = NDLS_DAEMON;
if(nd_log.overwrite_process_source)
source = nd_log.overwrite_process_source;
return source;
}
// --------------------------------------------------------------------------------------------------------------------
// public API for loggers
void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... )
{
int saved_errno = errno;
size_t saved_winerror = 0;
#if defined(OS_WINDOWS)
saved_winerror = GetLastError();
#endif
source = nd_log_validate_source(source);
if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
return;
va_list args;
va_start(args, fmt);
nd_logger(file, function, line, source, priority,
source == NDLS_DAEMON || source == NDLS_COLLECTORS,
saved_errno, saved_winerror, fmt, args);
va_end(args);
}
void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
int saved_errno = errno;
size_t saved_winerror = 0;
#if defined(OS_WINDOWS)
saved_winerror = GetLastError();
#endif
source = nd_log_validate_source(source);
if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
return;
if(erl->sleep_ut)
sleep_usec(erl->sleep_ut);
spinlock_lock(&erl->spinlock);
erl->count++;
time_t now = now_boottime_sec();
if(now - erl->last_logged < erl->log_every) {
spinlock_unlock(&erl->spinlock);
return;
}
spinlock_unlock(&erl->spinlock);
va_list args;
va_start(args, fmt);
nd_logger(file, function, line, source, priority,
source == NDLS_DAEMON || source == NDLS_COLLECTORS,
saved_errno, saved_winerror, fmt, args);
va_end(args);
erl->last_logged = now;
erl->count = 0;
}
void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
int saved_errno = errno;
size_t saved_winerror = 0;
#if defined(OS_WINDOWS)
saved_winerror = GetLastError();
#endif
ND_LOG_SOURCES source = NDLS_DAEMON;
source = nd_log_validate_source(source);
va_list args;
va_start(args, fmt);
nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args);
va_end(args);
char date[LOG_DATE_LENGTH];
log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
char action_data[70+1];
snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
const char *thread_tag = nd_thread_tag();
const char *tag_to_send = thread_tag;
// anonymize thread names
if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
tag_to_send = THREAD_TAG_STREAM_RECEIVER;
if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
tag_to_send = THREAD_TAG_STREAM_SENDER;
char action_result[60+1];
snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)
int fd = nd_log.sources[NDLS_DAEMON].fd;
if(fd == -1)
fd = STDERR_FILENO;
int nptrs;
void *buffer[10000];
nptrs = backtrace(buffer, sizeof(buffer));
if(nptrs)
backtrace_symbols_fd(buffer, nptrs, fd);
#endif
#ifdef NETDATA_INTERNAL_CHECKS
abort();
#endif
netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);
}

141
src/libnetdata/log/log.h → src/libnetdata/log/nd_log.h
View file

@ -1,149 +1,18 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_LOG_H
#define NETDATA_LOG_H 1
#ifndef NETDATA_ND_LOG_H
#define NETDATA_ND_LOG_H 1
# ifdef __cplusplus
extern "C" {
# endif
#include "../libnetdata.h"
#include "nd_log-common.h"
#define ND_LOG_DEFAULT_THROTTLE_LOGS 1000
#define ND_LOG_DEFAULT_THROTTLE_PERIOD 60
typedef enum __attribute__((__packed__)) {
NDLS_UNSET = 0, // internal use only
NDLS_ACCESS, // access.log
NDLS_ACLK, // aclk.log
NDLS_COLLECTORS, // collector.log
NDLS_DAEMON, // error.log
NDLS_HEALTH, // health.log
NDLS_DEBUG, // debug.log
// terminator
_NDLS_MAX,
} ND_LOG_SOURCES;
typedef enum __attribute__((__packed__)) {
NDLP_EMERG = LOG_EMERG,
NDLP_ALERT = LOG_ALERT,
NDLP_CRIT = LOG_CRIT,
NDLP_ERR = LOG_ERR,
NDLP_WARNING = LOG_WARNING,
NDLP_NOTICE = LOG_NOTICE,
NDLP_INFO = LOG_INFO,
NDLP_DEBUG = LOG_DEBUG,
} ND_LOG_FIELD_PRIORITY;
typedef enum __attribute__((__packed__)) {
// KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
// so that it easy to audit for missing fields
NDF_STOP = 0,
NDF_TIMESTAMP_REALTIME_USEC, // the timestamp of the log message - added automatically
NDF_SYSLOG_IDENTIFIER, // the syslog identifier of the application - added automatically
NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call
NDF_PRIORITY, // the syslog priority (severity) - set at the log call
NDF_ERRNO, // the ERRNO at the time of the log call - added automatically
#if defined(OS_WINDOWS)
NDF_WINERROR, // Windows GetLastError()
#endif
NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically
NDF_LINE, // the source code file line number - added automatically
NDF_FILE, // the source code filename - added automatically
NDF_FUNC, // the source code function - added automatically
NDF_TID, // the thread ID of the thread logging - added automatically
NDF_THREAD_TAG, // the thread tag of the thread logging - added automatically
NDF_MESSAGE_ID, // for specific events
NDF_MODULE, // for internal plugin module, all other get the NDF_THREAD_TAG
NDF_NIDL_NODE, // the node / rrdhost currently being worked
NDF_NIDL_INSTANCE, // the instance / rrdset currently being worked
NDF_NIDL_CONTEXT, // the context of the instance currently being worked
NDF_NIDL_DIMENSION, // the dimension / rrddim currently being worked
// web server, aclk and stream receiver
NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd
// Netdata Cloud Related
NDF_ACCOUNT_ID,
NDF_USER_NAME,
NDF_USER_ROLE,
NDF_USER_ACCESS,
// web server and stream receiver
NDF_SRC_IP, // the streaming / web server source IP
NDF_SRC_PORT, // the streaming / web server source Port
NDF_SRC_FORWARDED_HOST,
NDF_SRC_FORWARDED_FOR,
NDF_SRC_CAPABILITIES, // the stream receiver capabilities
// stream sender (established links)
NDF_DST_TRANSPORT, // the transport we send the request, one of: http, https
NDF_DST_IP, // the destination streaming IP
NDF_DST_PORT, // the destination streaming Port
NDF_DST_CAPABILITIES, // the destination streaming capabilities
// web server, aclk and stream receiver
NDF_REQUEST_METHOD, // for http like requests, the http request method
NDF_RESPONSE_CODE, // for http like requests, the http response code, otherwise a status string
// web server (all), aclk (queries)
NDF_CONNECTION_ID, // the web server connection ID
NDF_TRANSACTION_ID, // the web server and API transaction ID
NDF_RESPONSE_SENT_BYTES, // for http like requests, the response bytes
NDF_RESPONSE_SIZE_BYTES, // for http like requests, the uncompressed response size
NDF_RESPONSE_PREPARATION_TIME_USEC, // for http like requests, the preparation time
NDF_RESPONSE_SENT_TIME_USEC, // for http like requests, the time to send the response back
NDF_RESPONSE_TOTAL_TIME_USEC, // for http like requests, the total time to complete the response
// health alerts
NDF_ALERT_ID,
NDF_ALERT_UNIQUE_ID,
NDF_ALERT_EVENT_ID,
NDF_ALERT_TRANSITION_ID,
NDF_ALERT_CONFIG_HASH,
NDF_ALERT_NAME,
NDF_ALERT_CLASS,
NDF_ALERT_COMPONENT,
NDF_ALERT_TYPE,
NDF_ALERT_EXEC,
NDF_ALERT_RECIPIENT,
NDF_ALERT_DURATION,
NDF_ALERT_VALUE,
NDF_ALERT_VALUE_OLD,
NDF_ALERT_STATUS,
NDF_ALERT_STATUS_OLD,
NDF_ALERT_SOURCE,
NDF_ALERT_UNITS,
NDF_ALERT_SUMMARY,
NDF_ALERT_INFO,
NDF_ALERT_NOTIFICATION_REALTIME_USEC,
// NDF_ALERT_FLAGS,
// put new items here
// leave the request URL and the message last
NDF_REQUEST, // the request we are currently working on
NDF_MESSAGE, // the log message, if any
// terminator
_NDF_MAX,
} ND_LOG_FIELD_ID;
typedef enum __attribute__((__packed__)) {
NDFT_UNSET = 0,
NDFT_TXT,
NDFT_STR,
NDFT_BFR,
NDFT_U64,
NDFT_I64,
NDFT_DBL,
NDFT_UUID,
NDFT_CALLBACK,
} ND_LOG_STACK_FIELD_TYPE;
void errno_clear(void);
void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);
void nd_log_set_facility(const char *facility);
@ -156,7 +25,7 @@ void nd_log_set_flood_protection(size_t logs, time_t period);
void nd_log_initialize_for_external_plugins(const char *name);
void nd_log_reopen_log_files_for_spawn_server(void);
bool nd_log_journal_socket_available(void);
ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len);
ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len);
int nd_log_priority2id(const char *priority);
const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
const char *nd_log_method_for_external_plugins(const char *s);
@ -306,4 +175,4 @@ void netdata_logger_fatal( const char *file, const char *function, unsigned long
}
# endif
#endif /* NETDATA_LOG_H */
#endif /* NETDATA_ND_LOG_H */

100
src/libnetdata/log/nd_log_limit.c Normal file
View file

@ -0,0 +1,100 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "nd_log_limit.h"
void nd_log_limits_reset(void) {
usec_t now_ut = now_monotonic_usec();
spinlock_lock(&nd_log.std_output.spinlock);
spinlock_lock(&nd_log.std_error.spinlock);
for(size_t i = 0; i < _NDLS_MAX ;i++) {
spinlock_lock(&nd_log.sources[i].spinlock);
nd_log.sources[i].limits.prevented = 0;
nd_log.sources[i].limits.counter = 0;
nd_log.sources[i].limits.started_monotonic_ut = now_ut;
nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
spinlock_unlock(&nd_log.sources[i].spinlock);
}
spinlock_unlock(&nd_log.std_output.spinlock);
spinlock_unlock(&nd_log.std_error.spinlock);
}
void nd_log_limits_unlimited(void) {
nd_log_limits_reset();
for(size_t i = 0; i < _NDLS_MAX ;i++) {
nd_log.sources[i].limits.logs_per_period = 0;
}
}
bool nd_log_limit_reached(struct nd_log_source *source) {
if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
return false;
usec_t now_ut = now_monotonic_usec();
if(!source->limits.started_monotonic_ut)
source->limits.started_monotonic_ut = now_ut;
source->limits.counter++;
if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
if(source->limits.prevented) {
BUFFER *wb = buffer_create(1024, NULL);
buffer_sprintf(wb,
"LOG FLOOD PROTECTION: resuming logging "
"(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
source->limits.prevented,
source->limits.throttle_period);
if(source->pending_msg)
freez((void *)source->pending_msg);
source->pending_msg = strdupz(buffer_tostring(wb));
buffer_free(wb);
}
// restart the period accounting
source->limits.started_monotonic_ut = now_ut;
source->limits.counter = 1;
source->limits.prevented = 0;
// log this error
return false;
}
if(source->limits.counter > source->limits.logs_per_period) {
if(!source->limits.prevented) {
BUFFER *wb = buffer_create(1024, NULL);
buffer_sprintf(wb,
"LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
"in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
source->limits.counter,
(int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
source->limits.logs_per_period,
source->limits.throttle_period,
program_name,
(int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC)
);
if(source->pending_msg)
freez((void *)source->pending_msg);
source->pending_msg = strdupz(buffer_tostring(wb));
buffer_free(wb);
}
source->limits.prevented++;
// prevent logging this error
#ifdef NETDATA_INTERNAL_CHECKS
return false;
#else
return true;
#endif
}
return false;
}

26
src/libnetdata/log/nd_log_limit.h Normal file
View file

@ -0,0 +1,26 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_ND_LOG_LIMIT_H
#define NETDATA_ND_LOG_LIMIT_H
#include "../libnetdata.h"
struct nd_log_source;
bool nd_log_limit_reached(struct nd_log_source *source);
struct nd_log_limit {
usec_t started_monotonic_ut;
uint32_t counter;
uint32_t prevented;
uint32_t throttle_period;
uint32_t logs_per_period;
uint32_t logs_per_period_backup;
};
#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
#include "nd_log-internals.h"
#endif //NETDATA_ND_LOG_LIMIT_H

295
src/libnetdata/log/nd_wevents_manifest.xml Normal file
View file

@ -0,0 +1,295 @@
<?xml version="1.0" encoding="UTF-8"?>
<instrumentationManifest
xmlns="http://schemas.microsoft.com/win/2004/08/events"
xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events"
xmlns:xs="http://www.w3.org/2001/XMLSchema">
<instrumentation>
<events>
<provider name="Netdata"
guid="{96c5ca72-9bd8-4634-81e5-000014e7da7a}"
symbol="ND_PROVIDER_NAME"
messageFileName="%SystemRoot%\System32\nd_wevents.dll"
resourceFileName="%SystemRoot%\System32\nd_wevents.dll"
parameterFileName="%SystemRoot%\System32\nd_wevents.dll"
message="$(string.ND_PROVIDER_NAME)">
<!-- Define the channels -->
<channels>
<channel name="Netdata/Daemon"
symbol="ND_CHANNEL_DAEMON"
type="Operational"/>
<channel name="Netdata/Collectors"
symbol="ND_CHANNEL_COLLECTORS"
type="Operational"/>
<channel name="Netdata/Access"
symbol="ND_CHANNEL_ACCESS"
type="Operational"/>
<channel symbol="ND_CHANNEL_HEALTH"
name="Netdata/Alerts"
type="Operational"/>
<channel name="Netdata/ACLK"
symbol="ND_CHANNEL_ACLK"
type="Operational"/>
</channels>
<levels>
</levels>
<opcodes>
</opcodes>
<tasks>
<task name="Daemon" value="1" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Daemon)"/>
<task name="Collector" value="2" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Collector)"/>
<task name="Access" value="3" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Access)"/>
<task name="Health" value="4" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Health)"/>
<task name="Aclk" value="5" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Aclk)"/>
</tasks>
<templates>
<template tid="NetdataLogTemplate">
<!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->
<data name="Timestamp" inType="win:UnicodeString"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->
<data name="Program" inType="win:UnicodeString"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->
<data name="NetdataLogSource" inType="win:UnicodeString"/> <!-- 3 (NDF_LOG_SOURCE) -->
<data name="Level" inType="win:UnicodeString"/> <!-- 4 (NDF_PRIORITY) -->
<data name="UnixErrno" inType="win:UnicodeString"/> <!-- 5 (NDF_ERRNO) -->
<data name="WindowsLastError" inType="win:UnicodeString"/> <!-- 6 (NDF_WINERROR) -->
<data name="InvocationID" inType="win:UnicodeString"/> <!-- 7 (NDF_INVOCATION_ID) -->
<data name="CodeLine" inType="win:UInt32"/> <!-- 8 (NDF_LINE) -->
<data name="CodeFile" inType="win:UnicodeString"/> <!-- 9 (NDF_FILE) -->
<data name="CodeFunction" inType="win:UnicodeString"/> <!-- 10 (NDF_FUNC) -->
<data name="ThreadID" inType="win:UInt32"/> <!-- 11 (NDF_TID) -->
<data name="ThreadName" inType="win:UnicodeString"/> <!-- 12 (NDF_THREAD_TAG) -->
<data name="MessageID" inType="win:UnicodeString"/> <!-- 13 (NDF_MESSAGE_ID) -->
<data name="Module" inType="win:UnicodeString"/> <!-- 14 (NDF_MODULE) -->
<data name="Node" inType="win:UnicodeString"/> <!-- 15 (NDF_NIDL_NODE) -->
<data name="Instance" inType="win:UnicodeString"/> <!-- 16 (NDF_NIDL_INSTANCE) -->
<data name="Context" inType="win:UnicodeString"/> <!-- 17 (NDF_NIDL_CONTEXT) -->
<data name="Dimension" inType="win:UnicodeString"/> <!-- 18 (NDF_NIDL_DIMENSION) -->
<data name="SourceTransport" inType="win:UnicodeString"/> <!-- 19 (NDF_SRC_TRANSPORT) -->
<data name="AccountID" inType="win:UnicodeString"/> <!-- 20 (NDF_ACCOUNT_ID) -->
<data name="UserName" inType="win:UnicodeString"/> <!-- 21 (NDF_USER_NAME) -->
<data name="UserRole" inType="win:UnicodeString"/> <!-- 22 (NDF_USER_ROLE) -->
<data name="UserPermissions" inType="win:UnicodeString"/> <!-- 23 (NDF_USER_ACCESS) -->
<data name="SourceIP" inType="win:UnicodeString"/> <!-- 24 (NDF_SRC_IP) -->
<data name="SourceForwardedHost" inType="win:UnicodeString"/> <!-- 25 (NDF_SRC_PORT) -->
<data name="SourceForwardedFor" inType="win:UnicodeString"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->
<data name="SourcePort" inType="win:UInt32"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->
<data name="SourceCapabilities" inType="win:UnicodeString"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->
<data name="DestinationTransport" inType="win:UnicodeString"/> <!-- 29 (NDF_DST_TRANSPORT) -->
<data name="DestinationIP" inType="win:UnicodeString"/> <!-- 30 (NDF_DST_IP) -->
<data name="DestinationPort" inType="win:UInt32"/> <!-- 31 (NDF_DST_PORT) -->
<data name="DestinationCapabilities" inType="win:UnicodeString"/> <!-- 32 (NDF_DST_CAPABILITIES) -->
<data name="RequestMethod" inType="win:UnicodeString"/> <!-- 33 (NDF_REQUEST_METHOD) -->
<data name="ResponseCode" inType="win:UInt32"/> <!-- 34 (NDF_RESPONSE_CODE) -->
<data name="ConnectionID" inType="win:UnicodeString"/> <!-- 35 (NDF_CONNECTION_ID) -->
<data name="TransactionID" inType="win:UnicodeString"/> <!-- 36 (NDF_TRANSACTION_ID) -->
<data name="ResponseSentBytes" inType="win:UInt64"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->
<data name="ResponseSizeBytes" inType="win:UInt64"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->
<data name="ResponsePreparationTimeUsec" inType="win:UInt64"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->
<data name="ResponseSentTimeUsec" inType="win:UInt64"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->
<data name="ResponseTotalTimeUsec" inType="win:UInt64"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->
<data name="AlertID" inType="win:UnicodeString"/> <!-- 42 (NDF_ALERT_ID) -->
<data name="AlertUniqueID" inType="win:UnicodeString"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->
<data name="AlertTransitionID" inType="win:UnicodeString"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->
<data name="AlertEventID" inType="win:UnicodeString"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->
<data name="AlertConfig" inType="win:UnicodeString"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->
<data name="AlertName" inType="win:UnicodeString"/> <!-- 47 (NDF_ALERT_NAME) -->
<data name="AlertClass" inType="win:UnicodeString"/> <!-- 48 (NDF_ALERT_CLASS) -->
<data name="AlertComponent" inType="win:UnicodeString"/> <!-- 49 (NDF_ALERT_COMPONENT) -->
<data name="AlertType" inType="win:UnicodeString"/> <!-- 50 (NDF_ALERT_TYPE) -->
<data name="AlertExec" inType="win:UnicodeString"/> <!-- 51 (NDF_ALERT_EXEC) -->
<data name="AlertRecipient" inType="win:UnicodeString"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->
<data name="AlertDuration" inType="win:UInt64"/> <!-- 53 (NDF_ALERT_DURATION) -->
<data name="AlertValue" inType="win:Double"/> <!-- 54 (NDF_ALERT_VALUE) -->
<data name="AlertOldValue" inType="win:Double"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->
<data name="AlertStatus" inType="win:UnicodeString"/> <!-- 56 (NDF_ALERT_STATUS) -->
<data name="AlertOldStatus" inType="win:UnicodeString"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->
<data name="Source" inType="win:UnicodeString"/> <!-- 58 (NDF_ALERT_SOURCE) -->
<data name="AlertUnits" inType="win:UnicodeString"/> <!-- 59 (NDF_ALERT_UNITS) -->
<data name="AlertSummary" inType="win:UnicodeString"/> <!-- 60 (NDF_ALERT_SUMMARY) -->
<data name="AlertInfo" inType="win:UnicodeString"/> <!-- 61 (NDF_ALERT_INFO) -->
<data name="AlertNotificationTime" inType="win:UInt64"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->
<data name="Request" inType="win:UnicodeString"/> <!-- 63 (NDF_REQUEST) -->
<data name="Message" inType="win:UnicodeString"/> <!-- 64 (NDF_MESSAGE) -->
</template>
</templates>
<events>
<!-- Daemon Events -->
<event symbol="ND_EVENT_DAEMON_INFO"
value="0x1000"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Daemon"
level="win:Informational"
task="Daemon"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_DAEMON_WARNING"
value="0x1001"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Daemon"
level="win:Warning"
task="Daemon"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_DAEMON_ERROR"
value="0x1002"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Daemon"
level="win:Error"
task="Daemon"
opcode="win:Info"
template="NetdataLogTemplate"/>
<!-- Collector Events -->
<event symbol="ND_EVENT_COLLECTOR_INFO"
value="0x2000"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Collectors"
level="win:Informational"
task="Collector"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_COLLECTOR_WARNING"
value="0x2001"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Collectors"
level="win:Warning"
task="Collector"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_COLLECTOR_ERROR"
value="0x2002"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/Collectors"
level="win:Error"
task="Collector"
opcode="win:Info"
template="NetdataLogTemplate"/>
<!-- Access Events -->
<event symbol="ND_EVENT_ACCESS_INFO"
value="0x3000"
message="$(string.ND_ACCESS_EVENT_MESSAGE)"
channel="Netdata/Access"
level="win:Informational"
task="Access"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_ACCESS_WARNING"
value="0x3001"
message="$(string.ND_ACCESS_EVENT_MESSAGE)"
channel="Netdata/Access"
level="win:Warning"
task="Access"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_ACCESS_ERROR"
value="0x3002"
message="$(string.ND_ACCESS_EVENT_MESSAGE)"
channel="Netdata/Access"
level="win:Error"
task="Access"
opcode="win:Info"
template="NetdataLogTemplate"/>
<!-- Health Events -->
<event symbol="ND_EVENT_HEALTH_INFO"
value="0x4000"
message="$(string.ND_HEALTH_EVENT_MESSAGE)"
channel="Netdata/Alerts"
level="win:Informational"
task="Health"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_HEALTH_WARNING"
value="0x4001"
message="$(string.ND_HEALTH_EVENT_MESSAGE)"
channel="Netdata/Alerts"
level="win:Warning"
task="Health"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_HEALTH_ERROR"
value="0x4002"
message="$(string.ND_HEALTH_EVENT_MESSAGE)"
channel="Netdata/Alerts"
level="win:Error"
task="Health"
opcode="win:Info"
template="NetdataLogTemplate"/>
<!-- ACLK Events -->
<event symbol="ND_EVENT_ACLK_INFO"
value="0x5000"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/ACLK"
level="win:Informational"
task="Aclk"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_ACLK_WARNING"
value="0x5001"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/ACLK"
level="win:Warning"
task="Aclk"
opcode="win:Info"
template="NetdataLogTemplate"/>
<event symbol="ND_EVENT_ACLK_ERROR"
value="0x5002"
message="$(string.ND_GENERIC_LOG_MESSAGE)"
channel="Netdata/ACLK"
level="win:Error"
task="Aclk"
opcode="win:Info"
template="NetdataLogTemplate"/>
</events>
</provider>
</events>
</instrumentation>
<localization>
<resources culture="en-US">
<stringTable>
<string id="Task.Daemon" value="ND Daemon Log"/>
<string id="Task.Collector" value="ND Collector Log"/>
<string id="Task.Access" value="ND Access Log"/>
<string id="Task.Health" value="ND Health Log"/>
<string id="Task.Aclk" value="ND ACLK Log"/>
<string id="ND_PROVIDER_NAME" value="Netdata"/>
<string id="ND_GENERIC_LOG_MESSAGE" value="%64"/>
<string id="ND_ACCESS_EVENT_MESSAGE"
value="Transaction %36, method: %33, path: %63
Source IP : %24, Forwarded-For: %27
User : %21, role: %22, permissions: %23
Timings (usec): prep %39, sent %40, total %41
Response Size : sent %37, uncompressed %38
Response Code : %34
"/>
<string id="ND_HEALTH_EVENT_MESSAGE"
value="Alert '%47' of instance '%16' on node '%15', transitioned from %57 to %56"/>
</stringTable>
</resources>
</localization>
</instrumentationManifest>

2
src/libnetdata/log/systemd-cat-native.c
View file

@ -611,7 +611,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
if(equal) {
const char *field = line->buffer;
size_t field_len = equal - line->buffer;
ND_LOG_FIELD_ID id = nd_log_field_id_by_name(field, field_len);
ND_LOG_FIELD_ID id = nd_log_field_id_by_journal_name(field, field_len);
if(id != NDF_STOP) {
const char *value = ++equal;

2
src/libnetdata/log/journal.c → src/libnetdata/log/systemd-journal-helpers.c
View file

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "journal.h"
#include "systemd-journal-helpers.h"
bool is_path_unix_socket(const char *path) {
// Check if the path is valid

6
src/libnetdata/log/journal.h → src/libnetdata/log/systemd-journal-helpers.h
View file

@ -2,8 +2,8 @@
#include "../libnetdata.h"
#ifndef NETDATA_LOG_JOURNAL_H
#define NETDATA_LOG_JOURNAL_H
#ifndef NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
#define NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H
#define JOURNAL_DIRECT_SOCKET "/run/systemd/journal/socket"
@ -15,4 +15,4 @@ bool journal_direct_send(int fd, const char *msg, size_t msg_len);
bool is_path_unix_socket(const char *path);
bool is_stderr_connected_to_journal(void);
#endif //NETDATA_LOG_JOURNAL_H
#endif // NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H

121
src/libnetdata/log/wevt_netdata_compile.bat Normal file
View file

@ -0,0 +1,121 @@
@echo off
setlocal enabledelayedexpansion
echo PATH=%PATH%
if "%~1"=="" (
echo Error: Missing .mc file path.
goto :usage
)
if "%~2"=="" (
echo Error: Missing destination directory.
goto :usage
)
REM Set variables
set "SRC_DIR=%~1"
set "BIN_DIR=%~2"
set "MC_FILE=%BIN_DIR%\wevt_netdata.mc"
set "MAN_FILE=%BIN_DIR%\wevt_netdata_manifest.xml"
set "BASE_NAME=wevt_netdata"
set "SDK_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
set "VS_PATH=C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
if not exist "%SRC_DIR%" (
echo Error: Source directory does not exist.
exit /b 1
)
if not exist "%BIN_DIR%" (
echo Error: Destination directory does not exist.
exit /b 1
)
if not exist "%MC_FILE%" (
echo Error: %MC_FILE% not found.
exit /b 1
)
if not exist "%MAN_FILE%" (
echo Error: %MAN_FILE% not found.
exit /b 1
)
REM Add SDK paths to PATH
set "PATH=C:\Windows\System32;%SDK_PATH%;%VS_PATH%;%PATH%"
REM Check if commands are available
where mc >nul 2>nul
if %errorlevel% neq 0 (
echo Error: mc.exe not found in PATH.
exit /b 1
)
where rc >nul 2>nul
if %errorlevel% neq 0 (
echo Error: rc.exe not found in PATH.
exit /b 1
)
where link >nul 2>nul
if %errorlevel% neq 0 (
echo Error: link.exe not found in PATH.
exit /b 1
)
where wevtutil >nul 2>nul
if %errorlevel% neq 0 (
echo Error: wevtutil.exe not found in PATH.
exit /b 1
)
REM Change to the destination directory
cd /d "%BIN_DIR%"
echo.
echo Running mc.exe...
mc -v -b -U "%MC_FILE%" "%MAN_FILE%"
if %errorlevel% neq 0 (
echo Error: mc.exe failed on messages.
exit /b 1
)
if not exist "%BASE_NAME%.rc" (
echo Error: %BASE_NAME%.rc not found.
exit /b 1
)
echo.
echo Modifying %BASE_NAME%.rc to include the manifest...
copy "%MAN_FILE%" %BASE_NAME%_manifest.man
echo 1 2004 "%BASE_NAME%_manifest.man" >> %BASE_NAME%.rc
echo.
echo %BASE_NAME%.rc contents:
type %BASE_NAME%.rc
echo.
echo Running rc.exe...
rc /v /fo %BASE_NAME%.res %BASE_NAME%.rc
if %errorlevel% neq 0 (
echo Error: rc.exe failed.
exit /b 1
)
if not exist "%BASE_NAME%.res" (
echo Error: %BASE_NAME%.res not found.
exit /b 1
)
echo.
echo Running link.exe...
link /dll /noentry /machine:x64 /out:%BASE_NAME%.dll %BASE_NAME%.res
if %errorlevel% neq 0 (
echo Error: link.exe failed.
exit /b 1
)
echo.
echo Process completed successfully.
exit /b 0
:usage
echo Usage: %~nx0 [path_to_mc_file] [destination_directory]
exit /b 1

48
src/libnetdata/log/wevt_netdata_compile.sh Normal file
View file

@ -0,0 +1,48 @@
#!/bin/bash
mylocation=$(dirname "${0}")
# Check if both parameters are provided
if [ $# -ne 2 ]; then
echo "Error: Incorrect number of parameters."
echo "Usage: $0 <source_directory> <destination_directory>"
exit 1
fi
# Get the parameters
src_dir="$1"
dest_dir="$2"
# Get the directory of this script
SCRIPT_DIR="$(dirname "$0")"
# Create a temporary batch file
temp_bat=$(mktemp --suffix=.bat)
# Write the contents to the temporary batch file
# Use cygpath directly within the heredoc
cat << EOF > "$temp_bat"
@echo off
set "PATH=%SYSTEMROOT%;$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --sdk -w);$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --visualstudio -w)"
call "$(cygpath -w -a "$SCRIPT_DIR/wevt_netdata_compile.bat")" "$(cygpath -w -a "$src_dir")" "$(cygpath -w -a "$dest_dir")"
EOF
# Execute the temporary batch file
echo
echo "Executing Windows Batch File..."
echo
cat "$temp_bat"
cmd.exe //c "$(cygpath -w -a "$temp_bat")"
exit_status=$?
# Remove the temporary batch file
rm "$temp_bat"
# Check the exit status
if [ $exit_status -eq 0 ]; then
echo "nd_wevents_compile.bat executed successfully."
else
echo "nd_wevents_compile.bat failed with exit status $exit_status."
fi
exit $exit_status

52
src/libnetdata/log/wevt_netdata_install.bat Normal file
View file

@ -0,0 +1,52 @@
@echo off
setlocal enabledelayedexpansion
set "MAN_SRC=%~dp0wevt_netdata_manifest.xml"
set "DLL_SRC=%~dp0wevt_netdata.dll"
set "DLL_DST=%SystemRoot%\System32\wevt_netdata.dll"
where wevtutil >nul 2>nul
if %errorlevel% neq 0 (
echo Error: wevtutil.exe not found in PATH.
exit /b 1
)
echo.
echo Uninstalling previous manifest (if any)...
wevtutil um "%MAN_SRC%"
echo.
echo Copying %DLL_SRC% to %DLL_DST%
copy /y "%DLL_SRC%" "%DLL_DST%"
if %errorlevel% neq 0 (
echo Error: Failed to copy %DLL_SRC% to %DLL_DST%
exit /b 1
)
echo.
echo Granting access to %DLL_DST% for Windows Event Logging...
icacls "%DLL_DST%" /grant "NT SERVICE\EventLog":R
if %errorlevel% neq 0 (
echo Error: Failed to grant access to %DLL_DST%.
exit /b 1
)
echo.
echo Importing the manifest...
wevtutil im "%MAN_SRC%" /rf:"%DLL_DST%" /mf:"%DLL_DST%"
if %errorlevel% neq 0 (
echo Error: Failed to import the manifest.
exit /b 1
)
echo.
echo Verifying Netdata Publisher for Event Tracing for Windows (ETW)...
wevtutil gp "Netdata"
if %errorlevel% neq 0 (
echo Error: Failed to get publisher Netdata.
exit /b 1
)
echo.
echo Netdata Event Tracing for Windows manifest installed successfully.
exit /b 0

518
src/libnetdata/log/wevt_netdata_mc_generate.c Normal file
View file

@ -0,0 +1,518 @@
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <ctype.h>
#include <stdbool.h>
#include <string.h>
// from winnt.h
#define EVENTLOG_SUCCESS 0x0000
#define EVENTLOG_ERROR_TYPE 0x0001
#define EVENTLOG_WARNING_TYPE 0x0002
#define EVENTLOG_INFORMATION_TYPE 0x0004
#define EVENTLOG_AUDIT_SUCCESS 0x0008
#define EVENTLOG_AUDIT_FAILURE 0x0010
// the severities we define in .mc file
#define STATUS_SEVERITY_INFORMATIONAL 0x1
#define STATUS_SEVERITY_WARNING 0x2
#define STATUS_SEVERITY_ERROR 0x3
#define FACILITY_APPLICATION 0x0fff
#include "nd_log-common.h"
#include "nd_log-to-windows-common.h"
const char *get_msg_symbol(MESSAGE_ID msg) {
switch(msg) {
case MSGID_MESSAGE_ONLY:
return "MESSAGE_ONLY";
case MSGID_MESSAGE_ERRNO:
return "MESSAGE_ERRNO";
case MSGID_REQUEST_ONLY:
return "REQUEST_ONLY";
case MSGID_ACCESS_MESSAGE:
return "ACCESS_MESSAGE";
case MSGID_ACCESS_MESSAGE_REQUEST:
return "ACCESS_MESSAGE_REQUEST";
case MSGID_ACCESS_MESSAGE_USER:
return "ACCESS_MESSAGE_USER";
case MSGID_ACCESS:
return "ACCESS";
case MSGID_ACCESS_USER:
return "ACCESS_USER";
case MSGID_ACCESS_FORWARDER:
return "ACCESS_FORWARDER";
case MSGID_ACCESS_FORWARDER_USER:
return "ACCESS_FORWARDER_USER";
case MSGID_ALERT_TRANSITION:
return "ALERT_TRANSITION";
default:
fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
exit(1);
}
}
const char *get_msg_format(MESSAGE_ID msg) {
switch(msg) {
case MSGID_MESSAGE_ONLY:
return "%2(%12): %64\r\n";
case MSGID_MESSAGE_ERRNO:
return "%2(%12): %64%n\r\n"
"%n\r\n"
" Unix Errno : %5%n\r\n"
" Windows Error: %6%n\r\n"
;
case MSGID_REQUEST_ONLY:
return "%2(%12): %63\r\n";
case MSGID_ACCESS_MESSAGE:
return "%64\r\n";
case MSGID_ACCESS_MESSAGE_REQUEST:
return "%64%n\r\n"
"%n\r\n"
" Request: %63%n\r\n"
;
case MSGID_ACCESS_MESSAGE_USER:
return "%64%n\r\n"
"%n\r\n"
" User: %21, role: %22, permissions: %23%n\r\n"
;
case MSGID_ACCESS:
return "%33 %63%n\r\n"
"%n\r\n"
" Response Code : %34%n\r\n"
" Transaction ID: %36%n\r\n"
" Source IP : %24%n\r\n"
;
case MSGID_ACCESS_USER:
return "%33 %63%n\r\n"
"%n\r\n"
" Response Code : %34%n\r\n"
" Transaction ID: %36%n\r\n"
" Source IP : %24%n\r\n"
" User : %21, role: %22, permissions: %23%n\r\n"
;
case MSGID_ACCESS_FORWARDER:
return "%33 %63%n\r\n"
"%n\r\n"
" Response Code : %34%n\r\n"
" Transaction ID: %36%n\r\n"
" Source IP : %24, For %27%n\r\n"
;
case MSGID_ACCESS_FORWARDER_USER:
return "%33 %63%n\r\n"
"%n\r\n"
" Response Code : %34%n\r\n"
" Transaction ID: %36%n\r\n"
" Source IP : %24, For %27%n\r\n"
" User : %21, role: %22, permissions: %23%n\r\n"
;
case MSGID_ALERT_TRANSITION:
return "Alert '%47' of instance '%16' on node '%15' transitioned from %57 to %56\r\n";
default:
fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg);
exit(1);
}
}
int main(int argc, const char **argv) {
(void)argc; (void)argv;
const char *header = NULL, *footer = NULL, *s_header = NULL, *s_footer = NULL;
bool manifest = false;
if(argc == 2 && strcmp(argv[1], "--manifest") == 0) {
manifest = true;
header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n"
"<!--\r\n"
"\r\n"
" THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
"\r\n"
" This XML file can be verified by running mc.exe (the MS tool) with this manifest as param.\r\n"
"\r\n"
" \"c:\\Program Files (x86)\\Windows Kits\\10\\bin\\10.0.26100.0\\x64\\mc.exe\" wevt_netdata_manifest.xml wevt_netdata.mc\r\n"
"\r\n"
" -->\r\n"
"<instrumentationManifest\r\n"
" xmlns=\"http://schemas.microsoft.com/win/2004/08/events\"\r\n"
" xmlns:win=\"http://manifests.microsoft.com/win/2004/08/windows/events\"\r\n"
" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\r\n"
" <instrumentation>\r\n"
" <events>\r\n"
"\r\n"
" <provider name=\"" NETDATA_ETW_PROVIDER_NAME "\"\r\n"
" guid=\"" NETDATA_ETW_PROVIDER_GUID_STR "\"\r\n"
" symbol=\"NETDATA_ETW_PROVIDER_GUID\"\r\n"
" messageFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
" resourceFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n"
" message=\"$(string.ND_PROVIDER_NAME)\">\r\n"
"\r\n"
" <!-- Define the provider sub-channels -->\r\n"
" <channels>\r\n"
" <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON "\"\r\n"
" symbol=\"CHANNEL_DAEMON\"\r\n"
" type=\"Operational\"\r\n"
" message=\"$(string.Channel.Daemon)\"\r\n"
" enabled=\"true\"\r\n"
" />\r\n"
"\r\n"
" <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS "\"\r\n"
" symbol=\"CHANNEL_COLLECTORS\"\r\n"
" type=\"Operational\"\r\n"
" message=\"$(string.Channel.Collectors)\"\r\n"
" enabled=\"true\"\r\n"
" />\r\n"
"\r\n"
" <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS "\"\r\n"
" symbol=\"CHANNEL_ACCESS\"\r\n"
" type=\"Operational\"\r\n"
" message=\"$(string.Channel.Access)\"\r\n"
" enabled=\"true\"\r\n"
" />\r\n"
"\r\n"
" <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH "\"\r\n"
" symbol=\"CHANNEL_HEALTH\"\r\n"
" type=\"Operational\"\r\n"
" message=\"$(string.Channel.Health)\"\r\n"
" enabled=\"true\"\r\n"
" />\r\n"
"\r\n"
" <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK "\"\r\n"
" symbol=\"CHANNEL_ACLK\"\r\n"
" type=\"Operational\"\r\n"
" message=\"$(string.Channel.Aclk)\"\r\n"
" enabled=\"true\"\r\n"
" />\r\n"
" </channels>\r\n"
"\r\n"
" <levels>\r\n"
" </levels>\r\n"
"\r\n"
" <opcodes>\r\n"
" </opcodes>\r\n"
"\r\n"
" <tasks>\r\n"
" </tasks>\r\n"
"\r\n"
" <templates>\r\n"
" <template tid=\"AllFieldsTemplate\">\r\n"
" <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->\r\n"
" <data name=\"Timestamp\" inType=\"win:UnicodeString\"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->\r\n"
" <data name=\"Program\" inType=\"win:UnicodeString\"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->\r\n"
" <data name=\"NetdataLogSource\" inType=\"win:UnicodeString\"/> <!-- 3 (NDF_LOG_SOURCE) -->\r\n"
" <data name=\"Level\" inType=\"win:UnicodeString\"/> <!-- 4 (NDF_PRIORITY) -->\r\n"
" <data name=\"UnixErrno\" inType=\"win:UnicodeString\"/> <!-- 5 (NDF_ERRNO) -->\r\n"
" <data name=\"WindowsLastError\" inType=\"win:UnicodeString\"/> <!-- 6 (NDF_WINERROR) -->\r\n"
" <data name=\"InvocationID\" inType=\"win:UnicodeString\"/> <!-- 7 (NDF_INVOCATION_ID) -->\r\n"
" <data name=\"CodeLine\" inType=\"win:UnicodeString\"/> <!-- 8 (NDF_LINE) -->\r\n"
" <data name=\"CodeFile\" inType=\"win:UnicodeString\"/> <!-- 9 (NDF_FILE) -->\r\n"
" <data name=\"CodeFunction\" inType=\"win:UnicodeString\"/> <!-- 10 (NDF_FUNC) -->\r\n"
" <data name=\"ThreadID\" inType=\"win:UnicodeString\"/> <!-- 11 (NDF_TID) -->\r\n"
" <data name=\"ThreadName\" inType=\"win:UnicodeString\"/> <!-- 12 (NDF_THREAD_TAG) -->\r\n"
" <data name=\"MessageID\" inType=\"win:UnicodeString\"/> <!-- 13 (NDF_MESSAGE_ID) -->\r\n"
" <data name=\"Module\" inType=\"win:UnicodeString\"/> <!-- 14 (NDF_MODULE) -->\r\n"
" <data name=\"Node\" inType=\"win:UnicodeString\"/> <!-- 15 (NDF_NIDL_NODE) -->\r\n"
" <data name=\"Instance\" inType=\"win:UnicodeString\"/> <!-- 16 (NDF_NIDL_INSTANCE) -->\r\n"
" <data name=\"Context\" inType=\"win:UnicodeString\"/> <!-- 17 (NDF_NIDL_CONTEXT) -->\r\n"
" <data name=\"Dimension\" inType=\"win:UnicodeString\"/> <!-- 18 (NDF_NIDL_DIMENSION) -->\r\n"
" <data name=\"SourceTransport\" inType=\"win:UnicodeString\"/> <!-- 19 (NDF_SRC_TRANSPORT) -->\r\n"
" <data name=\"AccountID\" inType=\"win:UnicodeString\"/> <!-- 20 (NDF_ACCOUNT_ID) -->\r\n"
" <data name=\"UserName\" inType=\"win:UnicodeString\"/> <!-- 21 (NDF_USER_NAME) -->\r\n"
" <data name=\"UserRole\" inType=\"win:UnicodeString\"/> <!-- 22 (NDF_USER_ROLE) -->\r\n"
" <data name=\"UserPermissions\" inType=\"win:UnicodeString\"/> <!-- 23 (NDF_USER_ACCESS) -->\r\n"
" <data name=\"SourceIP\" inType=\"win:UnicodeString\"/> <!-- 24 (NDF_SRC_IP) -->\r\n"
" <data name=\"SourceForwardedHost\" inType=\"win:UnicodeString\"/> <!-- 25 (NDF_SRC_PORT) -->\r\n"
" <data name=\"SourceForwardedFor\" inType=\"win:UnicodeString\"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->\r\n"
" <data name=\"SourcePort\" inType=\"win:UnicodeString\"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->\r\n"
" <data name=\"SourceCapabilities\" inType=\"win:UnicodeString\"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->\r\n"
" <data name=\"DestinationTransport\" inType=\"win:UnicodeString\"/> <!-- 29 (NDF_DST_TRANSPORT) -->\r\n"
" <data name=\"DestinationIP\" inType=\"win:UnicodeString\"/> <!-- 30 (NDF_DST_IP) -->\r\n"
" <data name=\"DestinationPort\" inType=\"win:UnicodeString\"/> <!-- 31 (NDF_DST_PORT) -->\r\n"
" <data name=\"DestinationCapabilities\" inType=\"win:UnicodeString\"/> <!-- 32 (NDF_DST_CAPABILITIES) -->\r\n"
" <data name=\"RequestMethod\" inType=\"win:UnicodeString\"/> <!-- 33 (NDF_REQUEST_METHOD) -->\r\n"
" <data name=\"ResponseCode\" inType=\"win:UnicodeString\"/> <!-- 34 (NDF_RESPONSE_CODE) -->\r\n"
" <data name=\"ConnectionID\" inType=\"win:UnicodeString\"/> <!-- 35 (NDF_CONNECTION_ID) -->\r\n"
" <data name=\"TransactionID\" inType=\"win:UnicodeString\"/> <!-- 36 (NDF_TRANSACTION_ID) -->\r\n"
" <data name=\"ResponseSentBytes\" inType=\"win:UnicodeString\"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->\r\n"
" <data name=\"ResponseSizeBytes\" inType=\"win:UnicodeString\"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->\r\n"
" <data name=\"ResponsePreparationTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->\r\n"
" <data name=\"ResponseSentTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->\r\n"
" <data name=\"ResponseTotalTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->\r\n"
" <data name=\"AlertID\" inType=\"win:UnicodeString\"/> <!-- 42 (NDF_ALERT_ID) -->\r\n"
" <data name=\"AlertUniqueID\" inType=\"win:UnicodeString\"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->\r\n"
" <data name=\"AlertTransitionID\" inType=\"win:UnicodeString\"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->\r\n"
" <data name=\"AlertEventID\" inType=\"win:UnicodeString\"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->\r\n"
" <data name=\"AlertConfig\" inType=\"win:UnicodeString\"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->\r\n"
" <data name=\"AlertName\" inType=\"win:UnicodeString\"/> <!-- 47 (NDF_ALERT_NAME) -->\r\n"
" <data name=\"AlertClass\" inType=\"win:UnicodeString\"/> <!-- 48 (NDF_ALERT_CLASS) -->\r\n"
" <data name=\"AlertComponent\" inType=\"win:UnicodeString\"/> <!-- 49 (NDF_ALERT_COMPONENT) -->\r\n"
" <data name=\"AlertType\" inType=\"win:UnicodeString\"/> <!-- 50 (NDF_ALERT_TYPE) -->\r\n"
" <data name=\"AlertExec\" inType=\"win:UnicodeString\"/> <!-- 51 (NDF_ALERT_EXEC) -->\r\n"
" <data name=\"AlertRecipient\" inType=\"win:UnicodeString\"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->\r\n"
" <data name=\"AlertDuration\" inType=\"win:UnicodeString\"/> <!-- 53 (NDF_ALERT_DURATION) -->\r\n"
" <data name=\"AlertValue\" inType=\"win:UnicodeString\"/> <!-- 54 (NDF_ALERT_VALUE) -->\r\n"
" <data name=\"AlertOldValue\" inType=\"win:UnicodeString\"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->\r\n"
" <data name=\"AlertStatus\" inType=\"win:UnicodeString\"/> <!-- 56 (NDF_ALERT_STATUS) -->\r\n"
" <data name=\"AlertOldStatus\" inType=\"win:UnicodeString\"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->\r\n"
" <data name=\"Source\" inType=\"win:UnicodeString\"/> <!-- 58 (NDF_ALERT_SOURCE) -->\r\n"
" <data name=\"AlertUnits\" inType=\"win:UnicodeString\"/> <!-- 59 (NDF_ALERT_UNITS) -->\r\n"
" <data name=\"AlertSummary\" inType=\"win:UnicodeString\"/> <!-- 60 (NDF_ALERT_SUMMARY) -->\r\n"
" <data name=\"AlertInfo\" inType=\"win:UnicodeString\"/> <!-- 61 (NDF_ALERT_INFO) -->\r\n"
" <data name=\"AlertNotificationTime\" inType=\"win:UnicodeString\"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->\r\n"
" <data name=\"Request\" inType=\"win:UnicodeString\"/> <!-- 63 (NDF_REQUEST) -->\r\n"
" <data name=\"Message\" inType=\"win:UnicodeString\"/> <!-- 64 (NDF_MESSAGE) -->\r\n"
" </template>\r\n"
" </templates>\r\n"
"\r\n"
" <events>\r\n"
;
footer = " </events>\r\n"
" </provider>\r\n"
" </events>\r\n"
" </instrumentation>\r\n"
;
s_header = " <localization>\r\n"
" <resources culture=\"en-US\">\r\n"
" <stringTable>\r\n"
" <string id=\"ND_PROVIDER_NAME\" value=\"" NETDATA_ETW_PROVIDER_NAME "\"/>\r\n"
"\r\n"
" <string id=\"Channel.Daemon\" value=\"Daemon\"/>\r\n"
" <string id=\"Channel.Collectors\" value=\"Collectors\"/>\r\n"
" <string id=\"Channel.Access\" value=\"Access\"/>\r\n"
" <string id=\"Channel.Health\" value=\"Health\"/>\r\n"
" <string id=\"Channel.Aclk\" value=\"Aclk\"/>\r\n"
"\r\n"
;
s_footer = " </stringTable>\r\n"
" </resources>\r\n"
" </localization>\r\n"
"</instrumentationManifest>\r\n"
;
}
else {
header = ";// THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n"
"\r\n"
"MessageIdTypedef=DWORD\r\n"
"\r\n"
"SeverityNames=(\r\n"
" Informational=0x1:STATUS_SEVERITY_INFORMATIONAL\r\n"
" Warning=0x2:STATUS_SEVERITY_WARNING\r\n"
" Error=0x3:STATUS_SEVERITY_ERROR\r\n"
" )\r\n"
"\r\n"
"FacilityNames=(\r\n"
" " NETDATA_CHANNEL_NAME "=0x0FFF:FACILITY_NETDATA\r\n"
" )\r\n"
"\r\n"
"LanguageNames=(\r\n"
" English=0x409:MSG00409\r\n"
" )\r\n"
"\r\n"
;
footer = "";
}
bool done[UINT16_MAX] = { 0 };
char symbol[1024];
printf("%s", header);
for(size_t src = 1; src < _NDLS_MAX ;src++) {
for(size_t pri = 0; pri < _NDLP_MAX ;pri++) {
uint8_t severity = get_severity_from_priority(pri);
for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
if(src >= 16) {
fprintf(stderr, "\n\nSource %zu is bigger than 4 bits!\n\n", src);
return 1;
}
if(pri >= 16) {
fprintf(stderr, "\n\nPriority %zu is bigger than 4 bits!\n\n", pri);
return 1;
}
if(msg >= 256) {
fprintf(stderr, "\n\nMessageID %zu is bigger than 8 bits!\n\n", msg);
return 1;
}
uint16_t eventID = construct_event_code(src, pri, msg);
if((eventID & 0xFFFF) != eventID) {
fprintf(stderr, "\n\nEventID 0x%x is bigger than 16 bits!\n\n", eventID);
return 1;
}
if(done[eventID]) continue;
done[eventID] = true;
const char *level = get_level_from_priority_str(pri);
const char *pri_txt;
switch(pri) {
case NDLP_EMERG:
pri_txt = "EMERG";
break;
case NDLP_CRIT:
pri_txt = "CRIT";
break;
case NDLP_ALERT:
pri_txt = "ALERT";
break;
case NDLP_ERR:
pri_txt = "ERR";
break;
case NDLP_WARNING:
pri_txt = "WARN";
break;
case NDLP_INFO:
pri_txt = "INFO";
break;
case NDLP_NOTICE:
pri_txt = "NOTICE";
break;
case NDLP_DEBUG:
pri_txt = "DEBUG";
break;
default:
fprintf(stderr, "\n\nInvalid priority %zu!\n\n\n", pri);
return 1;
}
const char *channel;
const char *src_txt;
switch(src) {
case NDLS_COLLECTORS:
src_txt = "COLLECTORS";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS;
break;
case NDLS_ACCESS:
src_txt = "ACCESS";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS;
break;
case NDLS_HEALTH:
src_txt = "HEALTH";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH;
break;
case NDLS_DEBUG:
src_txt = "DEBUG";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
break;
case NDLS_DAEMON:
src_txt = "DAEMON";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON;
break;
case NDLS_ACLK:
src_txt = "ACLK";
channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK;
break;
default:
fprintf(stderr, "\n\nInvalid source %zu!\n\n\n", src);
return 1;
}
const char *msg_txt = get_msg_symbol(msg);
const char *format = get_msg_format(msg);
const char *severity_txt;
switch (severity) {
case STATUS_SEVERITY_INFORMATIONAL:
severity_txt = "Informational";
break;
case STATUS_SEVERITY_ERROR:
severity_txt = "Error";
break;
case STATUS_SEVERITY_WARNING:
severity_txt = "Warning";
break;
default:
fprintf(stderr, "\n\nInvalid severity id %u!\n\n\n", severity);
return 1;
}
if(manifest)
snprintf(symbol, sizeof(symbol), "ED_%s_%s_%s", src_txt, pri_txt, msg_txt);
else
snprintf(symbol, sizeof(symbol), "MC_%s_%s_%s", src_txt, pri_txt, msg_txt);
if(manifest)
printf(" <event symbol=\"%s\"\r\n"
" value=\"0x%x\"\r\n"
" message=\"$(string.msg.MAN_%s)\"\r\n"
" channel=\"%s\"\r\n"
" level=\"%s\"\r\n"
" task=\"win:None\"\r\n"
" opcode=\"win:Info\"\r\n"
" template=\"AllFieldsTemplate\"/>\r\n\r\n",
symbol, eventID, msg_txt, channel, level);
else
printf("MessageId=0x%x\r\n"
"Severity=%s\r\n"
"Facility=" NETDATA_CHANNEL_NAME "\r\n"
"SymbolicName=%s\r\n"
"Language=English\r\n"
"%s"
".\r\n"
"\r\n",
eventID, severity_txt, symbol, format);
}
}
}
printf("%s", footer);
if(s_header) {
printf("%s", s_header);
for(size_t msg = 1; msg < _MSGID_MAX ;msg++) {
const char *msg_txt = get_msg_symbol(msg);
const char *format = get_msg_format(msg);
printf(" <string id=\"msg.MAN_%s\" value=\"%s\"/>\r\n", msg_txt, format);
}
printf("%s", s_footer);
}
}

26
src/libnetdata/os/windows-perflib/perflib.c
View file

@ -386,44 +386,30 @@ static inline PERF_COUNTER_DEFINITION *getCounterDefinition(PERF_DATA_BLOCK *pDa
// --------------------------------------------------------------------------------------------------------------------
static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodePage, char *start, DWORD length) {
static __thread wchar_t unicode[PERFLIB_MAX_NAME_LENGTH];
WCHAR *tempBuffer; // Temporary buffer for Unicode data
DWORD charsCopied = 0;
BOOL free_tempBuffer;
if (CodePage == 0) {
// Input is already Unicode (UTF-16)
tempBuffer = (WCHAR *)start;
charsCopied = length / sizeof(WCHAR); // Convert byte length to number of WCHARs
free_tempBuffer = FALSE;
}
else {
// Convert the multi-byte instance name to Unicode (UTF-16)
// Calculate maximum possible characters in UTF-16
int charCount = MultiByteToWideChar(CodePage, 0, start, (int)length, NULL, 0);
tempBuffer = (WCHAR *)malloc(charCount * sizeof(WCHAR));
if (!tempBuffer) return FALSE;
charsCopied = MultiByteToWideChar(CodePage, 0, start, (int)length, tempBuffer, charCount);
if (charsCopied == 0) {
free(tempBuffer);
dst[0] = '\0';
return FALSE;
}
free_tempBuffer = TRUE;
tempBuffer = unicode;
charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length);
if(!charsCopied) return FALSE;
}
// Now convert from Unicode (UTF-16) to UTF-8
int bytesCopied = WideCharToMultiByte(CP_UTF8, 0, tempBuffer, (int)charsCopied, dst, (int)dst_len, NULL, NULL);
if (bytesCopied == 0) {
if (free_tempBuffer) free(tempBuffer);
dst[0] = '\0'; // Ensure the buffer is null-terminated even on failure
return FALSE;
}
dst[bytesCopied] = '\0'; // Ensure buffer is null-terminated
if (free_tempBuffer) free(tempBuffer); // Free temporary buffer if used
dst[bytesCopied - 1] = '\0'; // Ensure buffer is null-terminated
return TRUE;
}

1
src/libnetdata/os/windows-perflib/perflib.h
View file

@ -25,6 +25,7 @@ const char *RegistryFindNameByID(DWORD id);
const char *RegistryFindHelpByID(DWORD id);
DWORD RegistryFindIDByName(const char *name);
#define PERFLIB_REGISTRY_NAME_NOT_FOUND (DWORD)-1
#define PERFLIB_MAX_NAME_LENGTH 1024
PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id);
void perflibFreePerformanceData(void);

322
src/libnetdata/spawn_server/log-forwarder.c Normal file
View file

@ -0,0 +1,322 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "../libnetdata.h"
#include "log-forwarder.h"
typedef struct LOG_FORWARDER_ENTRY {
int fd;
char *cmd;
pid_t pid;
BUFFER *wb;
size_t pfds_idx;
bool delete;
struct LOG_FORWARDER_ENTRY *prev;
struct LOG_FORWARDER_ENTRY *next;
} LOG_FORWARDER_ENTRY;
typedef struct LOG_FORWARDER {
LOG_FORWARDER_ENTRY *entries;
ND_THREAD *thread;
SPINLOCK spinlock;
int pipe_fds[2]; // Pipe for notifications
bool running;
} LOG_FORWARDER;
static void *log_forwarder_thread_func(void *arg);
// --------------------------------------------------------------------------------------------------------------------
// helper functions
static inline LOG_FORWARDER_ENTRY *log_forwarder_find_entry_unsafe(LOG_FORWARDER *lf, int fd) {
for (LOG_FORWARDER_ENTRY *entry = lf->entries; entry; entry = entry->next) {
if (entry->fd == fd)
return entry;
}
return NULL;
}
static inline void log_forwarder_del_entry_unsafe(LOG_FORWARDER *lf, LOG_FORWARDER_ENTRY *entry) {
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(lf->entries, entry, prev, next);
buffer_free(entry->wb);
freez(entry->cmd);
close(entry->fd);
freez(entry);
}
static inline void log_forwarder_wake_up_worker(LOG_FORWARDER *lf) {
char ch = 0;
ssize_t bytes_written = write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
if (bytes_written != 1)
nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to write to notification pipe");
}
// --------------------------------------------------------------------------------------------------------------------
// starting / stopping
LOG_FORWARDER *log_forwarder_start(void) {
LOG_FORWARDER *lf = callocz(1, sizeof(LOG_FORWARDER));
spinlock_init(&lf->spinlock);
if (pipe(lf->pipe_fds) != 0) {
freez(lf);
return NULL;
}
// make sure read() will not block on this pipe
sock_setnonblock(lf->pipe_fds[PIPE_READ]);
lf->running = true;
lf->thread = nd_thread_create("log-fw", NETDATA_THREAD_OPTION_JOINABLE, log_forwarder_thread_func, lf);
return lf;
}
static inline void mark_all_entries_for_deletion_unsafe(LOG_FORWARDER *lf) {
for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ;entry = entry->next)
entry->delete = true;
}
void log_forwarder_stop(LOG_FORWARDER *lf) {
if(!lf || !lf->running) return;
// Signal the thread to stop
spinlock_lock(&lf->spinlock);
lf->running = false;
// mark them all for deletion
mark_all_entries_for_deletion_unsafe(lf);
// Send a byte to the pipe to wake up the thread
char ch = 0;
write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
spinlock_unlock(&lf->spinlock);
// Wait for the thread to finish
close(lf->pipe_fds[PIPE_WRITE]); // force it to quit
nd_thread_join(lf->thread);
close(lf->pipe_fds[PIPE_READ]);
freez(lf);
}
// --------------------------------------------------------------------------------------------------------------------
// managing entries
void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd) {
if(!lf || !lf->running || fd < 0) return;
LOG_FORWARDER_ENTRY *entry = callocz(1, sizeof(LOG_FORWARDER_ENTRY));
entry->fd = fd;
entry->cmd = NULL;
entry->pid = 0;
entry->pfds_idx = 0;
entry->delete = false;
entry->wb = buffer_create(0, NULL);
spinlock_lock(&lf->spinlock);
// Append to the entries list
DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(lf->entries, entry, prev, next);
// Send a byte to the pipe to wake up the thread
log_forwarder_wake_up_worker(lf);
spinlock_unlock(&lf->spinlock);
}
bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd) {
if(!lf || !lf->running || fd < 0) return false;
bool ret = false;
spinlock_lock(&lf->spinlock);
LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
if(entry) {
entry->delete = true;
// Send a byte to the pipe to wake up the thread
log_forwarder_wake_up_worker(lf);
ret = true;
}
spinlock_unlock(&lf->spinlock);
return ret;
}
void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd) {
if(!lf || !lf->running || fd < 0 || !cmd || !*cmd) return;
spinlock_lock(&lf->spinlock);
LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
if (entry) {
freez(entry->cmd);
entry->cmd = strdupz(cmd);
}
spinlock_unlock(&lf->spinlock);
}
void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid) {
if(!lf || !lf->running || fd < 0) return;
spinlock_lock(&lf->spinlock);
LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
if (entry)
entry->pid = pid;
spinlock_unlock(&lf->spinlock);
}
// --------------------------------------------------------------------------------------------------------------------
// log forwarder thread
static inline void log_forwarder_log(LOG_FORWARDER *lf __maybe_unused, LOG_FORWARDER_ENTRY *entry, const char *msg) {
const char *s = msg;
while(*s && isspace((uint8_t)*s)) s++;
if(*s == '\0') return; // do not log empty lines
ND_LOG_STACK lgs[] = {
ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, entry->cmd ? entry->cmd : "unknown"),
ND_LOG_FIELD_I64(NDF_TID, entry->pid),
ND_LOG_FIELD_END(),
};
ND_LOG_STACK_PUSH(lgs);
nd_log(NDLS_COLLECTORS, NDLP_WARNING, "STDERR: %s", msg);
}
// returns the number of entries active
static inline size_t log_forwarder_remove_deleted_unsafe(LOG_FORWARDER *lf) {
size_t entries = 0;
LOG_FORWARDER_ENTRY *entry = lf->entries;
while(entry) {
LOG_FORWARDER_ENTRY *next = entry->next;
if(entry->delete) {
if (buffer_strlen(entry->wb))
// there is something not logged in it - log it
log_forwarder_log(lf, entry, buffer_tostring(entry->wb));
log_forwarder_del_entry_unsafe(lf, entry);
}
else
entries++;
entry = next;
}
return entries;
}
static void *log_forwarder_thread_func(void *arg) {
LOG_FORWARDER *lf = (LOG_FORWARDER *)arg;
while (1) {
spinlock_lock(&lf->spinlock);
if (!lf->running) {
mark_all_entries_for_deletion_unsafe(lf);
log_forwarder_remove_deleted_unsafe(lf);
spinlock_unlock(&lf->spinlock);
break;
}
// Count the number of fds
size_t nfds = 1 + log_forwarder_remove_deleted_unsafe(lf);
struct pollfd pfds[nfds];
// First, the notification pipe
pfds[0].fd = lf->pipe_fds[PIPE_READ];
pfds[0].events = POLLIN;
int idx = 1;
for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next, idx++) {
pfds[idx].fd = entry->fd;
pfds[idx].events = POLLIN;
entry->pfds_idx = idx;
}
spinlock_unlock(&lf->spinlock);
int timeout = 200; // 200ms
int ret = poll(pfds, nfds, timeout);
if (ret > 0) {
// Check the notification pipe
if (pfds[0].revents & POLLIN) {
// Read and discard the data
char buf[256];
ssize_t bytes_read = read(lf->pipe_fds[PIPE_READ], buf, sizeof(buf));
// Ignore the data; proceed regardless of the result
if (bytes_read == -1) {
if (errno != EAGAIN && errno != EWOULDBLOCK) {
// Handle read error if necessary
nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to read from notification pipe");
return NULL;
}
}
}
// Now check the other fds
spinlock_lock(&lf->spinlock);
size_t to_remove = 0;
// read or mark them for deletion
for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next) {
if (entry->pfds_idx < 1 || entry->pfds_idx >= nfds || !(pfds[entry->pfds_idx].revents & POLLIN))
continue;
BUFFER *wb = entry->wb;
buffer_need_bytes(wb, 1024);
ssize_t bytes_read = read(entry->fd, &wb->buffer[wb->len], wb->size - wb->len - 1);
if(bytes_read > 0)
wb->len += bytes_read;
else if(bytes_read == 0 || (bytes_read == -1 && errno != EINTR && errno != EAGAIN)) {
// EOF or error
entry->delete = true;
to_remove++;
}
// log as many lines are they have been received
char *start = (char *)buffer_tostring(wb);
char *newline = strchr(start, '\n');
while(newline) {
*newline = '\0';
log_forwarder_log(lf, entry, start);
start = ++newline;
newline = strchr(newline, '\n');
}
if(start != wb->buffer) {
wb->len = strlen(start);
if (wb->len)
memmove(wb->buffer, start, wb->len);
}
entry->pfds_idx = 0;
}
spinlock_unlock(&lf->spinlock);
}
else if (ret == 0) {
// Timeout, nothing to do
continue;
}
else
nd_log(NDLS_COLLECTORS, NDLP_ERR, "Log forwarder: poll() error");
}
return NULL;
}

17
src/libnetdata/spawn_server/log-forwarder.h Normal file
View file

@ -0,0 +1,17 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_LOG_FORWARDER_H
#define NETDATA_LOG_FORWARDER_H
#include "../libnetdata.h"
typedef struct LOG_FORWARDER LOG_FORWARDER;
LOG_FORWARDER *log_forwarder_start(void); // done once, at spawn_server_create()
void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd); // to add a new fd
void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd); // set the syslog identifier
void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid); // set the pid of the child process
bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd); // to remove an fd
void log_forwarder_stop(LOG_FORWARDER *lf); // done once, at spawn_server_destroy()
#endif //NETDATA_LOG_FORWARDER_H

3
src/libnetdata/spawn_server/spawn_server_internals.h
View file

@ -6,6 +6,7 @@
#include "../libnetdata.h"
#include "spawn_server.h"
#include "spawn_library.h"
#include "log-forwarder.h"
#if defined(OS_WINDOWS)
#define SPAWN_SERVER_VERSION_WINDOWS 1
@ -62,6 +63,7 @@ struct spawn_server {
#endif
#if defined(SPAWN_SERVER_VERSION_WINDOWS)
LOG_FORWARDER *log_forwarder;
#endif
};
@ -70,6 +72,7 @@ struct spawn_instance {
int sock;
int write_fd;
int read_fd;
int stderr_fd;
pid_t child_pid;
#if defined(SPAWN_SERVER_VERSION_UV)

11
src/libnetdata/spawn_server/spawn_server_nofork.c
View file

@ -59,6 +59,7 @@ static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *rq) {
// close all open file descriptors of the parent, but keep ours
os_close_all_non_std_open_fds_except(rq->fds, 4, 0);
nd_log_reopen_log_files_for_spawn_server();
// set the process name
os_setproctitle("spawn-child", server->argc, server->argv);
@ -363,7 +364,6 @@ static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN
else if (pid == 0) {
// the child
gettid_uncached(); // make sure the logger logs valid pids
spawn_server_run_child(server, rq);
exit(63);
}
@ -984,22 +984,24 @@ static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) {
}
static void replace_stdio_with_dev_null() {
// we cannot log in this function - the logger is not yet initialized after fork()
int dev_null_fd = open("/dev/null", O_RDWR);
if (dev_null_fd == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
// nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno));
return;
}
// Redirect stdin (fd 0)
if (dup2(dev_null_fd, STDIN_FILENO) == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
// nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno));
close(dev_null_fd);
return;
}
// Redirect stdout (fd 1)
if (dup2(dev_null_fd, STDOUT_FILENO) == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
// nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno));
close(dev_null_fd);
return;
}
@ -1070,7 +1072,6 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name
pid_t pid = fork();
if (pid == 0) {
// the child - the spawn server
gettid_uncached(); // make sure the logger logs valid pids
{
char buf[15];

63
src/libnetdata/spawn_server/spawn_server_windows.c
View file

@ -40,11 +40,18 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, c
server->name = strdupz(name);
else
server->name = strdupz("unnamed");
server->log_forwarder = log_forwarder_start();
return server;
}
void spawn_server_destroy(SPAWN_SERVER *server) {
if (server) {
if (server->log_forwarder) {
log_forwarder_stop(server->log_forwarder);
server->log_forwarder = NULL;
}
freez((void *)server->name);
freez(server);
}
@ -136,13 +143,13 @@ int set_fd_blocking(int fd) {
// }
//}
SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd __maybe_unused, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) {
static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
if (type != SPAWN_INSTANCE_TYPE_EXEC)
return NULL;
int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 };
int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 }, pipe_stderr[2] = { -1, -1 };
errno_clear();
@ -166,12 +173,21 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
goto cleanup;
}
if (pipe(pipe_stderr) == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Cannot create stderr pipe() for request No %zu, command: %s",
instance->request_id, command);
goto cleanup;
}
// Ensure pipes are in blocking mode
if (set_fd_blocking(pipe_stdin[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdin[PIPE_WRITE]) == -1 ||
set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1) {
set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1 ||
set_fd_blocking(pipe_stderr[PIPE_READ]) == -1 || set_fd_blocking(pipe_stderr[PIPE_WRITE]) == -1) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Failed to set blocking I/O on pipes for request No %zu, command: %s",
instance->request_id, command);
goto cleanup;
}
// do not run multiple times this section
@ -181,9 +197,9 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Convert POSIX file descriptors to Windows handles
HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[PIPE_READ]);
HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[PIPE_WRITE]);
HANDLE stderr_handle = (HANDLE)_get_osfhandle(stderr_fd);
HANDLE stderr_write_handle = (HANDLE)_get_osfhandle(pipe_stderr[PIPE_WRITE]);
if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_handle == INVALID_HANDLE_VALUE) {
if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_write_handle == INVALID_HANDLE_VALUE) {
spinlock_unlock(&spinlock);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s",
@ -194,7 +210,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Set handle inheritance
if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
!SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) ||
!SetHandleInformation(stderr_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
!SetHandleInformation(stderr_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) {
spinlock_unlock(&spinlock);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s",
@ -210,18 +226,18 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdInput = stdin_read_handle;
si.hStdOutput = stdout_write_handle;
si.hStdError = stderr_handle;
si.hStdError = stderr_write_handle;
// Retrieve the current environment block
char* env_block = GetEnvironmentStrings();
// print_environment_block(env_block);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: Running request No %zu, command: '%s'",
instance->request_id, command);
int fds[3] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], stderr_fd };
os_close_all_non_std_open_fds_except(fds, 3, CLOSE_RANGE_CLOEXEC);
int fds_to_keep_open[] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], pipe_stderr[PIPE_WRITE] };
os_close_all_non_std_open_fds_except(fds_to_keep_open, 3, CLOSE_RANGE_CLOEXEC);
// Spawn the process
errno_clear();
@ -247,6 +263,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Close unused pipe ends
close(pipe_stdin[PIPE_READ]); pipe_stdin[PIPE_READ] = -1;
close(pipe_stdout[PIPE_WRITE]); pipe_stdout[PIPE_WRITE] = -1;
close(pipe_stderr[PIPE_WRITE]); pipe_stderr[PIPE_WRITE] = -1;
// Store process information in instance
instance->dwProcessId = pi.dwProcessId;
@ -256,9 +273,15 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
// Convert handles to POSIX file descriptors
instance->write_fd = pipe_stdin[PIPE_WRITE];
instance->read_fd = pipe_stdout[PIPE_READ];
instance->stderr_fd = pipe_stderr[PIPE_READ];
// Add stderr_fd to the log forwarder
log_forwarder_add_fd(server->log_forwarder, instance->stderr_fd);
log_forwarder_annotate_fd_name(server->log_forwarder, instance->stderr_fd, command);
log_forwarder_annotate_fd_pid(server->log_forwarder, instance->stderr_fd, spawn_server_instance_pid(instance));
errno_clear();
nd_log(NDLS_COLLECTORS, NDLP_ERR,
nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: created process for request No %zu, pid %d (winpid %d), command: %s",
instance->request_id, (int)instance->child_pid, (int)pi.dwProcessId, command);
@ -269,6 +292,8 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo
if (pipe_stdin[PIPE_WRITE] >= 0) close(pipe_stdin[PIPE_WRITE]);
if (pipe_stdout[PIPE_READ] >= 0) close(pipe_stdout[PIPE_READ]);
if (pipe_stdout[PIPE_WRITE] >= 0) close(pipe_stdout[PIPE_WRITE]);
if (pipe_stderr[PIPE_READ] >= 0) close(pipe_stderr[PIPE_READ]);
if (pipe_stderr[PIPE_WRITE] >= 0) close(pipe_stderr[PIPE_WRITE]);
freez(instance);
return NULL;
}
@ -314,7 +339,7 @@ static void TerminateChildProcesses(SPAWN_INSTANCE *si) {
if (pe.th32ParentProcessID == si->dwProcessId) {
HANDLE hChildProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pe.th32ProcessID);
if (hChildProcess) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
nd_log(NDLS_COLLECTORS, NDLP_WARNING,
"SPAWN PARENT: killing subprocess %u of request No %zu, pid %d (winpid %u)",
pe.th32ProcessID, si->request_id, (int)si->child_pid, si->dwProcessId);
@ -378,6 +403,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
// to have them, to avoid abnormal shutdown of the plugins
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
if(si->stderr_fd != -1) {
if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
close(si->stderr_fd);
si->stderr_fd = -1;
}
errno_clear();
if(TerminateProcess(si->process_handle, STATUS_CONTROL_C_EXIT) == 0)
@ -394,6 +425,12 @@ int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) {
if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; }
if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; }
if(si->stderr_fd != -1) {
if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd))
close(si->stderr_fd);
si->stderr_fd = -1;
}
// wait for the process to end
WaitForSingleObject(si->process_handle, INFINITE);
@ -404,7 +441,7 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *
char *err = GetErrorString(exit_code);
nd_log(NDLS_COLLECTORS, NDLP_ERR,
nd_log(NDLS_COLLECTORS, NDLP_INFO,
"SPAWN PARENT: child of request No %zu, pid %d (winpid %u), exited with code %u (0x%x): %s",
si->request_id, (int)si->child_pid, si->dwProcessId,
(unsigned)exit_code, (unsigned)exit_code, err ? err : "(no reason text)");

192
src/libnetdata/string/utf8.c Normal file
View file

@ -0,0 +1,192 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "../libnetdata.h"
#if defined(OS_WINDOWS)
#include <windows.h>
/*
* Convert any CodePage to UTF16
* Goals:
* 1. Destination is always NULL terminated
* 2. If the destination buffer is not enough, return as much as possible data (truncate)
* 3. Always return the number of wide characters written, including the null terminator
*/
size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len) {
if(!src || src_len == 0) {
// invalid input
if(dst && dst_size)
*dst = L'\0';
return 0;
}
if(!dst || !dst_size) {
// the caller wants to know the buffer to allocate for the conversion
int required = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
if(required <= 0) return 0; // error in the conversion
// Add 1 for null terminator only if src_len is not -1
// so that the caller can call us again to get the entire string (not truncated)
return (size_t)required + ((src_len != -1) ? 1 : 0);
}
// do the conversion directly to the destination buffer
int rc = MultiByteToWideChar(CodePage, 0, src, src_len, dst, (int)dst_size);
if(rc <= 0) {
// conversion failed, let's see why...
DWORD status = GetLastError();
if(status == ERROR_INSUFFICIENT_BUFFER) {
// it cannot fit entirely, let's allocate a new buffer to convert it
// and then truncate it to the destination buffer
// clear errno and LastError to clear the error of the
// MultiByteToWideChar() that failed
errno_clear();
// get the required size
int required_size = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0);
// mallocz() never fails (exits the program on NULL)
wchar_t *tmp = mallocz(required_size * sizeof(wchar_t));
// convert it, now it should fit
rc = MultiByteToWideChar(CodePage, 0, src, src_len, tmp, required_size);
if (rc <= 0) {
// it failed!
*dst = L'\0';
freez(tmp);
return 0;
}
size_t len = rc;
// copy as much as we can
memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(wchar_t));
// null terminate it
dst[MIN(len, (dst_size - 1))] = L'\0';
// free the temporary buffer
freez(tmp);
// return the actual bytes written
return MIN(len, dst_size);
}
// empty the destination
*dst = L'\0';
return 0;
}
size_t len = rc;
if(len >= dst_size) {
// truncate it to fit the null
dst[dst_size - 1] = L'\0';
return dst_size;
}
if(dst[len - 1] != L'\0') {
// the result is not null terminated
// append the null
dst[len] = L'\0';
return len + 1;
}
// the result is already null terminated
return len;
}
/*
* Convert UTF16 (wide-character string) to UTF8
* Goals:
* 1. Destination is always NULL terminated
* 2. If the destination buffer is not enough, return as much as possible data (truncate)
* 3. Always return the number of bytes written, including the null terminator
*/
size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len) {
if (!src || src_len == 0) {
// invalid input
if(dst && dst_size)
*dst = L'\0';
return 0;
}
if (!dst || dst_size == 0) {
// The caller wants to know the buffer size required for the conversion
int required = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
if (required <= 0) return 0; // error in the conversion
// Add 1 for null terminator only if src_len is not -1
return (size_t)required + ((src_len != -1) ? 1 : 0);
}
// Perform the conversion directly into the destination buffer
int rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, dst, (int)dst_size, NULL, NULL);
if (rc <= 0) {
// Conversion failed, let's see why...
DWORD status = GetLastError();
if (status == ERROR_INSUFFICIENT_BUFFER) {
// It cannot fit entirely, let's allocate a new buffer to convert it
// and then truncate it to the destination buffer
// Clear errno and LastError to clear the error of the
// WideCharToMultiByte() that failed
errno_clear();
// Get the required size
int required_size = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL);
// mallocz() never fails (exits the program on NULL)
char *tmp = mallocz(required_size * sizeof(char));
// Convert it, now it should fit
rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, tmp, required_size, NULL, NULL);
if (rc <= 0) {
// Conversion failed
*dst = '\0';
freez(tmp);
return 0;
}
size_t len = rc;
// Copy as much as we can
memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(char));
// Null-terminate it
dst[MIN(len, (dst_size - 1))] = '\0';
// Free the temporary buffer
freez(tmp);
// Return the actual bytes written
return MIN(len, dst_size);
}
// Empty the destination
*dst = '\0';
return 0;
}
size_t len = rc;
if (len >= dst_size) {
// Truncate it to fit the null terminator
dst[dst_size - 1] = '\0';
return dst_size;
}
if (dst[len - 1] != '\0') {
// The result is not null-terminated
// Append the null terminator
dst[len] = '\0';
return len + 1;
}
// The result is already null-terminated
return len;
}
#endif

21
src/libnetdata/string/utf8.h
View file

@ -6,4 +6,25 @@
#define IS_UTF8_BYTE(x) ((uint8_t)(x) & (uint8_t)0x80)
#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x) && ((uint8_t)(x) & (uint8_t)0x40))
#ifndef _countof
#define _countof(x) (sizeof(x) / sizeof(*(x)))
#endif
#if defined(OS_WINDOWS)
// return an always null terminated wide string, truncate to given size if destination is not big enough,
// src_len can be -1 use all of it.
// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len);
// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len)
// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it.
// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated).
size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len);
#endif
#endif /* NETDATA_STRING_UTF8_H */

11
src/libnetdata/template-enum.h
View file

@ -41,6 +41,7 @@
#define BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(type) \
type type ## _2id_one(const char *str); \
const char *type##_2str_one(type id); \
const char *type##_2json(BUFFER *wb, const char *key, type id);
#define BITMAP_STR_DEFINE_FUNCTIONS(type, def, def_str) \
@ -57,6 +58,16 @@
return def; \
} \
\
const char *type##_2str_one(type id) \
{ \
for (size_t i = 0; type ## _names[i].name; i++) { \
if (id == type ## _names[i].id) \
return type ## _names[i].name; \
} \
\
return def_str; \
} \
\
const char *type##_2json(BUFFER *wb, const char *key, type id) \
{ \
buffer_json_member_add_array(wb, key); \