Windows Events Improvements 1 (#18528)

* extract more information for the XML to form the message

* support slicing for EventID and Level

* the fastest it can get

* remove obsolete member

* stop service Netdata before starting Netdata

* do not reuse buffers

* convert newlines to unix style

* send progress based on the number of log events

* rework on opcodes, keywords and task to allow hard-coded values

* complete messages; facets annotated from XML

* working with field and provider caches

* cleanup

* query the event log in batch mode

* extract keywords from publishers

* detect closed sockets on non-Linux systems

* unified event log processing

* work on caching, improving accuracy

* optimization on keywords

* caching improvements

CMakeLists.txt

@@ -1411,6 +1411,10 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
         src/collectors/windows-events.plugin/windows-events-unicode.h
         src/collectors/windows-events.plugin/windows-events-xml.c
         src/collectors/windows-events.plugin/windows-events-xml.h
+        src/collectors/windows-events.plugin/windows-events-publishers.c
+        src/collectors/windows-events.plugin/windows-events-publishers.h
+        src/collectors/windows-events.plugin/windows-events-fields-cache.c
+        src/collectors/windows-events.plugin/windows-events-fields-cache.h
 )
 
 set(WINDOWS_PLUGIN_FILES

packaging/utils/compile-on-windows.sh

@@ -3,7 +3,7 @@
 # On MSYS2, install these dependencies to build netdata:
 install_dependencies() {
     pacman -S \
-        git cmake ninja base-devel msys2-devel \
+        git cmake ninja clang base-devel msys2-devel \
         libyaml-devel libzstd-devel libutil-linux libutil-linux-devel \
         mingw-w64-x86_64-toolchain mingw-w64-ucrt-x86_64-toolchain \
         mingw64/mingw-w64-x86_64-mold ucrt64/mingw-w64-ucrt-x86_64-mold \
@@ -75,6 +75,9 @@ ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1
 #echo "Compile with:"
 #echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"
 
+echo "Stopping service Netdata"
+sc stop "Netdata" || echo "Failed"
+
 echo "starting netdata..."
 # enable JIT debug with gdb
 export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"

src/collectors/systemd-journal.plugin/systemd-journal.c

@@ -336,10 +336,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
     LOGS_QUERY_STATUS *fqs) {
 
     usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
-
+    lqs_query_timeframe(fqs, anchor_delta);
-    usec_t start_ut = ((fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.before_ut) + anchor_delta;
+    usec_t start_ut = fqs->query.start_ut;
-    usec_t stop_ut = (fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.after_ut;
+    usec_t stop_ut = fqs->query.stop_ut;
-    bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
+    bool stop_when_full = fqs->query.stop_when_full;
 
     fqs->c.query_file.start_ut = start_ut;
     fqs->c.query_file.stop_ut = stop_ut;
@@ -451,10 +451,10 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
     LOGS_QUERY_STATUS *fqs) {
 
     usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
-
+    lqs_query_timeframe(fqs, anchor_delta);
-    usec_t start_ut = (fqs->rq.data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->rq.after_ut;
+    usec_t start_ut = fqs->query.start_ut;
-    usec_t stop_ut = ((fqs->rq.data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->rq.before_ut) + anchor_delta;
+    usec_t stop_ut = fqs->query.stop_ut;
-    bool stop_when_full = (fqs->rq.data_only && !fqs->anchor.stop_ut);
+    bool stop_when_full = fqs->query.stop_when_full;
 
     fqs->c.query_file.start_ut = start_ut;
     fqs->c.query_file.stop_ut = stop_ut;
@@ -747,8 +747,13 @@ static int netdata_systemd_journal_query(BUFFER *wb, LOGS_QUERY_STATUS *lqs) {
 
     lqs->c.files_matched = files_used;
 
-    if(lqs->rq.if_modified_since && !files_are_newer)
+    if(lqs->rq.if_modified_since && !files_are_newer) {
+        // release the files
+        for(size_t f = 0; f < files_used ;f++)
+            dictionary_acquired_item_release(journal_files_registry, file_items[f]);
+
         return rrd_call_function_error(wb, "not modified", HTTP_RESP_NOT_MODIFIED);
+    }
 
     // sort the files, so that they are optimal for facets
     if(files_used >= 2) {

src/collectors/windows-events.plugin/windows-events-fields-cache.c

@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "windows-events-fields-cache.h"
+
+typedef struct field_key {
+    uint64_t value;
+    ND_UUID provider;
+} WEVT_FIELD_KEY;
+
+typedef struct field_value {
+    WEVT_FIELD_KEY key;
+    uint32_t name_size;
+    char name[];
+} WEVT_FIELD_VALUE;
+
+#define SIMPLE_HASHTABLE_NAME _FIELDS_CACHE
+#define SIMPLE_HASHTABLE_VALUE_TYPE WEVT_FIELD_VALUE
+#define SIMPLE_HASHTABLE_KEY_TYPE WEVT_FIELD_KEY
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION field_cache_value_to_key
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION field_cache_cache_compar
+#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
+#include "libnetdata/simple_hashtable.h"
+
+static inline WEVT_FIELD_KEY *field_cache_value_to_key(WEVT_FIELD_VALUE *p) {
+    return &p->key;
+}
+
+static inline bool field_cache_cache_compar(WEVT_FIELD_KEY *a, WEVT_FIELD_KEY *b) {
+    return memcmp(a, b, sizeof(WEVT_FIELD_KEY)) == 0;
+}
+
+struct ht {
+    SPINLOCK spinlock;
+    size_t allocations;
+    size_t bytes;
+    struct simple_hashtable_FIELDS_CACHE ht;
+};
+
+static struct {
+    bool initialized;
+    struct ht ht[WEVT_FIELD_TYPE_MAX];
+} fdc = {
+        .initialized = false,
+};
+
+void field_cache_init(void) {
+    for(size_t type = 0; type < WEVT_FIELD_TYPE_MAX ; type++) {
+        spinlock_init(&fdc.ht[type].spinlock);
+        simple_hashtable_init_FIELDS_CACHE(&fdc.ht[type].ht, 10000);
+    }
+}
+
+static inline bool should_zero_provider(WEVT_FIELD_TYPE type, uint64_t value) {
+    switch(type) {
+        case WEVT_FIELD_TYPE_LEVEL:
+            return !is_valid_publisher_level(value, true);
+
+        case WEVT_FIELD_TYPE_KEYWORDS:
+            return !is_valid_publisher_keywords(value, true);
+
+        case WEVT_FIELD_TYPE_OPCODE:
+            return !is_valid_publisher_opcode(value, true);
+
+        case WEVT_FIELD_TYPE_TASK:
+            return !is_valid_publisher_task(value, true);
+
+        default:
+            return false;
+    }
+}
+
+bool field_cache_get(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *dst) {
+    fatal_assert(type < WEVT_FIELD_TYPE_MAX);
+
+    struct ht *ht = &fdc.ht[type];
+
+    WEVT_FIELD_KEY t = {
+            .value = value,
+            .provider = should_zero_provider(type, value) ? UUID_ZERO : *uuid,
+    };
+    XXH64_hash_t hash = XXH3_64bits(&t, sizeof(t));
+
+    spinlock_lock(&ht->spinlock);
+    SIMPLE_HASHTABLE_SLOT_FIELDS_CACHE *slot = simple_hashtable_get_slot_FIELDS_CACHE(&ht->ht, hash, &t, true);
+    WEVT_FIELD_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+    spinlock_unlock(&ht->spinlock);
+
+    if(v) {
+        txt_utf8_resize(dst, v->name_size, false);
+        memcpy(dst->data, v->name, v->name_size);
+        dst->used = v->name_size;
+        dst->src = TXT_SOURCE_FIELD_CACHE;
+        return true;
+    }
+
+    return false;
+}
+
+static WEVT_FIELD_VALUE *wevt_create_cache_entry(WEVT_FIELD_KEY *t, TXT_UTF8 *name, size_t *bytes) {
+    *bytes = sizeof(WEVT_FIELD_VALUE) + name->used;
+    WEVT_FIELD_VALUE *v = callocz(1, *bytes);
+    v->key = *t;
+    memcpy(v->name, name->data, name->used);
+    v->name_size = name->used;
+    return v;
+}
+
+//static bool is_numeric(const char *s) {
+//    while(*s) {
+//        if(!isdigit((uint8_t)*s++))
+//            return false;
+//    }
+//
+//    return true;
+//}
+
+void field_cache_set(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *name) {
+    fatal_assert(type < WEVT_FIELD_TYPE_MAX);
+
+    struct ht *ht = &fdc.ht[type];
+
+    WEVT_FIELD_KEY t = {
+            .value = value,
+            .provider = should_zero_provider(type, value) ? UUID_ZERO : *uuid,
+    };
+    XXH64_hash_t hash = XXH3_64bits(&t, sizeof(t));
+
+    spinlock_lock(&ht->spinlock);
+    SIMPLE_HASHTABLE_SLOT_FIELDS_CACHE *slot = simple_hashtable_get_slot_FIELDS_CACHE(&ht->ht, hash, &t, true);
+    WEVT_FIELD_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+    if(!v) {
+        size_t bytes;
+        v = wevt_create_cache_entry(&t, name, &bytes);
+        simple_hashtable_set_slot_FIELDS_CACHE(&ht->ht, slot, hash, v);
+
+        ht->allocations++;
+        ht->bytes += bytes;
+    }
+//    else {
+//        if((v->name_size == 1 && name->used > 0) || is_numeric(v->name)) {
+//            size_t bytes;
+//            WEVT_FIELD_VALUE *nv = wevt_create_cache_entry(&t, name, &bytes);
+//            simple_hashtable_set_slot_FIELDS_CACHE(&ht->ht, slot, hash, nv);
+//            ht->bytes += name->used;
+//            ht->bytes -= v->name_size;
+//            freez(v);
+//        }
+//        else if(name->used > 2 && !is_numeric(name->data) && (v->name_size != name->used || strcasecmp(v->name, name->data) != 0)) {
+//            const char *a = v->name;
+//            const char *b = name->data;
+//            int x = 0;
+//            x++;
+//        }
+//    }
+
+    spinlock_unlock(&ht->spinlock);
+}
+

src/collectors/windows-events.plugin/windows-events-fields-cache.h

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H
+#define NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H
+
+#include "windows-events.h"
+
+typedef enum __attribute__((packed)) {
+    WEVT_FIELD_TYPE_LEVEL = 0,
+    WEVT_FIELD_TYPE_OPCODE,
+    WEVT_FIELD_TYPE_KEYWORDS,
+    WEVT_FIELD_TYPE_TASK,
+
+    // terminator
+    WEVT_FIELD_TYPE_MAX,
+} WEVT_FIELD_TYPE;
+
+void field_cache_init(void);
+bool field_cache_get(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *dst);
+void field_cache_set(WEVT_FIELD_TYPE type, const ND_UUID *uuid, uint64_t value, TXT_UTF8 *name);
+
+#endif //NETDATA_WINDOWS_EVENTS_FIELDS_CACHE_H

src/collectors/windows-events.plugin/windows-events-publishers.c

@@ -0,0 +1,569 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "windows-events-publishers.h"
+
+#define MAX_OPEN_HANDLES_PER_PUBLISHER 5
+
+struct publisher;
+
+// typedef as PROVIDER_META_HANDLE in include file
+struct provider_meta_handle {
+    pid_t owner;                        // the owner of the handle, or zero
+    uint32_t locks;                     // the number of locks the owner has on this handle
+    EVT_HANDLE hMetadata;               // the handle
+    struct publisher *publisher;        // a pointer back to the publisher
+
+    // double linked list
+    PROVIDER_META_HANDLE *prev;
+    PROVIDER_META_HANDLE *next;
+};
+
+struct provider_data {
+    uint64_t value;                     // the mask of the keyword
+    XXH64_hash_t hash;                  // the hash of the name
+    uint32_t len;                       // the length of the name
+    char *name;                         // the name of the keyword in UTF-8
+};
+
+struct provider_list {
+    uint64_t min, max;
+    bool exceeds_data_type;             // true when the manifest values exceed the capacity of the EvtXXX() API
+    uint32_t total;                     // the number of entries in the array
+    struct provider_data *array;        // the array of entries, sorted (for binary search)
+};
+
+typedef struct publisher {
+    ND_UUID uuid;                       // the Provider GUID
+    const char *name;                   // the Provider name (UTF-8)
+    uint32_t total_handles;             // the number of handles allocated
+    uint32_t available_handles;         // the number of available handles
+    uint32_t deleted_handles;           // the number of deleted handles
+    PROVIDER_META_HANDLE *handles;      // a double linked list of all the handles
+
+    struct provider_list keywords;
+    struct provider_list tasks;
+    struct provider_list opcodes;
+    struct provider_list levels;
+} PUBLISHER;
+
+// A hashtable implementation for publishers
+// using the provider GUID as key and PUBLISHER as value
+#define SIMPLE_HASHTABLE_NAME _PROVIDER_GUID
+#define SIMPLE_HASHTABLE_VALUE_TYPE PUBLISHER
+#define SIMPLE_HASHTABLE_KEY_TYPE ND_UUID
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION publisher_value_to_key
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION publisher_cache_compar
+#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
+#include "libnetdata/simple_hashtable.h"
+
+static struct {
+    SPINLOCK spinlock;
+    uint32_t total_publishers;
+    uint32_t total_handles;
+    uint32_t deleted_handles;
+    struct simple_hashtable_PROVIDER_GUID hashtable;
+    ARAL *aral_publishers;
+    ARAL *aral_handles;
+} pbc = {
+        .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+};
+
+static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *unicode, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id);
+
+static inline ND_UUID *publisher_value_to_key(PUBLISHER *p) {
+    return &p->uuid;
+}
+
+static inline bool publisher_cache_compar(ND_UUID *a, ND_UUID *b) {
+    return UUIDeq(*a, *b);
+}
+
+void publisher_cache_init(void) {
+    simple_hashtable_init_PROVIDER_GUID(&pbc.hashtable, 100000);
+    pbc.aral_publishers = aral_create("wevt_publishers", sizeof(PUBLISHER), 0, 4096, NULL, NULL, NULL, false, true);
+    pbc.aral_handles = aral_create("wevt_handles", sizeof(PROVIDER_META_HANDLE), 0, 4096, NULL, NULL, NULL, false, true);
+}
+
+PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName) {
+    if(!providerName || !providerName[0] || UUIDiszero(uuid))
+        return NULL;
+
+    // XXH64_hash_t hash = XXH3_64bits(&uuid, sizeof(uuid));
+    uint64_t hash = uuid.parts.low64 + uuid.parts.hig64;
+
+    spinlock_lock(&pbc.spinlock);
+
+    SIMPLE_HASHTABLE_SLOT_PROVIDER_GUID *slot =
+            simple_hashtable_get_slot_PROVIDER_GUID(&pbc.hashtable, hash, &uuid, true);
+
+    bool load_it = false;
+    PUBLISHER *p = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+    if(!p) {
+        p = aral_callocz(pbc.aral_publishers);
+        p->uuid = uuid;
+        simple_hashtable_set_slot_PROVIDER_GUID(&pbc.hashtable, slot, hash, p);
+        load_it = true;
+        pbc.total_publishers++;
+    }
+
+    pid_t me = gettid_cached();
+    PROVIDER_META_HANDLE *h;
+    for(h = p->handles; h ;h = h->next) {
+        // find the first that is mine,
+        // or the first not owned by anyone
+        if(!h->owner || h->owner == me)
+            break;
+    }
+
+    if(!h) {
+        h = aral_callocz(pbc.aral_handles);
+        h->publisher = p;
+        h->hMetadata = EvtOpenPublisherMetadata(
+                NULL,          // Local machine
+                providerName,  // Provider name
+                NULL,          // Log file path (NULL for default)
+                0,             // Locale (0 for default locale)
+                0              // Flags
+        );
+
+        // we put it at the beginning of the list
+        // to find it first if the same owner needs more locks on it
+        DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(p->handles, h, prev, next);
+        pbc.total_handles++;
+        p->total_handles++;
+        p->available_handles++;
+    }
+
+    if(!h->owner) {
+        fatal_assert(p->available_handles > 0);
+        p->available_handles--;
+        h->owner = me;
+    }
+
+    h->locks++;
+
+    if(load_it) {
+        WEVT_VARIANT content = { 0 };
+        WEVT_VARIANT property = { 0 };
+        TXT_UNICODE unicode = { 0 };
+
+        publisher_load_list(h, &content, &property, &unicode, &p->keywords, EvtPublisherMetadataKeywords);
+        publisher_load_list(h, &content, &property, &unicode, &p->levels, EvtPublisherMetadataLevels);
+        publisher_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes);
+        publisher_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks);
+
+        txt_unicode_cleanup(&unicode);
+        wevt_variant_cleanup(&content);
+        wevt_variant_cleanup(&property);
+    }
+
+    spinlock_unlock(&pbc.spinlock);
+
+    return h;
+}
+
+EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h) {
+    return h ? h->hMetadata : NULL;
+}
+
+PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h) {
+    if(h) h->locks++;
+    return h;
+}
+
+void publisher_release(PROVIDER_META_HANDLE *h) {
+    if(!h) return;
+    pid_t me = gettid_cached();
+    fatal_assert(h->owner == me);
+    fatal_assert(h->locks > 0);
+    if(--h->locks == 0) {
+        PUBLISHER *p = h->publisher;
+
+        spinlock_lock(&pbc.spinlock);
+        h->owner = 0;
+
+        if(++p->available_handles > MAX_OPEN_HANDLES_PER_PUBLISHER) {
+            // there are multiple handles on this publisher
+            DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
+
+            if(h->hMetadata)
+                EvtClose(h->hMetadata);
+
+            aral_freez(pbc.aral_handles, h);
+
+            pbc.total_handles--;
+            p->total_handles--;
+
+            pbc.deleted_handles++;
+            p->deleted_handles++;
+
+            p->available_handles--;
+        }
+        else if(h->next) {
+            // it is not the last, put it at the end
+            DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(p->handles, h, prev, next);
+            DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(p->handles, h, prev, next);
+        }
+
+        spinlock_unlock(&pbc.spinlock);
+    }
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// load publisher lists
+
+static bool wevt_get_property_from_array(WEVT_VARIANT *property, EVT_HANDLE handle, DWORD dwIndex, EVT_PUBLISHER_METADATA_PROPERTY_ID PropertyId) {
+    DWORD used = 0;
+
+    if (!EvtGetObjectArrayProperty(handle, PropertyId, dwIndex, 0, property->size, property->data, &used)) {
+        DWORD status = GetLastError();
+        if (status != ERROR_INSUFFICIENT_BUFFER) {
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArrayProperty() failed");
+            return false;
+        }
+
+        wevt_variant_resize(property, used);
+        if (!EvtGetObjectArrayProperty(handle, PropertyId, dwIndex, 0, property->size, property->data, &used)) {
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArrayProperty() failed");
+            return false;
+        }
+    }
+
+    property->used = used;
+    return true;
+}
+
+// Comparison function for ascending order (for Levels, Opcodes, Tasks)
+static int compare_ascending(const void *a, const void *b) {
+    struct provider_data *d1 = (struct provider_data *)a;
+    struct provider_data *d2 = (struct provider_data *)b;
+
+    if (d1->value < d2->value) return -1;
+    if (d1->value > d2->value) return 1;
+    return 0;
+}
+
+//// Comparison function for descending order (for Keywords)
+//static int compare_descending(const void *a, const void *b) {
+//    struct provider_data *d1 = (struct provider_data *)a;
+//    struct provider_data *d2 = (struct provider_data *)b;
+//
+//    if (d1->value > d2->value) return -1;
+//    if (d1->value < d2->value) return 1;
+//    return 0;
+//}
+
+static void publisher_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *unicode, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) {
+    if(!h || !h->hMetadata) return;
+
+    EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id;
+    uint8_t value_bits = 32;
+    int (*compare_func)(const void *, const void *);
+    bool (*is_valid)(uint64_t, bool);
+
+    switch(property_id) {
+        case EvtPublisherMetadataLevels:
+            name_id = EvtPublisherMetadataLevelName;
+            message_id = EvtPublisherMetadataLevelMessageID;
+            value_id = EvtPublisherMetadataLevelValue;
+            value_bits = 32;
+            compare_func = compare_ascending;
+            is_valid = is_valid_publisher_level;
+            break;
+
+        case EvtPublisherMetadataOpcodes:
+            name_id = EvtPublisherMetadataOpcodeName;
+            message_id = EvtPublisherMetadataOpcodeMessageID;
+            value_id = EvtPublisherMetadataOpcodeValue;
+            value_bits = 32;
+            is_valid = is_valid_publisher_opcode;
+            compare_func = compare_ascending;
+            break;
+
+        case EvtPublisherMetadataTasks:
+            name_id = EvtPublisherMetadataTaskName;
+            message_id = EvtPublisherMetadataTaskMessageID;
+            value_id = EvtPublisherMetadataTaskValue;
+            value_bits = 32;
+            is_valid = is_valid_publisher_task;
+            compare_func = compare_ascending;
+            break;
+
+        case EvtPublisherMetadataKeywords:
+            name_id = EvtPublisherMetadataKeywordName;
+            message_id = EvtPublisherMetadataKeywordMessageID;
+            value_id = EvtPublisherMetadataKeywordValue;
+            value_bits = 64;
+            is_valid = is_valid_publisher_keywords;
+            compare_func = NULL;
+            break;
+
+        default:
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "Internal Error: Can't handle property id %u", property_id);
+            return;
+    }
+
+    EVT_HANDLE hMetadata = h->hMetadata;
+    EVT_HANDLE hArray = NULL;
+    DWORD bufferUsed = 0;
+    DWORD itemCount = 0;
+
+    // Get the metadata array for the list (e.g., opcodes, tasks, or levels)
+    if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, 0, NULL, &bufferUsed)) {
+        DWORD status = GetLastError();
+        if (status != ERROR_INSUFFICIENT_BUFFER) {
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed");
+            goto cleanup;
+        }
+    }
+
+    wevt_variant_resize(content, bufferUsed);
+    if (!EvtGetPublisherMetadataProperty(hMetadata, property_id, 0, content->size, content->data, &bufferUsed)) {
+        nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetPublisherMetadataProperty() failed after resize");
+        goto cleanup;
+    }
+
+    // Get the number of items (e.g., levels, tasks, or opcodes)
+    hArray = content->data->EvtHandleVal;
+    if (!EvtGetObjectArraySize(hArray, &itemCount)) {
+        nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtGetObjectArraySize() failed");
+        goto cleanup;
+    }
+
+    if (itemCount == 0) {
+        l->total = 0;
+        l->array = NULL;
+        goto cleanup;
+    }
+
+    // Allocate memory for the list items
+    l->array = callocz(itemCount, sizeof(struct provider_data));
+    l->total = itemCount;
+
+    uint64_t min = UINT64_MAX, max = 0;
+
+    // Iterate over the list and populate the entries
+    for (DWORD i = 0; i < itemCount; ++i) {
+        struct provider_data *d = &l->array[i];
+
+        // Get the value (e.g., opcode, task, or level)
+        if (wevt_get_property_from_array(property, hArray, i, value_id)) {
+            switch(value_bits) {
+                case 64:
+                    d->value = wevt_field_get_uint64(property->data);
+                    break;
+
+                case 32:
+                    d->value = wevt_field_get_uint32(property->data);
+                    break;
+            }
+
+            if(d->value < min)
+                min = d->value;
+
+            if(d->value > max)
+                max = d->value;
+
+            if(!is_valid(d->value, false))
+                l->exceeds_data_type = true;
+        }
+
+        // Get the message ID
+        if (wevt_get_property_from_array(property, hArray, i, message_id)) {
+            uint32_t messageID = wevt_field_get_uint32(property->data);
+
+            if (messageID != (uint32_t)-1) {
+                if (wevt_get_message_unicode(unicode, hMetadata, NULL, messageID, EvtFormatMessageId)) {
+                    size_t len;
+                    d->name = unicode2utf8_strdupz(unicode->data, &len);
+                    d->len = len;
+                }
+            }
+        }
+
+        // Get the name if the message is missing
+        if (!d->name && wevt_get_property_from_array(property, hArray, i, name_id)) {
+            fatal_assert(property->data->Type == EvtVarTypeString);
+            size_t len;
+            d->name = unicode2utf8_strdupz(property->data->StringVal, &len);
+            d->len = len;
+        }
+
+        // Calculate the hash for the name
+        if (d->name)
+            d->hash = XXH3_64bits(d->name, d->len);
+    }
+
+    l->min = min;
+    l->max = max;
+
+    if(itemCount > 1 && compare_func != NULL) {
+        // Sort the array based on the value (ascending for all except keywords, descending for keywords)
+        qsort(l->array, itemCount, sizeof(struct provider_data), compare_func);
+    }
+
+cleanup:
+    if (hArray)
+        EvtClose(hArray);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// lookup functions
+
+// lookup bitmap metdata (returns a comma separated list of strings)
+static bool publisher_bitmap_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+    if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
+        return false;
+
+    dst->used = 0;
+
+    for(size_t k = 0; value && k < l->total; k++) {
+        struct provider_data *d = &l->array[k];
+
+        if(d->value && (value & d->value) == d->value && d->name && d->len) {
+            const char *s = d->name;
+            size_t slen = d->len;
+
+            // remove the mask from the value
+            value &= ~(d->value);
+
+            txt_utf8_resize(dst, dst->used + slen + 2 + 1, true);
+
+            if(dst->used) {
+                // add a comma and a space
+                dst->data[dst->used++] = ',';
+                dst->data[dst->used++] = ' ';
+            }
+
+            memcpy(&dst->data[dst->used], s, slen);
+            dst->used += slen;
+            dst->src = TXT_SOURCE_PUBLISHER;
+        }
+    }
+
+    if(dst->used) {
+        txt_utf8_resize(dst, dst->used + 1, true);
+        dst->data[dst->used++] = 0;
+    }
+
+    fatal_assert(dst->used <= dst->size);
+
+    return (dst->used > 0);
+}
+
+//// lookup a single value (returns its string)
+//static bool publisher_value_metadata_linear(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+//    if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
+//        return false;
+//
+//    dst->used = 0;
+//
+//    for(size_t k = 0; k < l->total; k++) {
+//        struct provider_data *d = &l->array[k];
+//
+//        if(d->value == value && d->name && d->len) {
+//            const char *s = d->name;
+//            size_t slen = d->len;
+//
+//            txt_utf8_resize(dst, slen + 1, false);
+//
+//            memcpy(dst->data, s, slen);
+//            dst->used = slen;
+//            dst->src = TXT_SOURCE_PUBLISHER;
+//
+//            break;
+//        }
+//    }
+//
+//    if(dst->used) {
+//        txt_utf8_resize(dst, dst->used + 1, true);
+//        dst->data[dst->used++] = 0;
+//    }
+//
+//    fatal_assert(dst->used <= dst->size);
+//
+//    return (dst->used > 0);
+//}
+
+static bool publisher_value_metadata(TXT_UTF8 *dst, struct provider_list *l, uint64_t value) {
+    if(value < l->min || value > l->max || !l->total || !l->array || l->exceeds_data_type)
+        return false;
+
+    // if(l->total < 3) return publisher_value_metadata_linear(dst, l, value);
+
+    dst->used = 0;
+
+    size_t left = 0;
+    size_t right = l->total - 1;
+
+    // Binary search within bounds
+    while (left <= right) {
+        size_t mid = left + (right - left) / 2;
+        struct provider_data *d = &l->array[mid];
+
+        if (d->value == value) {
+            // Value found, now check if it has a valid name and length
+            if (d->name && d->len) {
+                const char *s = d->name;
+                size_t slen = d->len;
+
+                txt_utf8_resize(dst, slen + 1, false);
+                memcpy(dst->data, s, slen);
+                dst->used = slen;
+                dst->data[dst->used++] = 0;
+                dst->src = TXT_SOURCE_PUBLISHER;
+            }
+            break;
+        }
+
+        if (d->value < value)
+            left = mid + 1;
+        else {
+            if (mid == 0) break;
+            right = mid - 1;
+        }
+    }
+
+    fatal_assert(dst->used <= dst->size);
+
+    return (dst->used > 0);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// public API to lookup metadata
+
+bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h) {
+    return h && !h->publisher->keywords.exceeds_data_type;
+}
+
+bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h) {
+    return h && !h->publisher->tasks.exceeds_data_type;
+}
+
+bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h) {
+    return h && !h->publisher->levels.exceeds_data_type;
+}
+
+bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h) {
+    return h && !h->publisher->opcodes.exceeds_data_type;
+}
+
+bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+    if(!h) return false;
+    return publisher_bitmap_metadata(dst, &h->publisher->keywords, value);
+}
+
+bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+    if(!h) return false;
+    return publisher_value_metadata(dst, &h->publisher->levels, value);
+}
+
+bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+    if(!h) return false;
+    return publisher_value_metadata(dst, &h->publisher->tasks, value);
+}
+
+bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value) {
+    if(!h) return false;
+    return publisher_value_metadata(dst, &h->publisher->opcodes, value);
+}

src/collectors/windows-events.plugin/windows-events-publishers.h

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
+#define NETDATA_WINDOWS_EVENTS_PUBLISHERS_H
+
+#include "windows-events.h"
+
+struct provider_meta_handle;
+typedef struct provider_meta_handle PROVIDER_META_HANDLE;
+
+PROVIDER_META_HANDLE *publisher_get(ND_UUID uuid, LPCWSTR providerName);
+void publisher_release(PROVIDER_META_HANDLE *h);
+EVT_HANDLE publisher_handle(PROVIDER_META_HANDLE *h);
+PROVIDER_META_HANDLE *publisher_dup(PROVIDER_META_HANDLE *h);
+
+void publisher_cache_init(void);
+
+bool publisher_keywords_cacheable(PROVIDER_META_HANDLE *h);
+bool publisher_tasks_cacheable(PROVIDER_META_HANDLE *h);
+bool is_useful_publisher_for_levels(PROVIDER_META_HANDLE *h);
+bool publisher_opcodes_cacheable(PROVIDER_META_HANDLE *h);
+
+bool publisher_get_keywords(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool publisher_get_level(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool publisher_get_task(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+bool publisher_get_opcode(TXT_UTF8 *dst, PROVIDER_META_HANDLE *h, uint64_t value);
+
+#endif //NETDATA_WINDOWS_EVENTS_PUBLISHERS_H

src/collectors/windows-events.plugin/windows-events-query.h

@@ -1,81 +1,247 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#ifndef NETDATA_WINDOWS_EVENTS_QUERY_H
+#ifndef NETDATA_WINDOWS_EVENTS_QUERY_H
-#define NETDATA_WINDOWS_EVENTS_QUERY_H
+#define NETDATA_WINDOWS_EVENTS_QUERY_H
-
+
-#include "windows-events.h"
+#include "windows-events.h"
-
+
-typedef struct wevt_event {
+#define BATCH_NEXT_EVENT 500
-    uint64_t id;                        // EventRecordId (unique and sequential per channel)
+
-    uint16_t event_id;                  // This is the template that defines the message to be shown
+typedef struct wevt_event {
-    uint16_t opcode;
+    uint64_t id;                        // EventRecordId (unique and sequential per channel)
-    uint8_t  level;                     // The severity of event
+    uint8_t  version;
-    uint8_t  version;
+    uint8_t  level;                     // The severity of event
-    uint16_t task;
+    uint8_t  opcode;                    // we receive this as 8bit, but publishers use 32bit
-    uint32_t process_id;
+    uint16_t event_id;                  // This is the template that defines the message to be shown
-    uint32_t thread_id;
+    uint16_t task;
-    uint64_t keyword;                   // Categorization of the event
+    uint32_t process_id;
-    ND_UUID  provider;
+    uint32_t thread_id;
-    ND_UUID  correlation_activity_id;
+    uint64_t keywords;                  // Categorization of the event
-    nsec_t   created_ns;
+    ND_UUID  provider;
-} WEVT_EVENT;
+    ND_UUID  correlation_activity_id;
-
+    nsec_t   created_ns;
-#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
+} WEVT_EVENT;
-
+
-typedef struct {
+#define WEVT_EVENT_EMPTY (WEVT_EVENT){ .id = 0, .created_ns = 0, }
-    WEVT_EVENT first_event;
+
-    WEVT_EVENT last_event;
+typedef struct {
-
+    EVT_VARIANT	*data;
-    uint64_t entries;
+    size_t size;
-    nsec_t duration_ns;
+    size_t used;
-    uint64_t size_bytes;
+} WEVT_VARIANT;
-} EVT_RETENTION;
+
-
+typedef struct {
-typedef struct wevt_log {
+    WEVT_EVENT first_event;
-    EVT_HANDLE event_query;
+    WEVT_EVENT last_event;
-    EVT_HANDLE render_context;
+
-
+    uint64_t entries;
-    struct {
+    nsec_t duration_ns;
-        // temp buffer used for rendering event log messages
+    uint64_t size_bytes;
-        // never use directly
+} EVT_RETENTION;
-        struct {
+
-            EVT_VARIANT	*data;
+struct provider_meta_handle;
-            size_t size;
+
-            size_t len;
+typedef struct wevt_log {
-        } content;
+    struct {
-
+        DWORD size;
-        // temp buffer used for fetching and converting UNICODE and UTF-8
+        DWORD used;
-        // every string operation overwrites it, multiple times per event log entry
+        EVT_HANDLE bk[BATCH_NEXT_EVENT];
-        // it can be used within any function, for its own purposes,
+    } batch;
-        // but never share between functions
+
-        TXT_UNICODE unicode;
+    EVT_HANDLE bookmark;
-
+    EVT_HANDLE event_query;
-        // string attributes of the current event log entry
+    EVT_HANDLE render_context;
-        // valid until another event if fetched
+    struct provider_meta_handle *publisher;
-        TXT_UTF8 channel;
+
-        TXT_UTF8 provider;
+    struct {
-        TXT_UTF8 source;
+        // temp buffer used for rendering event log messages
-        TXT_UTF8 computer;
+        // never use directly
-        TXT_UTF8 event;
+        WEVT_VARIANT content;
-        TXT_UTF8 user;
+
-        TXT_UTF8 opcode;
+        // temp buffer used for fetching and converting UNICODE and UTF-8
-        TXT_UTF8 level;
+        // every string operation overwrites it, multiple times per event log entry
-        TXT_UTF8 keyword;
+        // it can be used within any function, for its own purposes,
-        TXT_UTF8 xml;
+        // but never share between functions
-
+        TXT_UNICODE unicode;
-        BUFFER *message;
+
-    } ops;
+        // string attributes of the current event log entry
-
+        // valid until another event if fetched
-} WEVT_LOG;
+
-
+        // IMPORTANT:
-WEVT_LOG *wevt_openlog6(void);
+        // EVERY FIELD NEEDS ITS OWN BUFFER!
-void wevt_closelog6(WEVT_LOG *log);
+        // the way facets work, all the field value pointers need to be valid
-
+        // until the entire row closes, so reusing a buffer for the same field
-bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention);
+        // actually copies the same value to all fields using the same buffer.
-
+
-EVT_HANDLE wevt_query(LPCWSTR channel, usec_t seek_to, bool backward);
+        TXT_UTF8 channel;
-void wevt_query_done(WEVT_LOG *log);
+        TXT_UTF8 provider;
-
+        TXT_UTF8 source;
-bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
+        TXT_UTF8 computer;
-
+        TXT_UTF8 user;
-#endif //NETDATA_WINDOWS_EVENTS_QUERY_H
+
+        TXT_UTF8 event;
+        TXT_UTF8 level;
+        TXT_UTF8 keywords;
+        TXT_UTF8 opcode;
+        TXT_UTF8 task;
+        TXT_UTF8 xml;
+    } ops;
+
+    struct {
+        size_t event_count;
+        size_t failed_count;
+    } query_stats;
+
+    struct {
+        size_t queries_count;
+        size_t queries_failed;
+
+        size_t event_count;
+        size_t failed_count;
+    } log_stats;
+
+} WEVT_LOG;
+
+WEVT_LOG *wevt_openlog6(void);
+void wevt_closelog6(WEVT_LOG *log);
+
+bool wevt_channel_retention(WEVT_LOG *log, const wchar_t *channel, EVT_RETENTION *retention);
+
+bool wevt_query(WEVT_LOG *log, LPCWSTR channel, LPCWSTR query, EVT_QUERY_FLAGS direction);
+void wevt_query_done(WEVT_LOG *log);
+
+bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev, bool full);
+
+bool wevt_get_message_unicode(TXT_UNICODE *unicode, EVT_HANDLE hMetadata, EVT_HANDLE bookmark, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags);
+
+struct provider_meta_handle;
+bool wevt_get_event_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE event_handle, TXT_UTF8 *dst);
+bool wevt_get_xml_utf8(WEVT_LOG *log, struct provider_meta_handle *p, EVT_HANDLE event_handle, TXT_UTF8 *dst);
+
+static inline void wevt_variant_cleanup(WEVT_VARIANT *v) {
+    freez(v->data);
+}
+
+static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) {
+    if(required_size < v->size)
+        return;
+
+    wevt_variant_cleanup(v);
+    v->size = compute_new_size(v->size, required_size);
+    v->data = mallocz(v->size);
+}
+
+static inline uint8_t wevt_field_get_uint8(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeByte);
+    return ev->ByteVal;
+}
+
+static inline uint16_t wevt_field_get_uint16(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt16);
+    return ev->UInt16Val;
+}
+
+static inline uint32_t wevt_field_get_uint32(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt32);
+    return ev->UInt32Val;
+}
+
+static inline uint64_t wevt_field_get_uint64(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeUInt64);
+    return ev->UInt64Val;
+}
+
+static inline uint64_t wevt_field_get_uint64_hex(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeHexInt64);
+    return ev->UInt64Val;
+}
+
+static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) {
+    if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
+        wevt_utf8_empty(dst);
+        return false;
+    }
+
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeString);
+    return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1);
+}
+
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
+
+static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst) {
+    if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
+        wevt_utf8_empty(dst);
+        return false;
+    }
+
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid);
+    return wevt_convert_user_id_to_name(ev->SidVal, dst);
+}
+
+static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) {
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeFileTime);
+    return os_windows_ulonglong_to_unix_epoch_ns(ev->FileTimeVal);
+}
+
+static inline bool wevt_GUID_to_ND_UUID(ND_UUID *nd_uuid, const GUID *guid) {
+    if(guid && sizeof(GUID) == sizeof(ND_UUID)) {
+        memcpy(nd_uuid->uuid, guid, sizeof(ND_UUID));
+        return true;
+    }
+    else {
+        *nd_uuid = UUID_ZERO;
+        return false;
+    }
+}
+
+static inline bool wevt_get_uuid_by_type(EVT_VARIANT *ev, ND_UUID *dst) {
+    if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) {
+        wevt_GUID_to_ND_UUID(dst, NULL);
+        return false;
+    }
+
+    fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeGuid);
+    return wevt_GUID_to_ND_UUID(dst, ev->GuidVal);
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/wes/defining-severity-levels
+static inline bool is_valid_publisher_level(uint64_t level, bool strict) {
+    if(strict)
+        // when checking if the name is publisher independent
+        return level >= 16 && level <= 255;
+    else
+        // when checking acceptable values in publisher manifests
+        return level <= 255;
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
+static inline bool is_valid_publisher_opcode(uint64_t opcode, bool strict) {
+    if(strict)
+        // when checking if the name is publisher independent
+        return opcode >= 10 && opcode <= 239;
+    else
+        // when checking acceptable values in publisher manifests
+        return opcode <= 255;
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/wes/defining-tasks-and-opcodes
+static inline bool is_valid_publisher_task(uint64_t task, bool strict) {
+    if(strict)
+        // when checking if the name is publisher independent
+        return task > 0 && task <= 0xFFFF;
+    else
+        // when checking acceptable values in publisher manifests
+        return task <= 0xFFFF;
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/wes/defining-keywords-used-to-classify-types-of-events
+static inline bool is_valid_publisher_keywords(uint64_t keyword, bool strict) {
+    if(strict)
+        // when checking if the name is publisher independent
+        return keyword > 0 && keyword <= 0x0000FFFFFFFFFFFF;
+    else
+        // when checking acceptable values in publisher manifests
+        return true;
+}
+
+#endif //NETDATA_WINDOWS_EVENTS_QUERY_H

src/collectors/windows-events.plugin/windows-events-sid.c

@@ -1,120 +1,118 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#include "windows-events-sid.h"
+#include "windows-events-sid.h"
-#include <sddl.h>
+#include <sddl.h>
-
+
-typedef struct {
+typedef struct {
-    size_t len;
+    size_t len;
-    uint8_t sid[];
+    uint8_t sid[];
-} SID_KEY;
+} SID_KEY;
-
+
-typedef struct {
+typedef struct {
-    const char *user;
+    const char *user;
-    size_t user_len;
+    size_t user_len;
-
+
-    // this needs to be last, because of its variable size
+    // this needs to be last, because of its variable size
-    SID_KEY key;
+    SID_KEY key;
-} SID_VALUE;
+} SID_VALUE;
-
+
-#define SIMPLE_HASHTABLE_NAME _SID
+#define SIMPLE_HASHTABLE_NAME _SID
-#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE
+#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE
-#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY
+#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY
-#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key
+#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key
-#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar
+#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar
-#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
+#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1
-#include "libnetdata/simple_hashtable.h"
+#include "libnetdata/simple_hashtable.h"
-
+
-static struct {
+static struct {
-    SPINLOCK spinlock;
+    SPINLOCK spinlock;
-    bool initialized;
+    struct simple_hashtable_SID hashtable;
-    struct simple_hashtable_SID hashtable;
+} sid_globals = {
-} sid_globals = {
+        .spinlock = NETDATA_SPINLOCK_INITIALIZER,
-        .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+};
-        .hashtable = { 0 },
+
-};
+static inline SID_KEY *sid_value_to_key(SID_VALUE *s) {
-
+    return &s->key;
-static inline SID_KEY *sid_value_to_key(SID_VALUE *s) {
+}
-    return &s->key;
+
-}
+static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) {
-
+    return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0;
-static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) {
+}
-    return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0;
+
-}
+void sid_cache_init(void) {
-
+    simple_hashtable_init_SID(&sid_globals.hashtable, 100);
-static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
+}
-    if(found && found->user) {
+
-        txt_utf8_resize(dst, found->user_len + 1);
+static bool update_user(SID_VALUE *found, TXT_UTF8 *dst) {
-        memcpy(dst->data, found->user, found->user_len + 1);
+    if(found && found->user) {
-        dst->used = found->user_len + 1;
+        txt_utf8_resize(dst, found->user_len + 1, false);
-        return true;
+        memcpy(dst->data, found->user, found->user_len + 1);
-    }
+        dst->used = found->user_len + 1;
-
+        return true;
-    txt_utf8_resize(dst, 1);
+    }
-    dst->data[0] = '\0';
+
-    dst->used = 1;
+    txt_utf8_resize(dst, 1, false);
-    return false;
+    dst->data[0] = '\0';
-}
+    dst->used = 1;
-
+    return false;
-static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
+}
-    static __thread wchar_t account_unicode[256];
+
-    static __thread wchar_t domain_unicode[256];
+static void lookup_user(PSID *sid, TXT_UTF8 *dst) {
-    DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
+    static __thread wchar_t account_unicode[256];
-    DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
+    static __thread wchar_t domain_unicode[256];
-    SID_NAME_USE sid_type;
+    DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]);
-
+    DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]);
-    txt_utf8_resize(dst, 1024);
+    SID_NAME_USE sid_type;
-
+
-    if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
+    txt_utf8_resize(dst, 1024, false);
-        const char *user = account2utf8(account_unicode);
+
-        const char *domain = domain2utf8(domain_unicode);
+    if (LookupAccountSidW(NULL, sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) {
-        dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
+        const char *user = account2utf8(account_unicode);
-    }
+        const char *domain = domain2utf8(domain_unicode);
-    else {
+        dst->used = snprintfz(dst->data, dst->size, "%s\\%s", domain, user) + 1;
-        wchar_t *sid_string = NULL;
+    }
-        if (ConvertSidToStringSidW(sid, &sid_string)) {
+    else {
-            const char *user = account2utf8(sid_string);
+        wchar_t *sid_string = NULL;
-            dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
+        if (ConvertSidToStringSidW(sid, &sid_string)) {
-        }
+            const char *user = account2utf8(sid_string);
-        else
+            dst->used = snprintfz(dst->data, dst->size, "%s", user) + 1;
-            dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
+        }
-    }
+        else
-}
+            dst->used = snprintfz(dst->data, dst->size, "[invalid]") + 1;
-
+    }
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
+}
-    if(!sid || !IsValidSid(sid))
+
-        return update_user(NULL, dst);
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst) {
-
+    if(!sid || !IsValidSid(sid))
-    size_t size = GetLengthSid(sid);
+        return update_user(NULL, dst);
-
+
-    size_t tmp_size = sizeof(SID_VALUE) + size;
+    size_t size = GetLengthSid(sid);
-    size_t tmp_key_size = sizeof(SID_KEY) + size;
+
-    uint8_t buf[tmp_size];
+    size_t tmp_size = sizeof(SID_VALUE) + size;
-    SID_VALUE *tmp = (SID_VALUE *)&buf;
+    size_t tmp_key_size = sizeof(SID_KEY) + size;
-    memcpy(&tmp->key.sid, sid, size);
+    uint8_t buf[tmp_size];
-    tmp->key.len = size;
+    SID_VALUE *tmp = (SID_VALUE *)&buf;
-
+    memcpy(&tmp->key.sid, sid, size);
-    spinlock_lock(&sid_globals.spinlock);
+    tmp->key.len = size;
-    if(!sid_globals.initialized) {
+
-        simple_hashtable_init_SID(&sid_globals.hashtable, 100);
+    spinlock_lock(&sid_globals.spinlock);
-        sid_globals.initialized = true;
+    SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
-    }
+    spinlock_unlock(&sid_globals.spinlock);
-    SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size);
+    if(found) return update_user(found, dst);
-    spinlock_unlock(&sid_globals.spinlock);
+
-    if(found) return update_user(found, dst);
+    // allocate the SID_VALUE
-
+    found = mallocz(tmp_size);
-    // allocate the SID_VALUE
+    memcpy(found, buf, tmp_size);
-    found = mallocz(tmp_size);
+
-    memcpy(found, buf, tmp_size);
+    // lookup the user
-
+    lookup_user(sid, dst);
-    // lookup the user
+    found->user = strdupz(dst->data);
-    lookup_user(sid, dst);
+    found->user_len = dst->used - 1;
-    found->user = strdupz(dst->data);
+
-    found->user_len = dst->used - 1;
+    // add it to the cache
-
+    spinlock_lock(&sid_globals.spinlock);
-    // add it to the cache
+    simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
-    spinlock_lock(&sid_globals.spinlock);
+    spinlock_unlock(&sid_globals.spinlock);
-    simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found);
+
-    spinlock_unlock(&sid_globals.spinlock);
+    return update_user(found, dst);
-
+}
-    return update_user(found, dst);
-}

src/collectors/windows-events.plugin/windows-events-sid.h

@@ -1,11 +1,12 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#ifndef NETDATA_WINDOWS_EVENTS_SID_H
+#ifndef NETDATA_WINDOWS_EVENTS_SID_H
-#define NETDATA_WINDOWS_EVENTS_SID_H
+#define NETDATA_WINDOWS_EVENTS_SID_H
-
+
-#include "windows-events.h"
+#include "windows-events.h"
-
+
-struct wevt_log;
+struct wevt_log;
-bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
+bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst);
-
+void sid_cache_init(void);
-#endif //NETDATA_WINDOWS_EVENTS_SID_H
+
+#endif //NETDATA_WINDOWS_EVENTS_SID_H

src/collectors/windows-events.plugin/windows-events-sources.c

@@ -1,267 +1,317 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#include "windows-events-sources.h"
+#include "windows-events-sources.h"
-
+
-DICTIONARY *wevt_sources = NULL;
+DICTIONARY *wevt_sources = NULL;
-DICTIONARY *used_hashes_registry = NULL;
+DICTIONARY *used_hashes_registry = NULL;
-static usec_t wevt_session = 0;
+static usec_t wevt_session = 0;
-
+
-WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
+WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value) {
-    if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
+    if(strcmp(value, WEVT_SOURCE_ALL_NAME) == 0)
-        return WEVTS_ALL;
+        return WEVTS_ALL;
-
+
-    return WEVTS_NONE;
+    if(strcmp(value, WEVT_SOURCE_ALL_ADMIN_NAME) == 0)
-}
+        return WEVTS_ADMIN;
-
+
-void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+    if(strcmp(value, WEVT_SOURCE_ALL_OPERATIONAL_NAME) == 0)
-    LOGS_QUERY_SOURCE *src = value;
+        return WEVTS_OPERATIONAL;
-    freez((void *)src->fullname);
+
-    string_freez(src->source);
+    if(strcmp(value, WEVT_SOURCE_ALL_ANALYTIC_NAME) == 0)
-
+        return WEVTS_ANALYTIC;
-    src->fullname = NULL;
+
-    src->source = NULL;
+    if(strcmp(value, WEVT_SOURCE_ALL_DEBUG_NAME) == 0)
-}
+        return WEVTS_DEBUG;
-
+
-static bool wevt_sources_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
+    if(strcmp(value, WEVT_SOURCE_ALL_WINDOWS_NAME) == 0)
-    LOGS_QUERY_SOURCE *src_old = old_value;
+        return WEVTS_WINDOWS;
-    LOGS_QUERY_SOURCE *src_new = new_value;
+
-
+    return WEVTS_NONE;
-    bool ret = false;
+}
-    if(src_new->last_scan_monotonic_ut > src_old->last_scan_monotonic_ut) {
+
-        src_old->last_scan_monotonic_ut = src_new->last_scan_monotonic_ut;
+void wevt_sources_del_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
-
+    LOGS_QUERY_SOURCE *src = value;
-        if (src_old->source != src_new->source) {
+    freez((void *)src->fullname);
-            string_freez(src_old->source);
+    string_freez(src->source);
-            src_old->source = src_new->source;
+
-            src_new->source = NULL;
+    src->fullname = NULL;
-        }
+    src->source = NULL;
-        src_old->source_type = src_new->source_type;
+}
-
+
-        src_old->msg_first_ut = src_new->msg_first_ut;
+static bool wevt_sources_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
-        src_old->msg_last_ut = src_new->msg_last_ut;
+    LOGS_QUERY_SOURCE *src_old = old_value;
-        src_old->msg_first_id = src_new->msg_first_id;
+    LOGS_QUERY_SOURCE *src_new = new_value;
-        src_old->msg_last_id = src_new->msg_last_id;
+
-        src_old->entries = src_new->entries;
+    bool ret = false;
-        src_old->size = src_new->size;
+    if(src_new->last_scan_monotonic_ut > src_old->last_scan_monotonic_ut) {
-
+        src_old->last_scan_monotonic_ut = src_new->last_scan_monotonic_ut;
-        ret = true;
+
-    }
+        if (src_old->source != src_new->source) {
-
+            string_freez(src_old->source);
-    freez((void *)src_new->fullname);
+            src_old->source = src_new->source;
-    string_freez(src_new->source);
+            src_new->source = NULL;
-    src_new->fullname = NULL;
+        }
-    src_new->source = NULL;
+        src_old->source_type = src_new->source_type;
-
+
-    return ret;
+        src_old->msg_first_ut = src_new->msg_first_ut;
-}
+        src_old->msg_last_ut = src_new->msg_last_ut;
-
+        src_old->msg_first_id = src_new->msg_first_id;
-void wevt_sources_init(void) {
+        src_old->msg_last_id = src_new->msg_last_id;
-    wevt_session = now_realtime_usec();
+        src_old->entries = src_new->entries;
-
+        src_old->size = src_new->size;
-    used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
+
-
+        ret = true;
-    wevt_sources = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
+    }
-                                              NULL, sizeof(LOGS_QUERY_SOURCE));
+
-
+    freez((void *)src_new->fullname);
-    dictionary_register_delete_callback(wevt_sources, wevt_sources_del_cb, NULL);
+    string_freez(src_new->source);
-    dictionary_register_conflict_callback(wevt_sources, wevt_sources_conflict_cb, NULL);
+    src_new->fullname = NULL;
-}
+    src_new->source = NULL;
-
+
-void buffer_json_wevt_versions(BUFFER *wb __maybe_unused) {
+    return ret;
-    buffer_json_member_add_object(wb, "versions");
+}
-    {
+
-        buffer_json_member_add_uint64(wb, "sources",
+void wevt_sources_init(void) {
-                wevt_session + dictionary_version(wevt_sources));
+    wevt_session = now_realtime_usec();
-    }
+
-    buffer_json_object_close(wb);
+    used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
-}
+
-
+    wevt_sources = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE,
-// --------------------------------------------------------------------------------------------------------------------
+                                              NULL, sizeof(LOGS_QUERY_SOURCE));
-
+
-int wevt_sources_dict_items_backward_compar(const void *a, const void *b) {
+    dictionary_register_delete_callback(wevt_sources, wevt_sources_del_cb, NULL);
-    const DICTIONARY_ITEM **da = (const DICTIONARY_ITEM **)a, **db = (const DICTIONARY_ITEM **)b;
+    dictionary_register_conflict_callback(wevt_sources, wevt_sources_conflict_cb, NULL);
-    LOGS_QUERY_SOURCE *sa = dictionary_acquired_item_value(*da);
+}
-    LOGS_QUERY_SOURCE *sb = dictionary_acquired_item_value(*db);
+
-
+void buffer_json_wevt_versions(BUFFER *wb __maybe_unused) {
-    // compare the last message timestamps
+    buffer_json_member_add_object(wb, "versions");
-    if(sa->msg_last_ut < sb->msg_last_ut)
+    {
-        return 1;
+        buffer_json_member_add_uint64(wb, "sources",
-
+                wevt_session + dictionary_version(wevt_sources));
-    if(sa->msg_last_ut > sb->msg_last_ut)
+    }
-        return -1;
+    buffer_json_object_close(wb);
-
+}
-    // compare the first message timestamps
+
-    if(sa->msg_first_ut < sb->msg_first_ut)
+// --------------------------------------------------------------------------------------------------------------------
-        return 1;
+
-
+int wevt_sources_dict_items_backward_compar(const void *a, const void *b) {
-    if(sa->msg_first_ut > sb->msg_first_ut)
+    const DICTIONARY_ITEM **da = (const DICTIONARY_ITEM **)a, **db = (const DICTIONARY_ITEM **)b;
-        return -1;
+    LOGS_QUERY_SOURCE *sa = dictionary_acquired_item_value(*da);
-
+    LOGS_QUERY_SOURCE *sb = dictionary_acquired_item_value(*db);
-    return 0;
+
-}
+    // compare the last message timestamps
-
+    if(sa->msg_last_ut < sb->msg_last_ut)
-int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
+        return 1;
-    return -wevt_sources_dict_items_backward_compar(a, b);
+
-}
+    if(sa->msg_last_ut > sb->msg_last_ut)
-
+        return -1;
-// --------------------------------------------------------------------------------------------------------------------
+
-
+    // compare the first message timestamps
-struct wevt_source {
+    if(sa->msg_first_ut < sb->msg_first_ut)
-    usec_t first_ut;
+        return 1;
-    usec_t last_ut;
+
-    size_t count;
+    if(sa->msg_first_ut > sb->msg_first_ut)
-    uint64_t size;
+        return -1;
-};
+
-
+    return 0;
-static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
+}
-    const struct wevt_source *s = entry;
+
-    BUFFER *wb = data;
+int wevt_sources_dict_items_forward_compar(const void *a, const void *b) {
-
+    return -wevt_sources_dict_items_backward_compar(a, b);
-    const char *name = dictionary_acquired_item_name(item);
+}
-
+
-    buffer_json_add_array_item_object(wb);
+// --------------------------------------------------------------------------------------------------------------------
-    {
+
-        char size_for_humans[128];
+struct wevt_source {
-        size_snprintf(size_for_humans, sizeof(size_for_humans), s->size, "B", false);
+    usec_t first_ut;
-
+    usec_t last_ut;
-        char duration_for_humans[128];
+    size_t count;
-        duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
+    uint64_t size;
-                          (time_t)((s->last_ut - s->first_ut) / USEC_PER_SEC), "s", true);
+};
-
+
-        char info[1024];
+static int wevt_source_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) {
-        snprintfz(info, sizeof(info), "%zu channels, with a total size of %s, covering %s",
+    const struct wevt_source *s = entry;
-                s->count, size_for_humans, duration_for_humans);
+    BUFFER *wb = data;
-
+
-        buffer_json_member_add_string(wb, "id", name);
+    const char *name = dictionary_acquired_item_name(item);
-        buffer_json_member_add_string(wb, "name", name);
+
-        buffer_json_member_add_string(wb, "pill", size_for_humans);
+    buffer_json_add_array_item_object(wb);
-        buffer_json_member_add_string(wb, "info", info);
+    {
-    }
+        char size_for_humans[128];
-    buffer_json_object_close(wb); // options object
+        size_snprintf(size_for_humans, sizeof(size_for_humans), s->size, "B", false);
-
+
-    return 1;
+        char duration_for_humans[128];
-}
+        duration_snprintf(duration_for_humans, sizeof(duration_for_humans),
-
+                          (time_t)((s->last_ut - s->first_ut) / USEC_PER_SEC), "s", true);
-static bool wevt_source_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) {
+
-    struct wevt_source *old_v = old_value;
+        char info[1024];
-    const struct wevt_source *new_v = new_value;
+        snprintfz(info, sizeof(info), "%zu channel%s, with a total size of %s, covering %s",
-
+                s->count, s->count > 1 ? "s":"", size_for_humans, duration_for_humans);
-    old_v->count += new_v->count;
+
-    old_v->size += new_v->size;
+        buffer_json_member_add_string(wb, "id", name);
-
+        buffer_json_member_add_string(wb, "name", name);
-    if(new_v->first_ut && new_v->first_ut < old_v->first_ut)
+        buffer_json_member_add_string(wb, "pill", size_for_humans);
-        old_v->first_ut = new_v->first_ut;
+        buffer_json_member_add_string(wb, "info", info);
-
+    }
-    if(new_v->last_ut && new_v->last_ut > old_v->last_ut)
+    buffer_json_object_close(wb); // options object
-        old_v->last_ut = new_v->last_ut;
+
-
+    return 1;
-    return false;
+}
-}
+
-
+static bool wevt_source_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) {
-void wevt_sources_to_json_array(BUFFER *wb) {
+    struct wevt_source *old_v = old_value;
-    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE);
+    const struct wevt_source *new_v = new_value;
-    dictionary_register_conflict_callback(dict, wevt_source_merge_sizes, NULL);
+
-
+    old_v->count += new_v->count;
-    struct wevt_source t = { 0 };
+    old_v->size += new_v->size;
-
+
-    LOGS_QUERY_SOURCE *src;
+    if(new_v->first_ut && new_v->first_ut < old_v->first_ut)
-    dfe_start_read(wevt_sources, src) {
+        old_v->first_ut = new_v->first_ut;
-        t.first_ut = src->msg_first_ut;
+
-        t.last_ut = src->msg_last_ut;
+    if(new_v->last_ut && new_v->last_ut > old_v->last_ut)
-        t.count = 1;
+        old_v->last_ut = new_v->last_ut;
-        t.size = src->size;
+
-
+    return false;
-        dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
+}
-
+
-        if(src->source)
+void wevt_sources_to_json_array(BUFFER *wb) {
-            dictionary_set(dict, string2str(src->source), &t, sizeof(t));
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE);
-    }
+    dictionary_register_conflict_callback(dict, wevt_source_merge_sizes, NULL);
-    dfe_done(jf);
+
-
+    struct wevt_source t = { 0 };
-    dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
+
-}
+    LOGS_QUERY_SOURCE *src;
-
+    dfe_start_read(wevt_sources, src) {
-void wevt_sources_scan(void) {
+        t.first_ut = src->msg_first_ut;
-    static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
+        t.last_ut = src->msg_last_ut;
-    LPWSTR channel = NULL;
+        t.count = 1;
-    EVT_HANDLE hChannelEnum = NULL;
+        t.size = src->size;
-
+
-    if(spinlock_trylock(&spinlock)) {
+        dictionary_set(dict, WEVT_SOURCE_ALL_NAME, &t, sizeof(t));
-        const usec_t now_monotonic_ut = now_monotonic_usec();
+
-
+        if(src->source_type & WEVTS_ADMIN)
-        DWORD dwChannelBufferSize = 0;
+            dictionary_set(dict, WEVT_SOURCE_ALL_ADMIN_NAME, &t, sizeof(t));
-        DWORD dwChannelBufferUsed = 0;
+
-        DWORD status = ERROR_SUCCESS;
+        if(src->source_type & WEVTS_OPERATIONAL)
-
+            dictionary_set(dict, WEVT_SOURCE_ALL_OPERATIONAL_NAME, &t, sizeof(t));
-        // Open a handle to enumerate the event channels
+
-        hChannelEnum = EvtOpenChannelEnum(NULL, 0);
+        if(src->source_type & WEVTS_ANALYTIC)
-        if (!hChannelEnum) {
+            dictionary_set(dict, WEVT_SOURCE_ALL_ANALYTIC_NAME, &t, sizeof(t));
-            nd_log(NDLS_COLLECTORS, NDLP_ERR, "WINDOWS EVENTS: EvtOpenChannelEnum() failed with %" PRIu64 "\n",
+
-                    (uint64_t)GetLastError());
+        if(src->source_type & WEVTS_DEBUG)
-            goto cleanup;
+            dictionary_set(dict, WEVT_SOURCE_ALL_DEBUG_NAME, &t, sizeof(t));
-        }
+
-
+        if(src->source_type & WEVTS_WINDOWS)
-        WEVT_LOG *log = wevt_openlog6();
+            dictionary_set(dict, WEVT_SOURCE_ALL_WINDOWS_NAME, &t, sizeof(t));
-        if(!log) goto cleanup;
+
-
+        if(src->source)
-        while (true) {
+            dictionary_set(dict, string2str(src->source), &t, sizeof(t));
-            if (!EvtNextChannelPath(hChannelEnum, dwChannelBufferSize, channel, &dwChannelBufferUsed)) {
+    }
-                status = GetLastError();
+    dfe_done(jf);
-                if (status == ERROR_NO_MORE_ITEMS)
+
-                    break; // No more channels
+    dictionary_sorted_walkthrough_read(dict, wevt_source_to_json_array_cb, wb);
-                else if (status == ERROR_INSUFFICIENT_BUFFER) {
+}
-                    dwChannelBufferSize = dwChannelBufferUsed;
+
-                    freez(channel);
+void wevt_sources_scan(void) {
-                    channel = mallocz(dwChannelBufferSize * sizeof(WCHAR));
+    static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
-                    continue;
+    LPWSTR channel = NULL;
-                } else {
+    EVT_HANDLE hChannelEnum = NULL;
-                    nd_log(NDLS_COLLECTORS, NDLP_ERR,
+
-                           "WINDOWS EVENTS: EvtNextChannelPath() failed\n");
+    if(spinlock_trylock(&spinlock)) {
-                    break;
+        const usec_t now_monotonic_ut = now_monotonic_usec();
-                }
+
-            }
+        DWORD dwChannelBufferSize = 0;
-
+        DWORD dwChannelBufferUsed = 0;
-            EVT_RETENTION retention;
+        DWORD status = ERROR_SUCCESS;
-            if(!wevt_channel_retention(log, channel, &retention))
+
-                continue;
+        // Open a handle to enumerate the event channels
-
+        hChannelEnum = EvtOpenChannelEnum(NULL, 0);
-            const char *name = channel2utf8(channel);
+        if (!hChannelEnum) {
-            const char *fullname = strdupz(name);
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "WINDOWS EVENTS: EvtOpenChannelEnum() failed with %" PRIu64 "\n",
-            char *slash = strchr(name, '/');
+                    (uint64_t)GetLastError());
-            if(slash) *slash = '\0';
+            goto cleanup;
-
+        }
-            LOGS_QUERY_SOURCE src = {
+
-                .entries = retention.entries,
+        WEVT_LOG *log = wevt_openlog6();
-                .fullname = fullname,
+        if(!log) goto cleanup;
-                .fullname_len = strlen(fullname),
+
-                .last_scan_monotonic_ut = now_monotonic_usec(),
+        while (true) {
-                .msg_first_id = retention.first_event.id,
+            if (!EvtNextChannelPath(hChannelEnum, dwChannelBufferSize, channel, &dwChannelBufferUsed)) {
-                .msg_last_id = retention.last_event.id,
+                status = GetLastError();
-                .msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC,
+                if (status == ERROR_NO_MORE_ITEMS)
-                .msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
+                    break; // No more channels
-                .size = retention.size_bytes,
+                else if (status == ERROR_INSUFFICIENT_BUFFER) {
-                .source_type = WEVTS_ALL,
+                    dwChannelBufferSize = dwChannelBufferUsed;
-                .source = string_strdupz(name),
+                    freez(channel);
-            };
+                    channel = mallocz(dwChannelBufferSize * sizeof(WCHAR));
-
+                    continue;
-            dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
+                } else {
-        }
+                    nd_log(NDLS_COLLECTORS, NDLP_ERR,
-
+                           "WINDOWS EVENTS: EvtNextChannelPath() failed\n");
-        wevt_closelog6(log);
+                    break;
-
+                }
-        LOGS_QUERY_SOURCE *src;
+            }
-        dfe_start_write(wevt_sources, src)
+
-        {
+            EVT_RETENTION retention;
-            if(src->last_scan_monotonic_ut < now_monotonic_ut)
+            if(!wevt_channel_retention(log, channel, &retention))
-                dictionary_del(wevt_sources, src->fullname);
+                continue;
-        }
+
-        dfe_done(src);
+            const char *name = channel2utf8(channel);
-        dictionary_garbage_collect(wevt_sources);
+            const char *fullname = strdupz(name);
-
+            char *slash = strchr(name, '/');
-        spinlock_unlock(&spinlock);
+            WEVT_SOURCE_TYPE sources = WEVTS_ALL;
-    }
+            if(slash) {
-
+                *slash++ = '\0';
-cleanup:
+                if(strcasecmp(slash, "Admin") == 0)
-    freez(channel);
+                    sources |= WEVTS_ADMIN;
-    EvtClose(hChannelEnum);
+                if(strcasecmp(slash, "Operational") == 0)
-}
+                    sources |= WEVTS_OPERATIONAL;
+                if(strcasecmp(slash, "Analytic") == 0)
+                    sources |= WEVTS_ANALYTIC;
+                if(strcasecmp(slash, "Debug") == 0)
+                    sources |= WEVTS_DEBUG;
+            }
+
+            if(strcasecmp(name, "Application") == 0)
+                sources |= WEVTS_WINDOWS;
+            if(strcasecmp(name, "Security") == 0)
+                sources |= WEVTS_WINDOWS;
+            if(strcasecmp(name, "Setup") == 0)
+                sources |= WEVTS_WINDOWS;
+            if(strcasecmp(name, "System") == 0)
+                sources |= WEVTS_WINDOWS;
+
+            LOGS_QUERY_SOURCE src = {
+                .entries = retention.entries,
+                .fullname = fullname,
+                .fullname_len = strlen(fullname),
+                .last_scan_monotonic_ut = now_monotonic_usec(),
+                .msg_first_id = retention.first_event.id,
+                .msg_last_id = retention.last_event.id,
+                .msg_first_ut = retention.first_event.created_ns / NSEC_PER_USEC,
+                .msg_last_ut = retention.last_event.created_ns / NSEC_PER_USEC,
+                .size = retention.size_bytes,
+                .source_type = sources,
+                .source = string_strdupz(name),
+            };
+
+            dictionary_set(wevt_sources, src.fullname, &src, sizeof(src));
+        }
+
+        wevt_closelog6(log);
+
+        LOGS_QUERY_SOURCE *src;
+        dfe_start_write(wevt_sources, src)
+        {
+            if(src->last_scan_monotonic_ut < now_monotonic_ut)
+                dictionary_del(wevt_sources, src->fullname);
+        }
+        dfe_done(src);
+        dictionary_garbage_collect(wevt_sources);
+
+        spinlock_unlock(&spinlock);
+    }
+
+cleanup:
+    freez(channel);
+    EvtClose(hChannelEnum);
+}

src/collectors/windows-events.plugin/windows-events-sources.h

@@ -1,45 +1,55 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#ifndef NETDATA_WINDOWS_EVENTS_SOURCES_H
+#ifndef NETDATA_WINDOWS_EVENTS_SOURCES_H
-#define NETDATA_WINDOWS_EVENTS_SOURCES_H
+#define NETDATA_WINDOWS_EVENTS_SOURCES_H
-
+
-#include "windows-events.h"
+#include "windows-events.h"
-
+
-typedef enum {
+typedef enum {
-    WEVTS_NONE               = 0,
+    WEVTS_NONE               = 0,
-    WEVTS_ALL                = (1 << 0),
+    WEVTS_ALL                = (1 << 0),
-} WEVT_SOURCE_TYPE;
+    WEVTS_ADMIN              = (1 << 1),
-
+    WEVTS_OPERATIONAL        = (1 << 2),
-typedef struct {
+    WEVTS_ANALYTIC           = (1 << 3),
-    const char *fullname;
+    WEVTS_DEBUG              = (1 << 4),
-    size_t fullname_len;
+    WEVTS_WINDOWS            = (1 << 5),
-
+} WEVT_SOURCE_TYPE;
-    STRING *source;
+
-    WEVT_SOURCE_TYPE source_type;
+typedef struct {
-    usec_t msg_first_ut;
+    const char *fullname;
-    usec_t msg_last_ut;
+    size_t fullname_len;
-    size_t size;
+
-
+    STRING *source;
-    usec_t last_scan_monotonic_ut;
+    WEVT_SOURCE_TYPE source_type;
-
+    usec_t msg_first_ut;
-    uint64_t msg_first_id;
+    usec_t msg_last_ut;
-    uint64_t msg_last_id;
+    size_t size;
-    uint64_t entries;
+
-} LOGS_QUERY_SOURCE;
+    usec_t last_scan_monotonic_ut;
-
+
-extern DICTIONARY *wevt_sources;
+    uint64_t msg_first_id;
-extern DICTIONARY *used_hashes_registry;
+    uint64_t msg_last_id;
-
+    uint64_t entries;
-#define WEVT_SOURCE_ALL_NAME "All"
+} LOGS_QUERY_SOURCE;
-
+
-void wevt_sources_init(void);
+extern DICTIONARY *wevt_sources;
-void wevt_sources_scan(void);
+extern DICTIONARY *used_hashes_registry;
-void buffer_json_wevt_versions(BUFFER *wb);
+
-
+#define WEVT_SOURCE_ALL_NAME                "All"
-void wevt_sources_to_json_array(BUFFER *wb);
+#define WEVT_SOURCE_ALL_ADMIN_NAME          "All-Admin"
-WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value);
+#define WEVT_SOURCE_ALL_OPERATIONAL_NAME    "All-Operational"
-
+#define WEVT_SOURCE_ALL_ANALYTIC_NAME       "All-Analytic"
-int wevt_sources_dict_items_backward_compar(const void *a, const void *b);
+#define WEVT_SOURCE_ALL_DEBUG_NAME          "All-Debug"
-int wevt_sources_dict_items_forward_compar(const void *a, const void *b);
+#define WEVT_SOURCE_ALL_WINDOWS_NAME        "All-Windows"
-
+
-#endif //NETDATA_WINDOWS_EVENTS_SOURCES_H
+void wevt_sources_init(void);
+void wevt_sources_scan(void);
+void buffer_json_wevt_versions(BUFFER *wb);
+
+void wevt_sources_to_json_array(BUFFER *wb);
+WEVT_SOURCE_TYPE wevt_internal_source_type(const char *value);
+
+int wevt_sources_dict_items_backward_compar(const void *a, const void *b);
+int wevt_sources_dict_items_forward_compar(const void *a, const void *b);
+
+#endif //NETDATA_WINDOWS_EVENTS_SOURCES_H

src/collectors/windows-events.plugin/windows-events-unicode.c

@@ -1,121 +1,139 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#include "windows-events-unicode.h"
+#include "windows-events-unicode.h"
-
+
-inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
+inline void utf82unicode(wchar_t *dst, size_t dst_size, const char *src) {
-    if (src) {
+    if (src) {
-        // Convert from UTF-8 to wide char (UTF-16)
+        // Convert from UTF-8 to wide char (UTF-16)
-        if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
+        if (MultiByteToWideChar(CP_UTF8, 0, src, -1, dst, (int)dst_size) == 0)
-            wcsncpy(dst, L"[failed conv.]", dst_size - 1);
+            wcsncpy(dst, L"[failed conv.]", dst_size - 1);
-    }
+    }
-    else
+    else
-        wcsncpy(dst, L"[null]", dst_size - 1);
+        wcsncpy(dst, L"[null]", dst_size - 1);
-}
+}
-
+
-inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) {
+inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) {
-    if (src) {
+    if (src) {
-        if(WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, (int)dst_size, NULL, NULL) == 0)
+        if(WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, (int)dst_size, NULL, NULL) == 0)
-            strncpyz(dst, "[failed conv.]", dst_size - 1);
+            strncpyz(dst, "[failed conv.]", dst_size - 1);
-    }
+    }
-    else
+    else
-        strncpyz(dst, "[null]", dst_size - 1);
+        strncpyz(dst, "[null]", dst_size - 1);
-}
+}
-
+
-wchar_t *channel2unicode(const char *utf8str) {
+char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) {
-    static __thread wchar_t buffer[1024];
+    int size = WideCharToMultiByte(CP_UTF8, 0, src, -1, NULL, 0, NULL, NULL);
-    utf82unicode(buffer, sizeof(buffer) / sizeof(buffer[0]), utf8str);
+    if (size > 0) {
-    return buffer;
+        char *dst = mallocz(size);
-}
+        WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, size, NULL, NULL);
-
+
-char *channel2utf8(const wchar_t *channel) {
+        if(utf8_len)
-    static __thread char buffer[1024];
+            *utf8_len = size - 1;
-    unicode2utf8(buffer, sizeof(buffer), channel);
+
-    return buffer;
+        return dst;
-}
+    }
-
+
-char *account2utf8(const wchar_t *user) {
+    if(utf8_len)
-    static __thread char buffer[1024];
+        *utf8_len = 0;
-    unicode2utf8(buffer, sizeof(buffer), user);
+
-    return buffer;
+    return NULL;
-}
+}
-
+
-char *domain2utf8(const wchar_t *domain) {
+wchar_t *channel2unicode(const char *utf8str) {
-    static __thread char buffer[1024];
+    static __thread wchar_t buffer[1024];
-    unicode2utf8(buffer, sizeof(buffer), domain);
+    utf82unicode(buffer, sizeof(buffer) / sizeof(buffer[0]), utf8str);
-    return buffer;
+    return buffer;
-}
+}
-
+
-char *query2utf8(const wchar_t *query) {
+char *channel2utf8(const wchar_t *channel) {
-    static __thread char buffer[16384];
+    static __thread char buffer[1024];
-    unicode2utf8(buffer, sizeof(buffer), query);
+    unicode2utf8(buffer, sizeof(buffer), channel);
-    return buffer;
+    return buffer;
-}
+}
-
+
-bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null) {
+char *account2utf8(const wchar_t *user) {
-    if(!src || !src_len_with_null)
+    static __thread char buffer[1024];
-        goto cleanup;
+    unicode2utf8(buffer, sizeof(buffer), user);
-
+    return buffer;
-    // make sure the input is null terminated at the exact point we need it
+}
-    // (otherwise, the output will not be null terminated either)
+
-    fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0));
+char *domain2utf8(const wchar_t *domain) {
-
+    static __thread char buffer[1024];
-    // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size)
+    unicode2utf8(buffer, sizeof(buffer), domain);
-    int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
+    return buffer;
-    if(size <= 0 || !utf8->data) {
+}
-        // we have to set a buffer, or increase it
+
-
+char *query2utf8(const wchar_t *query) {
-        if(utf8->data) {
+    static __thread char buffer[16384];
-            // we need to increase it the buffer size
+    unicode2utf8(buffer, sizeof(buffer), query);
-
+    return buffer;
-            if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+}
-                nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed.");
+
-                goto cleanup;
+bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null) {
-            }
+    if(!src || !src_len_with_null)
-
+        goto cleanup;
-            // we have to find the required buffer size
+
-            size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL);
+    // make sure the input is null terminated at the exact point we need it
-            if(size <= 0)
+    // (otherwise, the output will not be null terminated either)
-                goto cleanup;
+    fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0));
-        }
+
-
+    // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size)
-        // Retry conversion with the new buffer
+    int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
-        txt_utf8_resize(utf8, size);
+    if(size <= 0 || !utf8->data) {
-        size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
+        // we have to set a buffer, or increase it
-        if (size <= 0) {
+
-            nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing.");
+        if(utf8->data) {
-            goto cleanup;
+            // we need to increase it the buffer size
-        }
+
-    }
+            if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
-
+                nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed.");
-    // Make sure it is not zero padded at the end
+                goto cleanup;
-    while(size >= 2 && utf8->data[size - 2] == 0)
+            }
-        size--;
+
-
+            // we have to find the required buffer size
-    utf8->used = (size_t)size;
+            size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL);
-
+            if(size <= 0)
-    internal_fatal(strlen(utf8->data) + 1 != utf8->used,
+                goto cleanup;
-                   "Wrong UTF8 string length");
+        }
-
+
-    return true;
+        // Retry conversion with the new buffer
-
+        txt_utf8_resize(utf8, size, false);
-cleanup:
+        size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, utf8->data, (int)utf8->size, NULL, NULL);
-    txt_utf8_resize(utf8, 128);
+        if (size <= 0) {
-    if(src)
+            nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing.");
-        utf8->used = snprintfz(utf8->data, utf8->size, "[failed conv.]") + 1;
+            goto cleanup;
-    else {
+        }
-        utf8->data[0] = '\0';
+    }
-        utf8->used = 1;
+
-    }
+    // Make sure it is not zero padded at the end
-
+    while(size >= 2 && utf8->data[size - 2] == 0)
-    return false;
+        size--;
-}
+
-
+    utf8->used = (size_t)size;
-bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode) {
+
-    fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size)));
+    internal_fatal(strlen(utf8->data) + 1 != utf8->used,
-    fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size)));
+                   "Wrong UTF8 string length");
-
+
-    // pass the entire unicode size, including the null terminator
+    return true;
-    // so that the resulting utf8 message will be null terminated too.
+
-    return wevt_str_wchar_to_utf8(utf8, unicode->data, (int)unicode->used);
+cleanup:
-}
+    txt_utf8_resize(utf8, 128, false);
-
+    if(src)
+        utf8->used = snprintfz(utf8->data, utf8->size, "[failed conv.]") + 1;
+    else {
+        utf8->data[0] = '\0';
+        utf8->used = 1;
+    }
+
+    return false;
+}
+
+bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode) {
+    fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size)));
+    fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size)));
+
+    // pass the entire unicode size, including the null terminator
+    // so that the resulting utf8 message will be null terminated too.
+    return wevt_str_wchar_to_utf8(utf8, unicode->data, (int)unicode->used);
+}
+

src/collectors/windows-events.plugin/windows-events-unicode.h

@@ -1,72 +1,99 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#ifndef NETDATA_WINDOWS_EVENTS_UNICODE_H
+#ifndef NETDATA_WINDOWS_EVENTS_UNICODE_H
-#define NETDATA_WINDOWS_EVENTS_UNICODE_H
+#define NETDATA_WINDOWS_EVENTS_UNICODE_H
-
+
-#include "libnetdata/libnetdata.h"
+#include "libnetdata/libnetdata.h"
-#include <windows.h>
+#include <windows.h>
-#include <wchar.h>
+#include <wchar.h>
-
+
-typedef struct {
+typedef enum __attribute__((packed)) {
-    char *data;
+    TXT_SOURCE_UNKNOWN = 0,
-    size_t size; // the allocated size of data buffer
+    TXT_SOURCE_PUBLISHER,
-    size_t used;  // the used size of the data buffer (including null terminators, if any)
+    TXT_SOURCE_FIELD_CACHE,
-} TXT_UTF8;
+    TXT_SOURCE_EVENT_LOG,
-
+    TXT_SOURCE_HARDCODED,
-typedef struct {
+
-    wchar_t *data;
+    // terminator
-    size_t size; // the allocated size of data buffer
+    TXT_SOURCE_MAX,
-    size_t used;  // the used size of the data buffer (including null terminators, if any)
+} TXT_SOURCE;
-} TXT_UNICODE;
+
-
+typedef struct {
-static inline size_t compute_new_size(size_t old_size, size_t required_size) {
+    char *data;
-    size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048;
+    size_t size; // the allocated size of data buffer
-    size = (size / 2048) * 2048;
+    size_t used;  // the used size of the data buffer (including null terminators, if any)
-
+    TXT_SOURCE src;
-    if(size < old_size * 2)
+} TXT_UTF8;
-        size = old_size * 2;
+
-
+typedef struct {
-    return size;
+    wchar_t *data;
-}
+    size_t size; // the allocated size of data buffer
-
+    size_t used;  // the used size of the data buffer (including null terminators, if any)
-static inline void txt_utf8_cleanup(TXT_UTF8 *utf8) {
+} TXT_UNICODE;
-    freez(utf8->data);
+
-}
+static inline size_t compute_new_size(size_t old_size, size_t required_size) {
-
+    size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048;
-static inline void txt_utf8_resize(TXT_UTF8 *utf8, size_t required_size) {
+    size = (size / 2048) * 2048;
-    if(required_size < utf8->size)
+
-        return;
+    if(size < old_size * 2)
-
+        size = old_size * 2;
-    txt_utf8_cleanup(utf8);
+
-    utf8->size = compute_new_size(utf8->size, required_size);
+    return size;
-    utf8->data = mallocz(utf8->size);
+}
-}
+
-
+static inline void txt_utf8_cleanup(TXT_UTF8 *utf8) {
-static inline void txt_unicode_cleanup(TXT_UNICODE *unicode) {
+    freez(utf8->data);
-    freez(unicode->data);
+}
-}
+
-
+static inline void txt_utf8_resize(TXT_UTF8 *utf8, size_t required_size, bool keep) {
-static inline void txt_unicode_resize(TXT_UNICODE *unicode, size_t required_size) {
+    if(required_size < utf8->size)
-    if(required_size < unicode->size)
+        return;
-        return;
+
-
+    if(keep) {
-    txt_unicode_cleanup(unicode);
+        size_t new_size = compute_new_size(utf8->size, required_size);
-    unicode->size = compute_new_size(unicode->size, required_size);
+        utf8->data = reallocz(utf8->data, new_size);
-    unicode->data = mallocz(unicode->size * sizeof(wchar_t));
+        utf8->size = new_size;
-}
+    }
-
+    else {
-bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode);
+        txt_utf8_cleanup(utf8);
-bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null);
+        utf8->size = compute_new_size(utf8->size, required_size);
-
+        utf8->data = mallocz(utf8->size);
-void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src);
+    }
-void utf82unicode(wchar_t *dst, size_t dst_size, const char *src);
+}
-
+
-char *account2utf8(const wchar_t *user);
+static inline void txt_unicode_cleanup(TXT_UNICODE *unicode) {
-char *domain2utf8(const wchar_t *domain);
+    freez(unicode->data);
-
+}
-char *channel2utf8(const wchar_t *channel);
+
-wchar_t *channel2unicode(const char *utf8str);
+static inline void txt_unicode_resize(TXT_UNICODE *unicode, size_t required_size) {
-
+    if(required_size < unicode->size)
-char *query2utf8(const wchar_t *query);
+        return;
-
+
-#endif //NETDATA_WINDOWS_EVENTS_UNICODE_H
+    txt_unicode_cleanup(unicode);
+    unicode->size = compute_new_size(unicode->size, required_size);
+    unicode->data = mallocz(unicode->size * sizeof(wchar_t));
+}
+
+bool wevt_str_unicode_to_utf8(TXT_UTF8 *utf8, TXT_UNICODE *unicode);
+bool wevt_str_wchar_to_utf8(TXT_UTF8 *utf8, const wchar_t *src, int src_len_with_null);
+
+void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src);
+void utf82unicode(wchar_t *dst, size_t dst_size, const char *src);
+
+char *account2utf8(const wchar_t *user);
+char *domain2utf8(const wchar_t *domain);
+
+char *channel2utf8(const wchar_t *channel);
+wchar_t *channel2unicode(const char *utf8str);
+
+char *query2utf8(const wchar_t *query);
+
+char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len);
+
+static inline void wevt_utf8_empty(TXT_UTF8 *dst) {
+    txt_utf8_resize(dst, 1, false);
+    dst->data[0] = '\0';
+    dst->used = 1;
+}
+
+#endif //NETDATA_WINDOWS_EVENTS_UNICODE_H

src/collectors/windows-events.plugin/windows-events-xml.c

@@ -32,15 +32,20 @@ const char *parse_value_and_closing_tag(BUFFER *buffer, const char *xml, const c
     const char *start = xml;
     bool has_subnodes = false;
 
+    // const char *tag_start = NULL, *tag_end = NULL;
     while (xml < end) {
         if(*xml == '<') {
             if(xml + 1 < end && *(xml + 1) == '/') {
                 // a closing tag
                 xml += 2;
 
+//                tag_start = xml;
+
                 while(xml < end && *xml != '>')
                     xml++;
 
+//                tag_end = xml;
+
                 if(xml < end && *xml == '>')
                     xml++;
 
@@ -56,7 +61,7 @@ const char *parse_value_and_closing_tag(BUFFER *buffer, const char *xml, const c
                 // an opening tag
                 buffer_fast_strcat(buffer, start, xml - start);
                 xml = start = parse_node(buffer, xml, end, level + 1);
-                while(xml < end && isspace(*xml))
+                while(xml < end && isspace((uint8_t)*xml))
                     xml++;
                 has_subnodes = true;
             }
@@ -101,7 +106,7 @@ const char *parse_field_value(BUFFER *buffer, const char *xml, const char *end)
 
 // Parse a field name and return the next position to parse
 const char *parse_field(BUFFER *buffer, const char *xml, const char *end) {
-    while(isspace(*xml) && xml < end) xml++;
+    while(isspace((uint8_t)*xml) && xml < end) xml++;
 
     const char *start = xml;
 
@@ -136,16 +141,18 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
     buffer_add_xml_indent(buffer, level);
 
     // skip spaces before the tag name
-    while(xml < end && isspace(*xml)) xml++;
+    while(xml < end && isspace((uint8_t)*xml)) xml++;
 
     // Parse the tag name
+//    const char *tag_start = xml, *tag_end = NULL;
     while (xml < end && *xml != '>' && *xml != '/') {
         xml++;
 
-        if(xml < end && isspace(*xml)) {
+        if(xml < end && isspace((uint8_t)*xml)) {
             xml++;
+//            tag_end = xml;
 
-            while(xml < end && isspace(*xml))
+            while(xml < end && isspace((uint8_t)*xml))
                 xml++;
 
             if(xml < end && *xml == '/') {
@@ -168,7 +175,7 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
             else {
                 buffer_fast_strcat(buffer, start, xml - start);
                 xml = start = parse_field(buffer, xml, end);
-                while(xml < end && isspace(*xml))
+                while(xml < end && isspace((uint8_t)*xml))
                     xml++;
             }
         }
@@ -193,12 +200,9 @@ static inline const char *parse_node(BUFFER *buffer, const char *xml, const char
     return append_the_rest(buffer, start, end);
 }
 
-// Main pretty-print XML function
+static inline void buffer_pretty_print_xml_object(BUFFER *buffer, const char *xml, const char *end) {
-void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
-    const char *end = xml + xml_len;
-
     while(xml < end) {
-        while(xml < end && isspace(*xml))
+        while(xml < end && isspace((uint8_t)*xml))
             xml++;
 
         if(xml < end && *xml == '<')
@@ -209,3 +213,132 @@ void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
         }
     }
 }
+
+void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len) {
+    const char *end = xml + xml_len;
+    buffer_pretty_print_xml_object(buffer, xml, end);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+bool buffer_extract_and_print_xml_with_cb(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[],
+                                          void (*cb)(BUFFER *, const char *, const char *, const char *)) {
+    if(!keys || !*keys[0]) {
+        buffer_pretty_print_xml(buffer, xml, xml_len);
+        return true;
+    }
+
+    const char *start = xml, *end = NULL;
+    for(size_t k = 0; keys[k] ; k++) {
+        if(!*keys[k]) continue;
+
+        size_t klen = strlen(keys[k]);
+        char tag_open[klen + 2];
+        tag_open[0] = '<';
+        strcpy(&tag_open[1], keys[k]);
+        tag_open[klen + 1] = '\0';
+
+        const char *new_start = strstr(start, tag_open);
+        if(!new_start)
+            return false;
+
+        start = new_start + klen + 1;
+
+        if(*start != '>' && !isspace((uint8_t)*start))
+            return false;
+
+        if(*start != '>') {
+            start = strchr(start, '>');
+            if(!start) return false;
+        }
+        start++; // skip the >
+
+        char tag_close[klen + 4];
+        tag_close[0] = '<';
+        tag_close[1] = '/';
+        strcpy(&tag_close[2], keys[k]);
+        tag_close[klen + 2] = '>';
+        tag_close[klen + 3] = '\0';
+
+        const char *new_end = strstr(start, tag_close);
+        if(!new_end || (end && new_end > end))
+            return false;
+
+        end = new_end;
+    }
+
+    if(!start || !end || start == end)
+        return false;
+
+    cb(buffer, prefix, start, end);
+    return true;
+}
+
+static void print_xml_cb(BUFFER *buffer, const char *prefix, const char *start, const char *end) {
+    if(prefix)
+        buffer_strcat(buffer, prefix);
+
+    buffer_pretty_print_xml_object(buffer, start, end);
+}
+
+bool buffer_extract_and_print_xml(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]) {
+    return buffer_extract_and_print_xml_with_cb(
+        buffer, xml, xml_len,
+        prefix, keys,
+        print_xml_cb);
+}
+
+static void print_value_cb(BUFFER *buffer, const char *prefix, const char *start, const char *end) {
+    if(prefix)
+        buffer_strcat(buffer, prefix);
+
+    buffer_need_bytes(buffer, end - start + 1);
+
+    char *started = &buffer->buffer[buffer->len];
+    char *d = started;
+    const char *s = start;
+
+    while(s < end && s) {
+        if(*s == '&' && s + 3 < end) {
+            if(*(s + 1) == '#') {
+                if(s + 4 < end && *(s + 2) == '1' && *(s + 4) == ';') {
+                    if (*(s + 3) == '0') {
+                        s += 5;
+                        *d++ = '\n';
+                        continue;
+                    } else if (*(s + 3) == '3') {
+                        s += 5;
+                        // *d++ = '\r';
+                        continue;
+                    }
+                } else if (*(s + 2) == '9' && *(s + 3) == ';') {
+                    s += 4;
+                    *d++ = '\t';
+                    continue;
+                }
+            }
+            else if(s + 3 < end && *(s + 2) == 't' && *(s + 3) == ';') {
+                if(*(s + 1) == 'l') {
+                    s += 4;
+                    *d++ = '<';
+                    continue;
+                }
+                else if(*(s + 1) == 'g') {
+                    s += 4;
+                    *d++ = '>';
+                    continue;
+                }
+            }
+        }
+        *d++ = *s++;
+    }
+    *d = '\0';
+    buffer->len += d - started;
+}
+
+bool buffer_xml_extract_and_print_value(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]) {
+    return buffer_extract_and_print_xml_with_cb(
+        buffer, xml, xml_len,
+        prefix, keys,
+        print_value_cb);
+}

src/collectors/windows-events.plugin/windows-events-xml.h

@@ -6,5 +6,7 @@
 #include "libnetdata/libnetdata.h"
 
 void buffer_pretty_print_xml(BUFFER *buffer, const char *xml, size_t xml_len);
+bool buffer_extract_and_print_xml(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]);
+bool buffer_xml_extract_and_print_value(BUFFER *buffer, const char *xml, size_t xml_len, const char *prefix, const char *keys[]);
 
 #endif //WINDOWS_EVENTS_XML_H

src/collectors/windows-events.plugin/windows-events.h

@@ -1,28 +1,30 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
+// SPDX-License-Identifier: GPL-3.0-or-later
-
+
-#ifndef NETDATA_WINDOWS_EVENTS_H
+#ifndef NETDATA_WINDOWS_EVENTS_H
-#define NETDATA_WINDOWS_EVENTS_H
+#define NETDATA_WINDOWS_EVENTS_H
-
+
-#include "libnetdata/libnetdata.h"
+#include "libnetdata/libnetdata.h"
-#include "collectors/all.h"
+#include "collectors/all.h"
-#include <windows.h>
+#include <windows.h>
-#include <winevt.h>
+#include <winevt.h>
-#include <wchar.h>
+#include <wchar.h>
-
+
-typedef enum {
+typedef enum {
-    WEVT_NO_CHANNEL_MATCHED,
+    WEVT_NO_CHANNEL_MATCHED,
-    WEVT_FAILED_TO_OPEN,
+    WEVT_FAILED_TO_OPEN,
-    WEVT_FAILED_TO_SEEK,
+    WEVT_FAILED_TO_SEEK,
-    WEVT_TIMED_OUT,
+    WEVT_TIMED_OUT,
-    WEVT_OK,
+    WEVT_OK,
-    WEVT_NOT_MODIFIED,
+    WEVT_NOT_MODIFIED,
-    WEVT_CANCELLED,
+    WEVT_CANCELLED,
-} WEVT_QUERY_STATUS;
+} WEVT_QUERY_STATUS;
-
+
-#include "windows-events-unicode.h"
+#include "windows-events-unicode.h"
-#include "windows-events-query.h"
+#include "windows-events-query.h"
-#include "windows-events-sources.h"
+#include "windows-events-sources.h"
-#include "windows-events-sid.h"
+#include "windows-events-sid.h"
-#include "windows-events-xml.h"
+#include "windows-events-xml.h"
-
+#include "windows-events-publishers.h"
-#endif //NETDATA_WINDOWS_EVENTS_H
+#include "windows-events-fields-cache.h"
+
+#endif //NETDATA_WINDOWS_EVENTS_H

src/libnetdata/buffer/buffer.h

@@ -414,6 +414,16 @@ static inline char *print_uint64_hex_reversed(char *dst, uint64_t value) {
 #endif
 }
 
+static inline char *print_uint64_hex_reversed_full(char *dst, uint64_t value) {
+    char *d = dst;
+    for(size_t c = 0; c < sizeof(uint64_t) * 2; c++) {
+        *d++ = hex_digits[value & 0xf];
+        value >>= 4;
+    }
+
+    return d;
+}
+
 static inline char *print_uint64_base64_reversed(char *dst, uint64_t value) {
     char *d = dst;
     do *d++ = base64_digits[value & 63]; while ((value >>= 6));
@@ -509,6 +519,7 @@ static inline size_t print_int64(char *dst, int64_t value) {
     return print_uint64(dst, value) + len;
 }
 
+#define UINT64_MAX_LENGTH (24) // 21 should be enough
 static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
     buffer_need_bytes(wb, 50);
     wb->len += print_uint64(&wb->buffer[wb->len], value);
@@ -521,7 +532,7 @@ static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
     buffer_overflow_check(wb);
 }
 
-#define UINT64_HEX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 2 + 1)
+#define UINT64_HEX_MAX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
 static inline size_t print_uint64_hex(char *dst, uint64_t value) {
     char *d = dst;
 
@@ -534,14 +545,33 @@ static inline size_t print_uint64_hex(char *dst, uint64_t value) {
     return e - dst;
 }
 
+static inline size_t print_uint64_hex_full(char *dst, uint64_t value) {
+    char *d = dst;
+
+    const char *s = HEX_PREFIX;
+    while(*s) *d++ = *s++;
+
+    char *e = print_uint64_hex_reversed_full(d, value);
+    char_array_reverse(d, e - 1);
+    *e = '\0';
+    return e - dst;
+}
+
 static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {
-    buffer_need_bytes(wb, UINT64_HEX_LENGTH);
+    buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);
     wb->len += print_uint64_hex(&wb->buffer[wb->len], value);
     buffer_overflow_check(wb);
 }
 
+static inline void buffer_print_uint64_hex_full(BUFFER *wb, uint64_t value) {
+    buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);
+    wb->len += print_uint64_hex_full(&wb->buffer[wb->len], value);
+    buffer_overflow_check(wb);
+}
+
+#define UINT64_B64_MAX_LENGTH ((sizeof(IEEE754_UINT64_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
 static inline void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) {
-    buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1);
+    buffer_need_bytes(wb, UINT64_B64_MAX_LENGTH);
 
     buffer_fast_strcat(wb, IEEE754_UINT64_B64_PREFIX, sizeof(IEEE754_UINT64_B64_PREFIX) - 1);
 
@@ -580,8 +610,9 @@ static inline void buffer_print_int64_base64(BUFFER *wb, int64_t value) {
     buffer_overflow_check(wb);
 }
 
+#define DOUBLE_MAX_LENGTH (512) // 318 should be enough, including null
 static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) {
-    buffer_need_bytes(wb, 512 + 2);
+    buffer_need_bytes(wb, DOUBLE_MAX_LENGTH);
 
     if(isnan(value) || isinf(value)) {
         buffer_fast_strcat(wb, "null", 4);
@@ -597,8 +628,9 @@ static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value)
     buffer_overflow_check(wb);
 }
 
+#define DOUBLE_HEX_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
 static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) {
-    buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
+    buffer_need_bytes(wb, DOUBLE_HEX_MAX_LENGTH);
 
     uint64_t *ptr = (uint64_t *) (&value);
     buffer_fast_strcat(wb, IEEE754_DOUBLE_HEX_PREFIX, sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1);
@@ -612,8 +644,9 @@ static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE va
     buffer_overflow_check(wb);
 }
 
+#define DOUBLE_B64_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)
 static inline void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) {
-    buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
+    buffer_need_bytes(wb, DOUBLE_B64_MAX_LENGTH);
 
     uint64_t *ptr = (uint64_t *) (&value);
     buffer_fast_strcat(wb, IEEE754_DOUBLE_B64_PREFIX, sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1);

src/libnetdata/facets/facets.c

@@ -256,6 +256,7 @@ struct facets {
     } keys_in_row;
 
     FACET_ROW *base;    // double linked list of the selected facets rows
+    FACET_ROW_BIN_DATA bin_data;
 
     uint32_t items_to_return;
     uint32_t max_items_to_return;
@@ -329,6 +330,10 @@ struct facets {
         struct {
             size_t searches;
         } fts;
+
+        struct {
+            size_t bin_data_inflight;
+        };
     } operations;
 
     struct {
@@ -576,7 +581,7 @@ static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, const char *
             .hash = hash,
             .selected = true,
             .name = name,
-            .name_len = 0,
+            .name_len = name ? strlen(name) : 0,
     };
     FACET_VALUE_ADD_TO_INDEX(k, &tv);
 }
@@ -644,6 +649,35 @@ bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, s
     return (v && v->selected) ? true : false;
 }
 
+bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data) {
+    FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
+    FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
+    if(!k || k->default_selected_for_values)
+        return false;
+
+    size_t selected = 0;
+    for(FACET_VALUE *v = k->values.ll; v ;v = v->next) {
+        if(!v->selected) continue;
+
+        const char *value = v->name;
+        if(!value) {
+            if(used_hashes_registry) {
+                char hash_str[FACET_STRING_HASH_SIZE];
+                facets_hash_to_str(v->hash, hash_str);
+                value = dictionary_get(used_hashes_registry, hash_str);
+            }
+
+            if(!value)
+                return false;
+        }
+
+        if(!cb(facets, selected++, k->name, value, data))
+            return false;
+    }
+
+    return selected > 0;
+}
+
 void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) {
     FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
     FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
@@ -1591,6 +1625,35 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
     return false;
 }
 
+// ----------------------------------------------------------------------------
+// bin_data management
+
+static inline void facets_row_bin_data_cleanup(FACETS *facets, FACET_ROW_BIN_DATA *bin_data) {
+    if(!bin_data->data)
+        return;
+
+    bin_data->cleanup_cb(bin_data->data);
+    *bin_data = FACET_ROW_BIN_DATA_EMPTY;
+
+    fatal_assert(facets->operations.bin_data_inflight > 0);
+    facets->operations.bin_data_inflight--;
+}
+
+void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data) {
+    // in case the caller tries to register bin_data multiple times
+    // for the same row.
+    facets_row_bin_data_cleanup(facets, &facets->bin_data);
+
+    // set the new values
+    facets->bin_data.cleanup_cb = cleanup_cb;
+    facets->bin_data.data = data;
+    facets->operations.bin_data_inflight++;
+}
+
+void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row) {
+    return row->bin_data.data;
+}
+
 // ----------------------------------------------------------------------------
 
 FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
@@ -1633,6 +1696,13 @@ void facets_destroy(FACETS *facets) {
         facets_row_free(facets, r);
     }
 
+    // in case the caller did not call facets_row_finished()
+    // on the last row.
+    facets_row_bin_data_cleanup(facets, &facets->bin_data);
+
+    // make sure we didn't lose any data
+    fatal_assert(facets->operations.bin_data_inflight == 0);
+
     freez(facets->histogram.chart);
     freez(facets);
 }
@@ -1930,7 +2000,9 @@ static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __ma
 // FACET_ROW management
 
 static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
+    facets_row_bin_data_cleanup(facets, &row->bin_data);
     dictionary_destroy(row->dict);
+    row->dict = NULL;
     freez(row);
 }
 
@@ -1940,6 +2012,7 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into
     if(into) {
         row = into;
         facets->operations.rows.reused++;
+        facets_row_bin_data_cleanup(facets, &row->bin_data);
     }
     else {
         row = callocz(1, sizeof(FACET_ROW));
@@ -1950,6 +2023,11 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into
         facets->operations.rows.created++;
     }
 
+    // copy the bin_data to the row
+    // and forget about them in facets
+    row->bin_data = facets->bin_data;
+    facets->bin_data = FACET_ROW_BIN_DATA_EMPTY;
+
     row->severity = facets->current_row.severity;
     row->usec = usec;
 
@@ -2134,6 +2212,8 @@ static void facets_reset_keys_with_value_and_row(FACETS *facets) {
     facets->current_row.keys_matched_by_query_positive = 0;
     facets->current_row.keys_matched_by_query_negative = 0;
     facets->keys_in_row.used = 0;
+
+    facets_row_bin_data_cleanup(facets, &facets->bin_data);
 }
 
 void facets_rows_begin(FACETS *facets) {

src/libnetdata/facets/facets.h

@@ -52,10 +52,18 @@ typedef struct facet_row_key_value {
     BUFFER *wb;
 } FACET_ROW_KEY_VALUE;
 
+typedef struct facet_row_bin_data {
+    void (*cleanup_cb)(void *data);
+    void *data;
+} FACET_ROW_BIN_DATA;
+
+#define FACET_ROW_BIN_DATA_EMPTY (FACET_ROW_BIN_DATA){.data = NULL, .cleanup_cb = NULL}
+
 typedef struct facet_row {
     usec_t usec;
     DICTIONARY *dict;
     FACET_ROW_SEVERITY severity;
+    FACET_ROW_BIN_DATA bin_data;
     struct facet_row *prev, *next;
 } FACET_ROW;
 
@@ -132,4 +140,10 @@ void facets_table_config(BUFFER *wb);
 
 const char *facets_severity_to_string(FACET_ROW_SEVERITY severity);
 
+typedef bool (*facets_foreach_selected_value_in_key_t)(FACETS *facets, size_t id, const char *key, const char *value, void *data);
+bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data);
+
+void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data);
+void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row);
+
 #endif

src/libnetdata/facets/logs_query_status.h

@@ -107,8 +107,15 @@ typedef struct {
     struct {
         usec_t start_ut;
         usec_t stop_ut;
+        usec_t delta_ut;
     } anchor;
 
+    struct {
+        usec_t start_ut;
+        usec_t stop_ut;
+        bool stop_when_full;
+    } query;
+
     usec_t last_modified;
 
     struct lqs_extension c;
@@ -144,6 +151,21 @@ static inline void lqs_log_error(LOGS_QUERY_STATUS *lqs, const char *msg) {
            , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
 }
 
+static inline void lqs_query_timeframe(LOGS_QUERY_STATUS *lqs, usec_t anchor_delta_ut) {
+    lqs->anchor.delta_ut = anchor_delta_ut;
+
+    if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
+        lqs->query.start_ut = (lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.after_ut;
+        lqs->query.stop_ut = ((lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut;
+    }
+    else {
+        lqs->query.start_ut = ((lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut;
+        lqs->query.stop_ut = (lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.after_ut;
+    }
+
+    lqs->query.stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut);
+}
+
 static inline void lqs_function_help(LOGS_QUERY_STATUS *lqs, BUFFER *wb) {
     buffer_reset(wb);
     wb->content_type = CT_TEXT_PLAIN;

src/libnetdata/socket/socket.c

@@ -119,22 +119,17 @@ bool fd_is_socket(int fd) {
     return true;
 }
 
-bool sock_has_output_error(int fd) {
+#ifdef POLLRDHUP
-    if(fd < 0) {
+bool is_socket_closed(int fd) {
-        //internal_error(true, "invalid socket %d", fd);
+    if(fd < 0)
-        return false;
+        return true;
-    }
 
 //    if(!fd_is_socket(fd)) {
 //        //internal_error(true, "fd %d is not a socket", fd);
 //        return false;
 //    }
 
-    short int errors = POLLERR | POLLHUP | POLLNVAL;
+    short int errors = POLLERR | POLLHUP | POLLNVAL | POLLRDHUP;
-
-#ifdef POLLRDHUP
-    errors |= POLLRDHUP;
-#endif
 
     struct pollfd pfd = {
             .fd = fd,
@@ -149,6 +144,31 @@ bool sock_has_output_error(int fd) {
 
     return ((pfd.revents & errors) || !(pfd.revents & POLLOUT));
 }
+#else
+bool is_socket_closed(int fd) {
+    if(fd < 0)
+        return true;
+
+    char buffer;
+    ssize_t result = recv(fd, &buffer, 1, MSG_PEEK | MSG_DONTWAIT);
+    if (result == 0) {
+        // Connection closed
+        return true;
+    }
+    else if (result < 0) {
+        if (errno == EAGAIN || errno == EWOULDBLOCK) {
+            // No data available, but socket is still open
+            return false;
+        } else {
+            // An error occurred
+            return true;
+        }
+    }
+
+    // Data is available, socket is open
+    return false;
+}
+#endif
 
 int sock_setnonblock(int fd) {
     int flags;

src/libnetdata/socket/socket.h

@@ -44,7 +44,7 @@ ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int fla
 int wait_on_socket_or_cancel_with_timeout(NETDATA_SSL *ssl, int fd, int timeout_ms, short int poll_events, short int *revents);
 
 bool fd_is_socket(int fd);
-bool sock_has_output_error(int fd);
+bool is_socket_closed(int fd);
 
 int sock_setnonblock(int fd);
 int sock_delnonblock(int fd);

src/web/api/v3/api_v3_settings.c

@@ -219,7 +219,7 @@ static inline bool is_settings_file_valid(char *file) {
         return false;
 
     while(*s) {
-        if(!isalnum(*s) && *s != '-' && *s != '_')
+        if(!isalnum((uint8_t)*s) && *s != '-' && *s != '_')
             return false;
         s++;
     }

src/web/api/web_api.c

@@ -121,10 +121,13 @@ RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
 bool web_client_interrupt_callback(void *data) {
     struct web_client *w = data;
 
+    bool ret;
     if(w->interrupt.callback)
-        return w->interrupt.callback(w, w->interrupt.callback_data);
+        ret = w->interrupt.callback(w, w->interrupt.callback_data);
+    else
+        ret = is_socket_closed(w->ofd);
 
-    return sock_has_output_error(w->ofd);
+    return ret;
 }
 
 void nd_web_api_init(void) {