better randomness for heartbeat (#18930)

* add 10ms buffering to streaming senders

* queue for 1ms

* added jitter to heartbeat and sender

* remove jitter and add os_random in libnetdata

* cleanup

* cleanup

* fix random for all operating systems

* simpler random number generator

* make microsleep wait at least 1 nsec

* detect random number generators

* use libnetdata os_random()

* use the buffer versions of the random generators; fallback to using random()

* provide random functions per size

* getrandom() can fail; handle this case too

* fix for wrong params

CMakeLists.txt

@@ -388,6 +388,9 @@ check_function_exists(sched_get_priority_max HAVE_SCHED_GET_PRIORITY_MAX)
 check_function_exists(close_range HAVE_CLOSE_RANGE)
 check_function_exists(backtrace HAVE_BACKTRACE)
 
+check_function_exists(arc4random_buf HAVE_ARC4RANDOM_BUF)
+check_function_exists(getrandom HAVE_GETRANDOM)
+
 #
 # check source compilation
 #
@@ -539,6 +542,17 @@ int my_function() {
 }
 " HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT)
 
+# Windows MSVCRT random number generator
+# used only when compiling natively (not MSYS/CYGWIN)
+check_c_source_compiles("
+    #define _CRT_RAND_S
+    #include <stdlib.h>
+    int main() {
+        unsigned int x;
+        return rand_s(&x);
+    }
+" HAVE_RAND_S)
+
 if(OS_FREEBSD OR OS_MACOS)
         set(HAVE_BUILTIN_ATOMICS True)
 endif()
@@ -759,8 +773,8 @@ set(LIBNETDATA_FILES
         src/libnetdata/os/os-windows-wrappers.h
         src/libnetdata/os/get_system_cpus.c
         src/libnetdata/os/get_system_cpus.h
-        src/libnetdata/os/tinysleep.c
-        src/libnetdata/os/tinysleep.h
+        src/libnetdata/os/sleep.c
+        src/libnetdata/os/sleep.h
         src/libnetdata/os/uuid_generate.c
         src/libnetdata/os/uuid_generate.h
         src/libnetdata/os/setenv.c
@@ -862,6 +876,8 @@ set(LIBNETDATA_FILES
         src/libnetdata/log/nd_log-to-windows-common.h
         src/libnetdata/common.h
         src/libnetdata/xxHash/xxhash.h
+        src/libnetdata/os/random.c
+        src/libnetdata/os/random.h
 )
 
 if(ENABLE_PLUGIN_EBPF)

packaging/cmake/config.cmake.h.in

@@ -69,6 +69,10 @@
 #cmakedefine HAVE_DLSYM
 #cmakedefine HAVE_LIBCURL
 
+#cmakedefine HAVE_ARC4RANDOM_BUF
+#cmakedefine HAVE_RAND_S
+#cmakedefine HAVE_GETRANDOM
+
 #cmakedefine HAVE_BACKTRACE
 #cmakedefine HAVE_CLOSE_RANGE
 #cmakedefine HAVE_SCHED_GETSCHEDULER

src/aclk/aclk_util.c

@@ -342,15 +342,13 @@ unsigned long int aclk_tbeb_delay(int reset, int base, unsigned long int min, un
 
     attempt++;
 
-    if (attempt == 0) {
-        srandom(time(NULL));
+    if (attempt == 0)
         return 0;
-    }
 
     unsigned long int delay = pow(base, attempt - 1);
     delay *= MSEC_PER_SEC;
 
-    delay += (random() % (MAX(1000, delay/2)));
+    delay += (os_random32() % (MAX(1000, delay/2)));
 
     if (delay <= min * MSEC_PER_SEC)
         return min;

src/libnetdata/clocks/clocks.c

@@ -260,6 +260,9 @@ void sleep_to_absolute_time(usec_t usec) {
 }
 #endif
 
+#define HEARTBEAT_MIN_OFFSET_UT     (150 * USEC_PER_MS)
+#define HEARTBEAT_RANDOM_OFFSET_UT  (350 * USEC_PER_MS)
+
 #define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 20
 static SPINLOCK heartbeat_alignment_spinlock = NETDATA_SPINLOCK_INITIALIZER;
 static size_t heartbeat_alignment_id = 0;
@@ -306,22 +309,27 @@ void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr,
     memcpy(old, current, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE);
 }
 
-static usec_t heartbeat_randomness(usec_t step __maybe_unused, size_t statistics_id) {
+static XXH64_hash_t heartbeat_hash(usec_t step, size_t statistics_id) {
     struct {
+        usec_t step;
         pid_t pid;
         pid_t tid;
         usec_t now_ut;
         size_t statistics_id;
         char tag[ND_THREAD_TAG_MAX + 1];
     } key = {
+        .step = step,
         .pid = getpid(),
         .tid = os_gettid(),
         .now_ut = now_realtime_usec(),
         .statistics_id = statistics_id,
     };
     strncpyz(key.tag, nd_thread_tag(), sizeof(key.tag) - 1);
-    XXH64_hash_t hash = XXH3_64bits(&key, sizeof(key));
-    usec_t offset_ut = (100 * USEC_PER_MS) + (hash % (400 * USEC_PER_MS));
+    return XXH3_64bits(&key, sizeof(key));
+}
+
+static usec_t heartbeat_randomness(XXH64_hash_t hash) {
+    usec_t offset_ut = HEARTBEAT_MIN_OFFSET_UT + (hash % HEARTBEAT_RANDOM_OFFSET_UT);
 
     // Calculate the scheduler tick interval in microseconds
     usec_t scheduler_step_ut = USEC_PER_SEC / (usec_t)system_hz;
@@ -345,7 +353,8 @@ inline void heartbeat_init(heartbeat_t *hb, usec_t step) {
 
     hb->step = step;
     hb->realtime = 0ULL;
-    hb->randomness = heartbeat_randomness(hb->step, hb->statistics_id);
+    hb->hash = heartbeat_hash(hb->step, hb->statistics_id);
+    hb->randomness = heartbeat_randomness(hb->hash);
 
     if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) {
         heartbeat_alignment_values[hb->statistics_id].dt = 0;

src/libnetdata/clocks/clocks.h

@@ -4,6 +4,7 @@
 #define NETDATA_CLOCKS_H 1
 
 #include "../libnetdata.h"
+#include "libnetdata/os/random.h"
 
 #ifndef HAVE_CLOCK_GETTIME
 struct timespec {
@@ -30,6 +31,7 @@ typedef struct heartbeat {
     usec_t realtime;
     usec_t randomness;
     size_t statistics_id;
+    XXH64_hash_t hash;
 } heartbeat_t;
 
 /* Linux value is as good as any other */

src/libnetdata/os/os.h

@@ -7,6 +7,7 @@
 #include <sys/syscall.h>
 #endif
 
+#include "random.h"
 #include "timestamps.h"
 #include "setproctitle.h"
 #include "close_range.h"
@@ -17,7 +18,7 @@
 #include "gettid.h"
 #include "get_pid_max.h"
 #include "get_system_cpus.h"
-#include "tinysleep.h"
+#include "sleep.h"
 #include "uuid_generate.h"
 #include "setenv.h"
 #include "os-freebsd-wrappers.h"

src/libnetdata/os/random.c

@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+
+#if !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_RAND_S)
+static SPINLOCK random_lock = NETDATA_SPINLOCK_INITIALIZER;
+static __attribute__((constructor)) void seed_random() {
+    // Use current time and process ID to create a high-entropy seed
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+
+    uint32_t seed = (uint32_t)(tv.tv_sec ^ tv.tv_usec ^ getpid());
+
+    // Seed the random number generator
+    srandom(seed);
+}
+
+#if defined(HAVE_GETRANDOM)
+#include <sys/random.h>
+void getrandom_helper(void *buf, size_t buflen) {
+    ssize_t result;
+    while (buflen > 0) {
+        result = getrandom(buf, buflen, 0);
+        if (result == -1) {
+            if (errno == EINTR) {
+                // Interrupted, retry
+                continue;
+            } else if (errno == EAGAIN) {
+                // Insufficient entropy; wait and retry
+                tinysleep();
+                continue;
+            } else {
+                // Fallback to using random() with a spinlock
+                spinlock_lock(&random_lock);
+                while (buflen > 0) {
+                    if (buflen >= sizeof(uint32_t)) {
+                        // Generate 4 bytes at a time
+                        uint32_t temp = random();
+                        memcpy(buf, &temp, sizeof(uint32_t));
+                        buf = (uint8_t *)buf + sizeof(uint32_t);
+                        buflen -= sizeof(uint32_t);
+                    } else if (buflen >= sizeof(uint16_t)) {
+                        // Generate 2 bytes at a time
+                        uint16_t temp = random();
+                        memcpy(buf, &temp, sizeof(uint16_t));
+                        buf = (uint8_t *)buf + sizeof(uint16_t);
+                        buflen -= sizeof(uint16_t);
+                    } else {
+                        // Generate remaining bytes
+                        uint32_t temp = random();
+                        for (size_t i = 0; i < buflen; i++) {
+                            ((uint8_t *)buf)[i] = temp & 0xFF;
+                            temp >>= 8;
+                        }
+                        buflen = 0;
+                    }
+                }
+                spinlock_unlock(&random_lock);
+                return;
+            }
+        }
+        buf = (uint8_t *)buf + result;
+        buflen -= result;
+    }
+}
+#endif // HAVE_GETRANDOM
+#endif // !HAVE_ARC4RANDOM_BUF && !HAVE_RAND_S
+
+// return a random number 0 to max - 1
+uint64_t os_random(uint64_t max) {
+    if (max <= 1) return 0;
+
+    uint64_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+    if (max <= UINT8_MAX) {
+        uint8_t v;
+        arc4random_buf(&v, sizeof(v));
+        value = v;
+    } else if(max <= UINT16_MAX) {
+        uint16_t v;
+        arc4random_buf(&v, sizeof(v));
+        value = v;
+    } else if (max <= UINT32_MAX) {
+        uint32_t v;
+        arc4random_buf(&v, sizeof(v));
+        value = v;
+    } else
+        arc4random_buf(&value, sizeof(value));
+
+#elif defined(HAVE_RAND_S)
+    if (max <= UINT_MAX) {
+        unsigned int temp;
+        rand_s(&temp);
+        value = temp;
+    } else {
+        unsigned int temp_lo, temp_hi;
+        rand_s(&temp_lo);
+        rand_s(&temp_hi);
+        value = ((uint64_t)temp_hi << 32) + (uint64_t)temp_lo;
+    }
+
+#elif defined(HAVE_GETRANDOM)
+    if (max <= UINT8_MAX) {
+        uint8_t v;
+        getrandom_helper(&v, sizeof(v));
+        value = v;
+    } else if(max <= UINT16_MAX) {
+        uint16_t v;
+        getrandom_helper(&v, sizeof(v));
+        value = v;
+    } else if (max <= UINT32_MAX) {
+        uint32_t v;
+        getrandom_helper(&v, sizeof(v));
+        value = v;
+    } else
+        getrandom_helper(&value, sizeof(value));
+
+#else
+    spinlock_lock(&random_lock);
+    if(max <= INT32_MAX)
+        value = random();
+    else
+        value = ((uint64_t) random() << 33) | ((uint64_t) random() << 2) | (random() & 0x3);
+    spinlock_unlock(&random_lock);
+#endif
+
+    return value % max;
+}
+
+// Generate an 8-bit random number
+uint8_t os_random8(void) {
+    uint8_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+    arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+    getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+    unsigned int temp;
+    rand_s(&temp);
+    value = (uint8_t)temp;
+#else
+    spinlock_lock(&random_lock);
+    value = (uint8_t)random();
+    spinlock_unlock(&random_lock);
+#endif
+
+    return value;
+}
+
+// Generate a 16-bit random number
+uint16_t os_random16(void) {
+    uint16_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+    arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+    getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+    unsigned int temp;
+    rand_s(&temp);
+    value = (uint16_t)temp;
+#else
+    spinlock_lock(&random_lock);
+    value = (uint16_t)random();
+    spinlock_unlock(&random_lock);
+#endif
+
+    return value;
+}
+
+// Generate a 32-bit random number
+uint32_t os_random32(void) {
+    uint32_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+    arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+    getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+    unsigned int temp;
+    rand_s(&temp);
+    value = temp;
+#else
+    spinlock_lock(&random_lock);
+    value = random();
+    spinlock_unlock(&random_lock);
+#endif
+
+    return value;
+}
+
+// Generate a 64-bit random number
+uint64_t os_random64(void) {
+    uint64_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+    arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+    getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+    unsigned int temp_lo, temp_hi;
+    rand_s(&temp_lo);
+    rand_s(&temp_hi);
+    value = ((uint64_t)temp_hi << 32) | (uint64_t)temp_lo;
+#else
+    spinlock_lock(&random_lock);
+    value = ((uint64_t)random() << 33) | ((uint64_t)random() << 2) | (random() & 0x3);
+    spinlock_unlock(&random_lock);
+#endif
+
+    return value;
+}

src/libnetdata/os/random.h

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RANDOM_H
+#define NETDATA_RANDOM_H
+
+#include "libnetdata/common.h"
+
+// return a random number 0 to max - 1
+uint64_t os_random(uint64_t max);
+
+uint8_t os_random8(void);
+uint16_t os_random16(void);
+uint32_t os_random32(void);
+uint64_t os_random64(void);
+
+#endif //NETDATA_RANDOM_H

src/libnetdata/os/sleep.c

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#ifdef OS_WINDOWS
+void tinysleep(void) {
+    Sleep(1);
+}
+#else
+void tinysleep(void) {
+    static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
+    nanosleep(&ns, NULL);
+}
+#endif
+
+#ifdef OS_WINDOWS
+void microsleep(usec_t ut) {
+    size_t ms = ut / USEC_PER_MS + ((ut == 0 || (ut % USEC_PER_MS)) ? 1 : 0);
+    Sleep(ms);
+}
+#else
+void microsleep(usec_t ut) {
+    time_t secs = (time_t)(ut / USEC_PER_SEC);
+    nsec_t nsec = (ut % USEC_PER_SEC) * NSEC_PER_USEC + ((ut == 0) ? 1 : 0);
+
+    struct timespec remaining = {
+        .tv_sec = secs,
+        .tv_nsec = nsec,
+    };
+
+    errno_clear();
+    while (nanosleep(&remaining, &remaining) == -1 && errno == EINTR && (remaining.tv_sec || remaining.tv_nsec)) {
+        // Loop continues if interrupted by a signal
+    }
+}
+#endif

src/libnetdata/os/sleep.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SLEEP_H
+#define NETDATA_SLEEP_H
+
+void tinysleep(void);
+void microsleep(usec_t ut);
+
+#endif //NETDATA_SLEEP_H

src/libnetdata/os/tinysleep.c

@@ -1,21 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-#ifdef OS_WINDOWS
-void tinysleep(void) {
-    // Improve the system timer resolution to 1 ms
-    timeBeginPeriod(1);
-
-    // Sleep for the desired duration
-    Sleep(1);
-
-    // Reset the system timer resolution
-    timeEndPeriod(1);
-}
-#else
-void tinysleep(void) {
-    static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
-    nanosleep(&ns, NULL);
-}
-#endif

src/libnetdata/os/tinysleep.h

@@ -1,8 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_TINYSLEEP_H
-#define NETDATA_TINYSLEEP_H
-
-void tinysleep(void);
-
-#endif //NETDATA_TINYSLEEP_H

src/streaming/compression.c

@@ -395,21 +395,17 @@ size_t rrdpush_decompress(struct decompressor_state *state, const char *compress
 // ----------------------------------------------------------------------------
 // unit test
 
-static inline long int my_random (void) {
-    return random();
-}
-
 void unittest_generate_random_name(char *dst, size_t size) {
     if(size < 7)
         size = 7;
 
-    size_t len = 5 + my_random() % (size - 6);
+    size_t len = 5 + os_random32() % (size - 6);
 
     for(size_t i = 0; i < len ; i++) {
-        if(my_random() % 2 == 0)
-            dst[i] = 'A' + my_random() % 26;
+        if(os_random8() % 2 == 0)
+            dst[i] = 'A' + os_random8() % 26;
         else
-            dst[i] = 'a' + my_random() % 26;
+            dst[i] = 'a' + os_random8() % 26;
     }
 
     dst[len] = '\0';
@@ -423,9 +419,9 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
     time_t point_end_time_s = now_s;
     time_t wall_clock_time_s = now_s;
     size_t chart_slot = counter + 1;
-    size_t dimensions = 2 + my_random() % 5;
+    size_t dimensions = 2 + os_random8() % 5;
     char chart[RRD_ID_LENGTH_MAX + 1] = "name";
-    unittest_generate_random_name(chart, 5 + my_random() % 30);
+    unittest_generate_random_name(chart, 5 + os_random8() % 30);
 
     buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
 
@@ -451,10 +447,10 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
     for(size_t d = 0; d < dimensions ;d++) {
         size_t dim_slot = d + 1;
         char dim_id[RRD_ID_LENGTH_MAX + 1] = "dimension";
-        unittest_generate_random_name(dim_id, 10 + my_random() % 20);
-        int64_t last_collected_value = (my_random() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)my_random();
-        NETDATA_DOUBLE value = (my_random() % 2 == 0) ? (NETDATA_DOUBLE)my_random() / ((NETDATA_DOUBLE)my_random() + 1) : (NETDATA_DOUBLE)last_collected_value;
-        SN_FLAGS flags = (my_random() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
+        unittest_generate_random_name(dim_id, 10 + os_random8() % 20);
+        int64_t last_collected_value = (os_random8() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)os_random32();
+        NETDATA_DOUBLE value = (os_random8() % 2 == 0) ? (NETDATA_DOUBLE)os_random64() / ((NETDATA_DOUBLE)os_random64() + 1) : (NETDATA_DOUBLE)last_collected_value;
+        SN_FLAGS flags = (os_random16() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
 
         buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);