From 4aba1a3c07797e80d58f3a1e4ddf5451c77e8b0d Mon Sep 17 00:00:00 2001
From: Costa Tsaousis <costa@netdata.cloud>
Date: Mon, 4 Nov 2024 13:55:02 +0200
Subject: [PATCH] better randomness for heartbeat (#18930)
* add 10ms buffering to streaming senders
* queue for 1ms
* added jitter to heartbeat and sender
* remove jitter and add os_random in libnetdata
* cleanup
* cleanup
* fix random for all operating systems
* simpler random number generator
* make microsleep wait at least 1 nsec
* detect random number generators
* use libnetdata os_random()
* use the buffer versions of the random generators; fallback to using random()
* provide random functions per size
* getrandom() can fail; handle this case too
* fix for wrong params
---
CMakeLists.txt | 20 ++-
packaging/cmake/config.cmake.h.in | 4 +
src/aclk/aclk_util.c | 6 +-
src/libnetdata/clocks/clocks.c | 17 ++-
src/libnetdata/clocks/clocks.h | 2 +
src/libnetdata/os/os.h | 3 +-
src/libnetdata/os/random.c | 214 ++++++++++++++++++++++++++++++
src/libnetdata/os/random.h | 16 +++
src/libnetdata/os/sleep.c | 36 +++++
src/libnetdata/os/sleep.h | 9 ++
src/libnetdata/os/tinysleep.c | 21 ---
src/libnetdata/os/tinysleep.h | 8 --
src/streaming/compression.c | 24 ++--
13 files changed, 326 insertions(+), 54 deletions(-)
create mode 100644 src/libnetdata/os/random.c
create mode 100644 src/libnetdata/os/random.h
create mode 100644 src/libnetdata/os/sleep.c
create mode 100644 src/libnetdata/os/sleep.h
delete mode 100644 src/libnetdata/os/tinysleep.c
delete mode 100644 src/libnetdata/os/tinysleep.h
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7cf2e5e17f..2653aeab62 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -388,6 +388,9 @@ check_function_exists(sched_get_priority_max HAVE_SCHED_GET_PRIORITY_MAX)
check_function_exists(close_range HAVE_CLOSE_RANGE)
check_function_exists(backtrace HAVE_BACKTRACE)
+check_function_exists(arc4random_buf HAVE_ARC4RANDOM_BUF)
+check_function_exists(getrandom HAVE_GETRANDOM)
+
#
# check source compilation
#
@@ -539,6 +542,17 @@ int my_function() {
}
" HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT)
+# Windows MSVCRT random number generator
+# used only when compiling natively (not MSYS/CYGWIN)
+check_c_source_compiles("
+ #define _CRT_RAND_S
+ #include <stdlib.h>
+ int main() {
+ unsigned int x;
+ return rand_s(&x);
+ }
+" HAVE_RAND_S)
+
if(OS_FREEBSD OR OS_MACOS)
set(HAVE_BUILTIN_ATOMICS True)
endif()
@@ -759,8 +773,8 @@ set(LIBNETDATA_FILES
src/libnetdata/os/os-windows-wrappers.h
src/libnetdata/os/get_system_cpus.c
src/libnetdata/os/get_system_cpus.h
- src/libnetdata/os/tinysleep.c
- src/libnetdata/os/tinysleep.h
+ src/libnetdata/os/sleep.c
+ src/libnetdata/os/sleep.h
src/libnetdata/os/uuid_generate.c
src/libnetdata/os/uuid_generate.h
src/libnetdata/os/setenv.c
@@ -862,6 +876,8 @@ set(LIBNETDATA_FILES
src/libnetdata/log/nd_log-to-windows-common.h
src/libnetdata/common.h
src/libnetdata/xxHash/xxhash.h
+ src/libnetdata/os/random.c
+ src/libnetdata/os/random.h
)
if(ENABLE_PLUGIN_EBPF)
diff --git a/packaging/cmake/config.cmake.h.in b/packaging/cmake/config.cmake.h.in
index d322826300..b183bb509e 100644
--- a/packaging/cmake/config.cmake.h.in
+++ b/packaging/cmake/config.cmake.h.in
@@ -69,6 +69,10 @@
#cmakedefine HAVE_DLSYM
#cmakedefine HAVE_LIBCURL
+#cmakedefine HAVE_ARC4RANDOM_BUF
+#cmakedefine HAVE_RAND_S
+#cmakedefine HAVE_GETRANDOM
+
#cmakedefine HAVE_BACKTRACE
#cmakedefine HAVE_CLOSE_RANGE
#cmakedefine HAVE_SCHED_GETSCHEDULER
diff --git a/src/aclk/aclk_util.c b/src/aclk/aclk_util.c
index 16f57fe9ae..d01fa8f2c7 100644
--- a/src/aclk/aclk_util.c
+++ b/src/aclk/aclk_util.c
@@ -342,15 +342,13 @@ unsigned long int aclk_tbeb_delay(int reset, int base, unsigned long int min, un
attempt++;
- if (attempt == 0) {
- srandom(time(NULL));
+ if (attempt == 0)
return 0;
- }
unsigned long int delay = pow(base, attempt - 1);
delay *= MSEC_PER_SEC;
- delay += (random() % (MAX(1000, delay/2)));
+ delay += (os_random32() % (MAX(1000, delay/2)));
if (delay <= min * MSEC_PER_SEC)
return min;
diff --git a/src/libnetdata/clocks/clocks.c b/src/libnetdata/clocks/clocks.c
index 362e87bb2f..c658861984 100644
--- a/src/libnetdata/clocks/clocks.c
+++ b/src/libnetdata/clocks/clocks.c
@@ -260,6 +260,9 @@ void sleep_to_absolute_time(usec_t usec) {
}
#endif
+#define HEARTBEAT_MIN_OFFSET_UT (150 * USEC_PER_MS)
+#define HEARTBEAT_RANDOM_OFFSET_UT (350 * USEC_PER_MS)
+
#define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 20
static SPINLOCK heartbeat_alignment_spinlock = NETDATA_SPINLOCK_INITIALIZER;
static size_t heartbeat_alignment_id = 0;
@@ -306,22 +309,27 @@ void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr,
memcpy(old, current, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE);
}
-static usec_t heartbeat_randomness(usec_t step __maybe_unused, size_t statistics_id) {
+static XXH64_hash_t heartbeat_hash(usec_t step, size_t statistics_id) {
struct {
+ usec_t step;
pid_t pid;
pid_t tid;
usec_t now_ut;
size_t statistics_id;
char tag[ND_THREAD_TAG_MAX + 1];
} key = {
+ .step = step,
.pid = getpid(),
.tid = os_gettid(),
.now_ut = now_realtime_usec(),
.statistics_id = statistics_id,
};
strncpyz(key.tag, nd_thread_tag(), sizeof(key.tag) - 1);
- XXH64_hash_t hash = XXH3_64bits(&key, sizeof(key));
- usec_t offset_ut = (100 * USEC_PER_MS) + (hash % (400 * USEC_PER_MS));
+ return XXH3_64bits(&key, sizeof(key));
+}
+
+static usec_t heartbeat_randomness(XXH64_hash_t hash) {
+ usec_t offset_ut = HEARTBEAT_MIN_OFFSET_UT + (hash % HEARTBEAT_RANDOM_OFFSET_UT);
// Calculate the scheduler tick interval in microseconds
usec_t scheduler_step_ut = USEC_PER_SEC / (usec_t)system_hz;
@@ -345,7 +353,8 @@ inline void heartbeat_init(heartbeat_t *hb, usec_t step) {
hb->step = step;
hb->realtime = 0ULL;
- hb->randomness = heartbeat_randomness(hb->step, hb->statistics_id);
+ hb->hash = heartbeat_hash(hb->step, hb->statistics_id);
+ hb->randomness = heartbeat_randomness(hb->hash);
if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) {
heartbeat_alignment_values[hb->statistics_id].dt = 0;
diff --git a/src/libnetdata/clocks/clocks.h b/src/libnetdata/clocks/clocks.h
index 41458527b0..03860d66db 100644
--- a/src/libnetdata/clocks/clocks.h
+++ b/src/libnetdata/clocks/clocks.h
@@ -4,6 +4,7 @@
#define NETDATA_CLOCKS_H 1
#include "../libnetdata.h"
+#include "libnetdata/os/random.h"
#ifndef HAVE_CLOCK_GETTIME
struct timespec {
@@ -30,6 +31,7 @@ typedef struct heartbeat {
usec_t realtime;
usec_t randomness;
size_t statistics_id;
+ XXH64_hash_t hash;
} heartbeat_t;
/* Linux value is as good as any other */
diff --git a/src/libnetdata/os/os.h b/src/libnetdata/os/os.h
index 77d530b01b..1846afb6d8 100644
--- a/src/libnetdata/os/os.h
+++ b/src/libnetdata/os/os.h
@@ -7,6 +7,7 @@
#include <sys/syscall.h>
#endif
+#include "random.h"
#include "timestamps.h"
#include "setproctitle.h"
#include "close_range.h"
@@ -17,7 +18,7 @@
#include "gettid.h"
#include "get_pid_max.h"
#include "get_system_cpus.h"
-#include "tinysleep.h"
+#include "sleep.h"
#include "uuid_generate.h"
#include "setenv.h"
#include "os-freebsd-wrappers.h"
diff --git a/src/libnetdata/os/random.c b/src/libnetdata/os/random.c
new file mode 100644
index 0000000000..90a876a9b9
--- /dev/null
+++ b/src/libnetdata/os/random.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata/libnetdata.h"
+
+#if !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_RAND_S)
+static SPINLOCK random_lock = NETDATA_SPINLOCK_INITIALIZER;
+static __attribute__((constructor)) void seed_random() {
+ // Use current time and process ID to create a high-entropy seed
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+
+ uint32_t seed = (uint32_t)(tv.tv_sec ^ tv.tv_usec ^ getpid());
+
+ // Seed the random number generator
+ srandom(seed);
+}
+
+#if defined(HAVE_GETRANDOM)
+#include <sys/random.h>
+void getrandom_helper(void *buf, size_t buflen) {
+ ssize_t result;
+ while (buflen > 0) {
+ result = getrandom(buf, buflen, 0);
+ if (result == -1) {
+ if (errno == EINTR) {
+ // Interrupted, retry
+ continue;
+ } else if (errno == EAGAIN) {
+ // Insufficient entropy; wait and retry
+ tinysleep();
+ continue;
+ } else {
+ // Fallback to using random() with a spinlock
+ spinlock_lock(&random_lock);
+ while (buflen > 0) {
+ if (buflen >= sizeof(uint32_t)) {
+ // Generate 4 bytes at a time
+ uint32_t temp = random();
+ memcpy(buf, &temp, sizeof(uint32_t));
+ buf = (uint8_t *)buf + sizeof(uint32_t);
+ buflen -= sizeof(uint32_t);
+ } else if (buflen >= sizeof(uint16_t)) {
+ // Generate 2 bytes at a time
+ uint16_t temp = random();
+ memcpy(buf, &temp, sizeof(uint16_t));
+ buf = (uint8_t *)buf + sizeof(uint16_t);
+ buflen -= sizeof(uint16_t);
+ } else {
+ // Generate remaining bytes
+ uint32_t temp = random();
+ for (size_t i = 0; i < buflen; i++) {
+ ((uint8_t *)buf)[i] = temp & 0xFF;
+ temp >>= 8;
+ }
+ buflen = 0;
+ }
+ }
+ spinlock_unlock(&random_lock);
+ return;
+ }
+ }
+ buf = (uint8_t *)buf + result;
+ buflen -= result;
+ }
+}
+#endif // HAVE_GETRANDOM
+#endif // !HAVE_ARC4RANDOM_BUF && !HAVE_RAND_S
+
+// return a random number 0 to max - 1
+uint64_t os_random(uint64_t max) {
+ if (max <= 1) return 0;
+
+ uint64_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+ if (max <= UINT8_MAX) {
+ uint8_t v;
+ arc4random_buf(&v, sizeof(v));
+ value = v;
+ } else if(max <= UINT16_MAX) {
+ uint16_t v;
+ arc4random_buf(&v, sizeof(v));
+ value = v;
+ } else if (max <= UINT32_MAX) {
+ uint32_t v;
+ arc4random_buf(&v, sizeof(v));
+ value = v;
+ } else
+ arc4random_buf(&value, sizeof(value));
+
+#elif defined(HAVE_RAND_S)
+ if (max <= UINT_MAX) {
+ unsigned int temp;
+ rand_s(&temp);
+ value = temp;
+ } else {
+ unsigned int temp_lo, temp_hi;
+ rand_s(&temp_lo);
+ rand_s(&temp_hi);
+ value = ((uint64_t)temp_hi << 32) + (uint64_t)temp_lo;
+ }
+
+#elif defined(HAVE_GETRANDOM)
+ if (max <= UINT8_MAX) {
+ uint8_t v;
+ getrandom_helper(&v, sizeof(v));
+ value = v;
+ } else if(max <= UINT16_MAX) {
+ uint16_t v;
+ getrandom_helper(&v, sizeof(v));
+ value = v;
+ } else if (max <= UINT32_MAX) {
+ uint32_t v;
+ getrandom_helper(&v, sizeof(v));
+ value = v;
+ } else
+ getrandom_helper(&value, sizeof(value));
+
+#else
+ spinlock_lock(&random_lock);
+ if(max <= INT32_MAX)
+ value = random();
+ else
+ value = ((uint64_t) random() << 33) | ((uint64_t) random() << 2) | (random() & 0x3);
+ spinlock_unlock(&random_lock);
+#endif
+
+ return value % max;
+}
+
+// Generate an 8-bit random number
+uint8_t os_random8(void) {
+ uint8_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+ arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+ getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+ unsigned int temp;
+ rand_s(&temp);
+ value = (uint8_t)temp;
+#else
+ spinlock_lock(&random_lock);
+ value = (uint8_t)random();
+ spinlock_unlock(&random_lock);
+#endif
+
+ return value;
+}
+
+// Generate a 16-bit random number
+uint16_t os_random16(void) {
+ uint16_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+ arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+ getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+ unsigned int temp;
+ rand_s(&temp);
+ value = (uint16_t)temp;
+#else
+ spinlock_lock(&random_lock);
+ value = (uint16_t)random();
+ spinlock_unlock(&random_lock);
+#endif
+
+ return value;
+}
+
+// Generate a 32-bit random number
+uint32_t os_random32(void) {
+ uint32_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+ arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+ getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+ unsigned int temp;
+ rand_s(&temp);
+ value = temp;
+#else
+ spinlock_lock(&random_lock);
+ value = random();
+ spinlock_unlock(&random_lock);
+#endif
+
+ return value;
+}
+
+// Generate a 64-bit random number
+uint64_t os_random64(void) {
+ uint64_t value;
+
+#if defined(HAVE_ARC4RANDOM_BUF)
+ arc4random_buf(&value, sizeof(value));
+#elif defined(HAVE_GETRANDOM)
+ getrandom_helper(&value, sizeof(value));
+#elif defined(HAVE_RAND_S)
+ unsigned int temp_lo, temp_hi;
+ rand_s(&temp_lo);
+ rand_s(&temp_hi);
+ value = ((uint64_t)temp_hi << 32) | (uint64_t)temp_lo;
+#else
+ spinlock_lock(&random_lock);
+ value = ((uint64_t)random() << 33) | ((uint64_t)random() << 2) | (random() & 0x3);
+ spinlock_unlock(&random_lock);
+#endif
+
+ return value;
+}
diff --git a/src/libnetdata/os/random.h b/src/libnetdata/os/random.h
new file mode 100644
index 0000000000..a40699e8e7
--- /dev/null
+++ b/src/libnetdata/os/random.h
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RANDOM_H
+#define NETDATA_RANDOM_H
+
+#include "libnetdata/common.h"
+
+// return a random number 0 to max - 1
+uint64_t os_random(uint64_t max);
+
+uint8_t os_random8(void);
+uint16_t os_random16(void);
+uint32_t os_random32(void);
+uint64_t os_random64(void);
+
+#endif //NETDATA_RANDOM_H
diff --git a/src/libnetdata/os/sleep.c b/src/libnetdata/os/sleep.c
new file mode 100644
index 0000000000..131b47c44d
--- /dev/null
+++ b/src/libnetdata/os/sleep.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#ifdef OS_WINDOWS
+void tinysleep(void) {
+ Sleep(1);
+}
+#else
+void tinysleep(void) {
+ static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
+ nanosleep(&ns, NULL);
+}
+#endif
+
+#ifdef OS_WINDOWS
+void microsleep(usec_t ut) {
+ size_t ms = ut / USEC_PER_MS + ((ut == 0 || (ut % USEC_PER_MS)) ? 1 : 0);
+ Sleep(ms);
+}
+#else
+void microsleep(usec_t ut) {
+ time_t secs = (time_t)(ut / USEC_PER_SEC);
+ nsec_t nsec = (ut % USEC_PER_SEC) * NSEC_PER_USEC + ((ut == 0) ? 1 : 0);
+
+ struct timespec remaining = {
+ .tv_sec = secs,
+ .tv_nsec = nsec,
+ };
+
+ errno_clear();
+ while (nanosleep(&remaining, &remaining) == -1 && errno == EINTR && (remaining.tv_sec || remaining.tv_nsec)) {
+ // Loop continues if interrupted by a signal
+ }
+}
+#endif
diff --git a/src/libnetdata/os/sleep.h b/src/libnetdata/os/sleep.h
new file mode 100644
index 0000000000..3582387623
--- /dev/null
+++ b/src/libnetdata/os/sleep.h
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SLEEP_H
+#define NETDATA_SLEEP_H
+
+void tinysleep(void);
+void microsleep(usec_t ut);
+
+#endif //NETDATA_SLEEP_H
diff --git a/src/libnetdata/os/tinysleep.c b/src/libnetdata/os/tinysleep.c
deleted file mode 100644
index efacf55b88..0000000000
--- a/src/libnetdata/os/tinysleep.c
+++ /dev/null
@@ -1,21 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-#ifdef OS_WINDOWS
-void tinysleep(void) {
- // Improve the system timer resolution to 1 ms
- timeBeginPeriod(1);
-
- // Sleep for the desired duration
- Sleep(1);
-
- // Reset the system timer resolution
- timeEndPeriod(1);
-}
-#else
-void tinysleep(void) {
- static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
- nanosleep(&ns, NULL);
-}
-#endif
diff --git a/src/libnetdata/os/tinysleep.h b/src/libnetdata/os/tinysleep.h
deleted file mode 100644
index 480575a3a7..0000000000
--- a/src/libnetdata/os/tinysleep.h
+++ /dev/null
@@ -1,8 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_TINYSLEEP_H
-#define NETDATA_TINYSLEEP_H
-
-void tinysleep(void);
-
-#endif //NETDATA_TINYSLEEP_H
diff --git a/src/streaming/compression.c b/src/streaming/compression.c
index 8e41c8c337..448f423533 100644
--- a/src/streaming/compression.c
+++ b/src/streaming/compression.c
@@ -395,21 +395,17 @@ size_t rrdpush_decompress(struct decompressor_state *state, const char *compress
// ----------------------------------------------------------------------------
// unit test
-static inline long int my_random (void) {
- return random();
-}
-
void unittest_generate_random_name(char *dst, size_t size) {
if(size < 7)
size = 7;
- size_t len = 5 + my_random() % (size - 6);
+ size_t len = 5 + os_random32() % (size - 6);
for(size_t i = 0; i < len ; i++) {
- if(my_random() % 2 == 0)
- dst[i] = 'A' + my_random() % 26;
+ if(os_random8() % 2 == 0)
+ dst[i] = 'A' + os_random8() % 26;
else
- dst[i] = 'a' + my_random() % 26;
+ dst[i] = 'a' + os_random8() % 26;
}
dst[len] = '\0';
@@ -423,9 +419,9 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
time_t point_end_time_s = now_s;
time_t wall_clock_time_s = now_s;
size_t chart_slot = counter + 1;
- size_t dimensions = 2 + my_random() % 5;
+ size_t dimensions = 2 + os_random8() % 5;
char chart[RRD_ID_LENGTH_MAX + 1] = "name";
- unittest_generate_random_name(chart, 5 + my_random() % 30);
+ unittest_generate_random_name(chart, 5 + os_random8() % 30);
buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
@@ -451,10 +447,10 @@ void unittest_generate_message(BUFFER *wb, time_t now_s, size_t counter) {
for(size_t d = 0; d < dimensions ;d++) {
size_t dim_slot = d + 1;
char dim_id[RRD_ID_LENGTH_MAX + 1] = "dimension";
- unittest_generate_random_name(dim_id, 10 + my_random() % 20);
- int64_t last_collected_value = (my_random() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)my_random();
- NETDATA_DOUBLE value = (my_random() % 2 == 0) ? (NETDATA_DOUBLE)my_random() / ((NETDATA_DOUBLE)my_random() + 1) : (NETDATA_DOUBLE)last_collected_value;
- SN_FLAGS flags = (my_random() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
+ unittest_generate_random_name(dim_id, 10 + os_random8() % 20);
+ int64_t last_collected_value = (os_random8() % 2 == 0) ? (int64_t)(counter + d) : (int64_t)os_random32();
+ NETDATA_DOUBLE value = (os_random8() % 2 == 0) ? (NETDATA_DOUBLE)os_random64() / ((NETDATA_DOUBLE)os_random64() + 1) : (NETDATA_DOUBLE)last_collected_value;
+ SN_FLAGS flags = (os_random16() % 1000 == 0) ? SN_FLAG_NONE : SN_FLAG_NOT_ANOMALOUS;
buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);