0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-26 13:54:48 +00:00
netdata_netdata/libnetdata/inlined.h
Costa Tsaousis 3e508c8f95
New logging layer ()
* cleanup of logging - wip

* first working iteration

* add errno annotator

* replace old logging functions with netdata_logger()

* cleanup

* update error_limit

* fix remanining error_limit references

* work on fatal()

* started working on structured logs

* full cleanup

* default logging to files; fix all plugins initialization

* fix formatting of numbers

* cleanup and reorg

* fix coverity issues

* cleanup obsolete code

* fix formatting of numbers

* fix log rotation

* fix for older systems

* add detection of systemd journal via stderr

* finished on access.log

* remove left-over transport

* do not add empty fields to the logs

* journal get compact uuids; X-Transaction-ID header is added in web responses

* allow compiling on systems without memfd sealing

* added libnetdata/uuid directory

* move datetime formatters to libnetdata

* add missing files

* link the makefiles in libnetdata

* added uuid_parse_flexi() to parse UUIDs with and without hyphens; the web server now read X-Transaction-ID and uses it for functions and web responses

* added stream receiver, sender, proc plugin and pluginsd log stack

* iso8601 advanced usage; line_splitter module in libnetdata; code cleanup

* add message ids to streaming inbound and outbound connections

* cleanup line_splitter between lines to avoid logging garbage; when killing children, kill them with SIGABRT if internal checks is enabled

* send SIGABRT to external plugins only if we are not shutting down

* fix cross cleanup in pluginsd parser

* fatal when there is a stack error in logs

* compile netdata with -fexceptions

* do not kill external plugins with SIGABRT

* metasync info logs to debug level

* added severity to logs

* added json output; added options per log output; added documentation; fixed issues mentioned

* allow memfd only on linux

* moved journal low level functions to journal.c/h

* move health logs to daemon.log with proper priorities

* fixed a couple of bugs; health log in journal

* updated docs

* systemd-cat-native command to push structured logs to journal from the command line

* fix makefiles

* restored NETDATA_LOG_SEVERITY_LEVEL

* fix makefiles

* systemd-cat-native can also work as the logger of Netdata scripts

* do not require a socket to systemd-journal to log-as-netdata

* alarm notify logs in native format

* properly compare log ids

* fatals log alerts; alarm-notify.sh working

* fix overflow warning

* alarm-notify.sh now logs the request (command line)

* anotate external plugins logs with the function cmd they run

* added context, component and type to alarm-notify.sh; shell sanitization removes control character and characters that may be expanded by bash

* reformatted alarm-notify logs

* unify cgroup-network-helper.sh

* added quotes around params

* charts.d.plugin switched logging to journal native

* quotes for logfmt

* unify the status codes of streaming receivers and senders

* alarm-notify: dont log anything, if there is nothing to do

* all external plugins log to stderr when running outside netdata; alarm-notify now shows an error when notifications menthod are needed but are not available

* migrate cgroup-name.sh to new logging

* systemd-cat-native now supports messages with newlines

* socket.c logs use priority

* cleanup log field types

* inherit the systemd set INVOCATION_ID if found

* allow systemd-cat-native to send messages to a systemd-journal-remote URL

* log2journal command that can convert structured logs to journal export format

* various fixes and documentation of log2journal

* updated log2journal docs

* updated log2journal docs

* updated documentation of fields

* allow compiling without libcurl

* do not use socket as format string

* added version information to newly added tools

* updated documentation and help messages

* fix the namespace socket path

* print errno with error

* do not timeout

* updated docs

* updated docs

* updated docs

* log2journal updated docs and params

* when talking to a remote journal, systemd-cat-native batches the messages

* enable lz4 compression for systemd-cat-native when sending messages to a systemd-journal-remote

* Revert "enable lz4 compression for systemd-cat-native when sending messages to a systemd-journal-remote"

This reverts commit b079d53c11.

* note about uncompressed traffic

* log2journal: code reorg and cleanup to make modular

* finished rewriting log2journal

* more comments

* rewriting rules support

* increased limits

* updated docs

* updated docs

* fix old log call

* use journal only when stderr is connected to journal

* update netdata.spec for libcurl, libpcre2 and log2journal

* pcre2-devel

* do not require pcre2 in centos < 8, amazonlinux < 2023, open suse

* log2journal only on systems pcre2 is available

* ignore log2journal in .gitignore

* avoid log2journal on centos 7, amazonlinux 2 and opensuse

* add pcre2-8 to static build

* undo last commit

* Bundle to static

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Add build deps for deb packages

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Add dependencies; build from source

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Test build for amazon linux and centos expect to fail for suse

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* fix minor oversight

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

* Reorg code

* Add the install from source (deps) as a TODO
* Not enable the build on suse ecosystem

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>

---------

Signed-off-by: Tasos Katsoulas <tasos@netdata.cloud>
Co-authored-by: Tasos Katsoulas <tasos@netdata.cloud>
2023-11-22 10:27:25 +02:00

600 lines
14 KiB
C

// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_INLINED_H
#define NETDATA_INLINED_H 1
#include "libnetdata.h"
#ifdef KERNEL_32BIT
typedef uint32_t kernel_uint_t;
#define str2kernel_uint_t(string) str2uint32_t(string, NULL)
#define KERNEL_UINT_FORMAT "%u"
#else
typedef uint64_t kernel_uint_t;
#define str2kernel_uint_t(string) str2uint64_t(string, NULL)
#define KERNEL_UINT_FORMAT "%" PRIu64
#endif
#define str2pid_t(string) str2uint32_t(string, NULL)
// for faster execution, allow the compiler to inline
// these functions that are called thousands of times per second
static inline uint32_t djb2_hash32(const char* name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 5381;
while (*s)
hash = ((hash << 5) + hash) + (uint32_t) *s++; // hash * 33 + char
return hash;
}
static inline uint32_t pluginsd_parser_hash32(const char *name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 0;
while (*s) {
hash <<= 5;
hash += *s++ - ' ';
}
return hash;
}
// https://stackoverflow.com/a/107657
static inline uint32_t larson_hash32(const char *name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 0;
while (*s)
hash = hash * 101 + (uint32_t) *s++;
return hash;
}
// http://isthe.com/chongo/tech/comp/fnv/
static inline uint32_t fnv1_hash32(const char *name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 0x811c9dc5;
while (*s) {
hash *= 0x01000193; // 16777619
hash ^= (uint32_t) *s++;
}
return hash;
}
// http://isthe.com/chongo/tech/comp/fnv/
static inline uint32_t fnv1a_hash32(const char *name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 0x811c9dc5;
while (*s) {
hash ^= (uint32_t) *s++;
hash *= 0x01000193; // 16777619
}
return hash;
}
static inline uint32_t fnv1a_uhash32(const char *name) {
unsigned char *s = (unsigned char *) name;
uint32_t hash = 0x811c9dc5, c;
while ((c = *s++)) {
if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
hash ^= c;
hash *= 0x01000193; // 16777619
}
return hash;
}
#define simple_hash(s) fnv1a_hash32(s)
#define simple_uhash(s) fnv1a_uhash32(s)
static uint32_t murmur32(uint32_t k) __attribute__((const));
static inline uint32_t murmur32(uint32_t k) {
k ^= k >> 16;
k *= 0x85ebca6b;
k ^= k >> 13;
k *= 0xc2b2ae35;
k ^= k >> 16;
return k;
}
static uint64_t murmur64(uint64_t k) __attribute__((const));
static inline uint64_t murmur64(uint64_t k) {
k ^= k >> 33;
k *= 0xff51afd7ed558ccdUL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53UL;
k ^= k >> 33;
return k;
}
static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const));
static inline size_t indexing_partition(Word_t ptr, Word_t modulo) {
#ifdef ENV64BIT
uint64_t hash = murmur64(ptr);
return hash % modulo;
#else
uint32_t hash = murmur32(ptr);
return hash % modulo;
#endif
}
static inline unsigned int str2u(const char *s) {
unsigned int n = 0;
while(*s >= '0' && *s <= '9')
n = n * 10 + (*s++ - '0');
return n;
}
static inline int str2i(const char *s) {
if(unlikely(*s == '-')) {
s++;
return -(int) str2u(s);
}
else {
if(unlikely(*s == '+')) s++;
return (int) str2u(s);
}
}
static inline unsigned long str2ul(const char *s) {
unsigned long n = 0;
while(*s >= '0' && *s <= '9')
n = n * 10 + (*s++ - '0');
return n;
}
static inline long str2l(const char *s) {
if(unlikely(*s == '-')) {
s++;
return -(long) str2ul(s);
}
else {
if(unlikely(*s == '+')) s++;
return (long) str2ul(s);
}
}
static inline uint32_t str2uint32_t(const char *s, char **endptr) {
uint32_t n = 0;
while(*s >= '0' && *s <= '9')
n = n * 10 + (*s++ - '0');
if(unlikely(endptr))
*endptr = (char *)s;
return n;
}
static inline uint64_t str2uint64_t(const char *s, char **endptr) {
uint64_t n = 0;
#ifdef ENV32BIT
unsigned long n32 = 0;
while (*s >= '0' && *s <= '9' && n32 < (ULONG_MAX / 10))
n32 = n32 * 10 + (*s++ - '0');
n = n32;
#endif
while(*s >= '0' && *s <= '9')
n = n * 10 + (*s++ - '0');
if(unlikely(endptr))
*endptr = (char *)s;
return n;
}
static inline unsigned long long int str2ull(const char *s, char **endptr) {
return str2uint64_t(s, endptr);
}
static inline long long str2ll(const char *s, char **endptr) {
if(unlikely(*s == '-')) {
s++;
return -(long long) str2uint64_t(s, endptr);
}
else {
if(unlikely(*s == '+')) s++;
return (long long) str2uint64_t(s, endptr);
}
}
static inline uint64_t str2uint64_hex(const char *src, char **endptr) {
uint64_t num = 0;
const unsigned char *s = (const unsigned char *)src;
unsigned char c;
while ((c = hex_value_from_ascii[toupper(*s)]) != 255) {
num = (num << 4) | c;
s++;
}
if(endptr)
*endptr = (char *)s;
return num;
}
static inline uint64_t str2uint64_base64(const char *src, char **endptr) {
uint64_t num = 0;
const unsigned char *s = (const unsigned char *)src;
unsigned char c;
while ((c = base64_value_from_ascii[*s]) != 255) {
num = (num << 6) | c;
s++;
}
if(endptr)
*endptr = (char *)s;
return num;
}
static inline NETDATA_DOUBLE str2ndd_parse_double_decimal_digits_internal(const char *src, int *digits) {
const char *s = src;
NETDATA_DOUBLE n = 0.0;
while(*s >= '0' && *s <= '9') {
// this works for both 32-bit and 64-bit systems
unsigned long ni = 0;
unsigned exponent = 0;
while (*s >= '0' && *s <= '9' && ni < (ULONG_MAX / 10)) {
ni = (ni * 10) + (*s++ - '0');
exponent++;
}
n = n * powndd(10.0, exponent) + (NETDATA_DOUBLE)ni;
}
*digits = (int)(s - src);
return n;
}
static inline NETDATA_DOUBLE str2ndd(const char *src, char **endptr) {
const char *s = src;
NETDATA_DOUBLE sign = 1.0;
NETDATA_DOUBLE result;
int integral_digits = 0;
NETDATA_DOUBLE fractional = 0.0;
int fractional_digits = 0;
NETDATA_DOUBLE exponent = 0.0;
int exponent_digits = 0;
switch(*s) {
case '-':
s++;
sign = -1.0;
break;
case '+':
s++;
break;
case 'n':
if(s[1] == 'a' && s[2] == 'n') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
if(s[1] == 'u' && s[2] == 'l' && s[3] == 'l') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
break;
case 'i':
if(s[1] == 'n' && s[2] == 'f') {
if(endptr) *endptr = (char *)&s[3];
return INFINITY;
}
break;
default:
break;
}
result = str2ndd_parse_double_decimal_digits_internal(s, &integral_digits);
s += integral_digits;
if(unlikely(*s == '.')) {
s++;
fractional = str2ndd_parse_double_decimal_digits_internal(s, &fractional_digits);
s += fractional_digits;
}
if (unlikely(*s == 'e' || *s == 'E')) {
const char *e_ptr = s;
s++;
int exponent_sign = 1;
if (*s == '-') {
exponent_sign = -1;
s++;
}
else if(*s == '+')
s++;
exponent = str2ndd_parse_double_decimal_digits_internal(s, &exponent_digits);
if(unlikely(!exponent_digits)) {
exponent = 0;
s = e_ptr;
}
else {
s += exponent_digits;
exponent *= exponent_sign;
}
}
if(unlikely(endptr))
*endptr = (char *)s;
if (unlikely(exponent_digits))
result *= powndd(10.0, exponent);
if (unlikely(fractional_digits))
result += fractional / powndd(10.0, fractional_digits) * (exponent_digits ? powndd(10.0, exponent) : 1.0);
return sign * result;
}
static inline unsigned long long str2ull_encoded(const char *s) {
if(*s == IEEE754_UINT64_B64_PREFIX[0])
return str2uint64_base64(s + sizeof(IEEE754_UINT64_B64_PREFIX) - 1, NULL);
if(s[0] == HEX_PREFIX[0] && s[1] == HEX_PREFIX[1])
return str2uint64_hex(s + 2, NULL);
return str2uint64_t(s, NULL);
}
static inline long long str2ll_encoded(const char *s) {
if(*s == '-')
return -(long long) str2ull_encoded(&s[1]);
else
return (long long) str2ull_encoded(s);
}
static inline NETDATA_DOUBLE str2ndd_encoded(const char *src, char **endptr) {
if (*src == IEEE754_DOUBLE_B64_PREFIX[0]) {
// double parsing from base64
uint64_t n = str2uint64_base64(src + sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1, endptr);
NETDATA_DOUBLE *ptr = (NETDATA_DOUBLE *) (&n);
return *ptr;
}
if (*src == IEEE754_DOUBLE_HEX_PREFIX[0]) {
// double parsing from hex
uint64_t n = str2uint64_hex(src + sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1, endptr);
NETDATA_DOUBLE *ptr = (NETDATA_DOUBLE *) (&n);
return *ptr;
}
double sign = 1.0;
if(*src == '-') {
sign = -1.0;
src++;
}
if(unlikely(*src == IEEE754_UINT64_B64_PREFIX[0]))
return (NETDATA_DOUBLE) str2uint64_base64(src + sizeof(IEEE754_UINT64_B64_PREFIX) - 1, endptr) * sign;
if(unlikely(*src == HEX_PREFIX[0] && src[1] == HEX_PREFIX[1]))
return (NETDATA_DOUBLE) str2uint64_hex(src + sizeof(HEX_PREFIX) - 1, endptr) * sign;
return str2ndd(src, endptr) * sign;
}
static inline char *strncpyz(char *dst, const char *src, size_t dst_size_minus_1) {
char *p = dst;
while (*src && dst_size_minus_1--)
*dst++ = *src++;
*dst = '\0';
return p;
}
static inline void sanitize_json_string(char *dst, const char *src, size_t dst_size) {
while (*src != '\0' && dst_size > 1) {
if (*src < 0x1F) {
*dst++ = '_';
src++;
dst_size--;
}
else if (*src == '\\' || *src == '\"') {
*dst++ = '\\';
*dst++ = *src++;
dst_size -= 2;
}
else {
*dst++ = *src++;
dst_size--;
}
}
*dst = '\0';
}
static inline bool sanitize_command_argument_string(char *dst, const char *src, size_t dst_size) {
if(dst_size)
*dst = '\0';
// skip leading dashes
while (*src == '-')
src++;
while (*src != '\0') {
if (dst_size < 1)
return false;
if (iscntrl(*src) || *src == '$') {
// remove control characters and characters that are expanded by bash
*dst++ = '_';
dst_size--;
}
else if (*src == '\'' || *src == '`') {
// escape single quotes
if (dst_size < 4)
return false;
dst[0] = '\''; dst[1] = '\\'; dst[2] = '\''; dst[3] = '\'';
dst += 4;
dst_size -= 4;
}
else {
*dst++ = *src;
dst_size--;
}
src++;
}
// make sure we have space to terminate the string
if (dst_size == 0)
return false;
*dst = '\0';
return true;
}
static inline int read_file(const char *filename, char *buffer, size_t size) {
if(unlikely(!size)) return 3;
int fd = open(filename, O_RDONLY, 0666);
if(unlikely(fd == -1)) {
buffer[0] = '\0';
return 1;
}
ssize_t r = read(fd, buffer, size);
if(unlikely(r == -1)) {
buffer[0] = '\0';
close(fd);
return 2;
}
buffer[r] = '\0';
close(fd);
return 0;
}
static inline int read_single_number_file(const char *filename, unsigned long long *result) {
char buffer[30 + 1];
int ret = read_file(filename, buffer, 30);
if(unlikely(ret)) {
*result = 0;
return ret;
}
buffer[30] = '\0';
*result = str2ull(buffer, NULL);
return 0;
}
static inline int read_single_signed_number_file(const char *filename, long long *result) {
char buffer[30 + 1];
int ret = read_file(filename, buffer, 30);
if(unlikely(ret)) {
*result = 0;
return ret;
}
buffer[30] = '\0';
*result = atoll(buffer);
return 0;
}
static inline int read_single_base64_or_hex_number_file(const char *filename, unsigned long long *result) {
char buffer[30 + 1];
int ret = read_file(filename, buffer, 30);
if(unlikely(ret)) {
*result = 0;
return ret;
}
buffer[30] = '\0';
if(likely(buffer[0])){
*result = str2ull_encoded(buffer);
return 0;
}
else {
*result = 0;
return -1;
}
}
static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) {
char *p = (char *)"";
while (p && !p[0] && *ptr) p = strsep(ptr, s);
return (p);
}
// remove leading and trailing spaces; may return NULL
static inline char *trim(char *s) {
// skip leading spaces
while (*s && isspace(*s)) s++;
if (!*s) return NULL;
// skip tailing spaces
// this way is way faster. Writes only one NUL char.
ssize_t l = (ssize_t)strlen(s);
if (--l >= 0) {
char *p = s + l;
while (p > s && isspace(*p)) p--;
*++p = '\0';
}
if (!*s) return NULL;
return s;
}
// like trim(), but also remove duplicate spaces inside the string; may return NULL
static inline char *trim_all(char *buffer) {
char *d = buffer, *s = buffer;
// skip spaces
while(isspace(*s)) s++;
while(*s) {
// copy the non-space part
while(*s && !isspace(*s)) *d++ = *s++;
// add a space if we have to
if(*s && isspace(*s)) {
*d++ = ' ';
s++;
}
// skip spaces
while(isspace(*s)) s++;
}
*d = '\0';
if(d > buffer) {
d--;
if(isspace(*d)) *d = '\0';
}
if(!buffer[0]) return NULL;
return buffer;
}
#endif //NETDATA_INLINED_H