diff --git a/Makefile.am b/Makefile.am
index c008eb2de1..c22e174840 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -154,6 +154,8 @@ LIBNETDATA_FILES = \
libnetdata/dictionary/dictionary.h \
libnetdata/eval/eval.c \
libnetdata/eval/eval.h \
+ libnetdata/facets/facets.c \
+ libnetdata/facets/facets.h \
libnetdata/gorilla/gorilla.h \
libnetdata/gorilla/gorilla.cc \
libnetdata/inlined.h \
@@ -302,6 +304,11 @@ FREEIPMI_PLUGIN_FILES = \
$(LIBNETDATA_FILES) \
$(NULL)
+SYSTEMD_JOURNAL_PLUGIN_FILES = \
+ collectors/systemd-journal.plugin/systemd-journal.c \
+ $(LIBNETDATA_FILES) \
+ $(NULL)
+
CUPS_PLUGIN_FILES = \
collectors/cups.plugin/cups_plugin.c \
$(LIBNETDATA_FILES) \
@@ -1232,6 +1239,15 @@ if ENABLE_PLUGIN_FREEIPMI
$(NULL)
endif
+if ENABLE_PLUGIN_SYSTEMD_JOURNAL
+ plugins_PROGRAMS += systemd-journal.plugin
+ systemd_journal_plugin_SOURCES = $(SYSTEMD_JOURNAL_PLUGIN_FILES)
+ systemd_journal_plugin_LDADD = \
+ $(NETDATA_COMMON_LIBS) \
+ $(OPTIONAL_SYSTEMD_LIBS) \
+ $(NULL)
+endif
+
if ENABLE_PLUGIN_EBPF
plugins_PROGRAMS += ebpf.plugin
ebpf_plugin_SOURCES = $(EBPF_PLUGIN_FILES)
diff --git a/claim/claim.c b/claim/claim.c
index c2b7b39562..d81440d2a1 100644
--- a/claim/claim.c
+++ b/claim/claim.c
@@ -356,7 +356,7 @@ int api_v2_claim(struct web_client *w, char *url) {
BUFFER *wb = w->response.data;
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
time_t now_s = now_realtime_sec();
CLOUD_STATUS status = buffer_json_cloud_status(wb, now_s);
@@ -462,7 +462,7 @@ int api_v2_claim(struct web_client *w, char *url) {
// our status may have changed
// refresh the status in our output
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
now_s = now_realtime_sec();
buffer_json_cloud_status(wb, now_s);
diff --git a/collectors/Makefile.am b/collectors/Makefile.am
index 2aec3dd3e8..d477e5b80e 100644
--- a/collectors/Makefile.am
+++ b/collectors/Makefile.am
@@ -25,6 +25,7 @@ SUBDIRS = \
statsd.plugin \
ebpf.plugin \
tc.plugin \
+ systemd-journal.plugin \
$(NULL)
usercustompluginsconfigdir=$(configdir)/custom-plugins.d
diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c
index 631980dffd..f258a30ad0 100644
--- a/collectors/apps.plugin/apps_plugin.c
+++ b/collectors/apps.plugin/apps_plugin.c
@@ -13,7 +13,7 @@
#define APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION "Detailed information on the currently running processes."
#define APPS_PLUGIN_FUNCTIONS() do { \
- fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" 10 \"%s\"\n", APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
+ fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
} while(0)
@@ -4572,7 +4572,7 @@ static int check_capabilities() {
}
#endif
-netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
+static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
#define PROCESS_FILTER_CATEGORY "category:"
#define PROCESS_FILTER_USER "user:"
@@ -4625,15 +4625,6 @@ static void get_MemTotal(void) {
#endif
}
-static void apps_plugin_function_error(const char *transaction, int code, const char *msg) {
- char buffer[PLUGINSD_LINE_MAX + 1];
- json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
-
- pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
- fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
- pluginsd_function_result_end_to_stdout();
-}
-
static void apps_plugin_function_processes_help(const char *transaction) {
pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
fprintf(stdout, "%s",
@@ -4681,7 +4672,7 @@ static void apps_plugin_function_processes_help(const char *transaction) {
buffer_json_add_array_item_double(wb, _tmp); \
} while(0)
-static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
+static void function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
struct pid_stat *p;
char *words[PLUGINSD_MAX_WORDS] = { NULL };
@@ -4702,21 +4693,21 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {
category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);
if(!category) {
- apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
+ pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
return;
}
}
else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
if(!user) {
- apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
+ pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
return;
}
}
else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {
group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);
if(!group) {
- apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
+ pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
return;
}
}
@@ -4742,7 +4733,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
else {
char msg[PLUGINSD_LINE_MAX];
snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
- apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
+ pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
return;
}
}
@@ -4755,7 +4746,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
unsigned int io_divisor = 1024 * RATES_DETAIL;
BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
buffer_json_member_add_string(wb, "type", "table");
buffer_json_member_add_time_t(wb, "update_every", update_every);
@@ -5232,7 +5223,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "Uptime", "Uptime in seconds", RRDF_FIELD_TYPE_DURATION,
- RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION, 2,
+ RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION_S, 2,
"seconds", Uptime_max, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MAX,
RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -5532,9 +5523,9 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
pluginsd_function_result_end_to_stdout();
}
-bool apps_plugin_exit = false;
+static bool apps_plugin_exit = false;
-void *reader_main(void *arg __maybe_unused) {
+static void *reader_main(void *arg __maybe_unused) {
char buffer[PLUGINSD_LINE_MAX + 1];
char *s = NULL;
@@ -5566,9 +5557,9 @@ void *reader_main(void *arg __maybe_unused) {
netdata_mutex_lock(&mutex);
if(strncmp(function, "processes", strlen("processes")) == 0)
- apps_plugin_function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
+ function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
else
- apps_plugin_function_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
+ pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
fflush(stdout);
netdata_mutex_unlock(&mutex);
@@ -5696,6 +5687,8 @@ int main(int argc, char **argv) {
netdata_thread_create(&reader_thread, "APPS_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
netdata_mutex_lock(&mutex);
+ APPS_PLUGIN_FUNCTIONS();
+
usec_t step = update_every * USEC_PER_SEC;
global_iterations_counter = 1;
heartbeat_t hb;
diff --git a/collectors/ebpf.plugin/ebpf_functions.c b/collectors/ebpf.plugin/ebpf_functions.c
index cc26044c46..8f0244cde4 100644
--- a/collectors/ebpf.plugin/ebpf_functions.c
+++ b/collectors/ebpf.plugin/ebpf_functions.c
@@ -206,7 +206,7 @@ static void ebpf_function_thread_manipulation(const char *transaction,
time_t expires = now_realtime_sec() + em->update_every;
BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
buffer_json_member_add_string(wb, "type", "table");
buffer_json_member_add_time_t(wb, "update_every", em->update_every);
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
index 1e183c2dc8..4988b50719 100644
--- a/collectors/plugins.d/plugins_d.h
+++ b/collectors/plugins.d/plugins_d.h
@@ -99,8 +99,6 @@ void pluginsd_process_thread_cleanup(void *ptr);
size_t pluginsd_initialize_plugin_directories();
-
-
#define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
buffer_sprintf(wb \
, PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
@@ -125,4 +123,13 @@ size_t pluginsd_initialize_plugin_directories();
#define pluginsd_function_result_end_to_stdout() \
fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
+static inline void pluginsd_function_json_error(const char *transaction, int code, const char *msg) {
+ char buffer[PLUGINSD_LINE_MAX + 1];
+ json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
+
+ pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
+ fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
+ pluginsd_function_result_end_to_stdout();
+}
+
#endif /* NETDATA_PLUGINS_D_H */
diff --git a/collectors/systemd-journal.plugin/Makefile.am b/collectors/systemd-journal.plugin/Makefile.am
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/collectors/systemd-journal.plugin/README.md b/collectors/systemd-journal.plugin/README.md
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/collectors/systemd-journal.plugin/systemd-journal.c b/collectors/systemd-journal.plugin/systemd-journal.c
new file mode 100644
index 0000000000..95954355df
--- /dev/null
+++ b/collectors/systemd-journal.plugin/systemd-journal.c
@@ -0,0 +1,578 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+/*
+ * netdata systemd-journal.plugin
+ * Copyright (C) 2023 Netdata Inc.
+ * GPL v3+
+ */
+
+// TODO - 1) MARKDOC
+
+#include "collectors/all.h"
+#include "libnetdata/libnetdata.h"
+#include "libnetdata/required_dummies.h"
+
+#include <systemd/sd-journal.h>
+#include <syslog.h>
+
+#define FACET_MAX_VALUE_LENGTH 8192
+
+#define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
+#define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal"
+#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 30
+#define SYSTEMD_JOURNAL_MAX_PARAMS 100
+#define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (3 * 3600)
+#define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200
+
+#define JOURNAL_PARAMETER_HELP "help"
+#define JOURNAL_PARAMETER_AFTER "after"
+#define JOURNAL_PARAMETER_BEFORE "before"
+#define JOURNAL_PARAMETER_ANCHOR "anchor"
+#define JOURNAL_PARAMETER_LAST "last"
+#define JOURNAL_PARAMETER_QUERY "query"
+
+#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
+#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL
+#define SYSTEMD_KEYS_INCLUDED_IN_FACETS \
+ "_TRANSPORT" \
+ "|SYSLOG_IDENTIFIER" \
+ "|SYSLOG_FACILITY" \
+ "|PRIORITY" \
+ "|_HOSTNAME" \
+ "|_RUNTIME_SCOPE" \
+ "|_PID" \
+ "|_UID" \
+ "|_GID" \
+ "|_SYSTEMD_UNIT" \
+ "|_SYSTEMD_SLICE" \
+ "|_SYSTEMD_USER_SLICE" \
+ "|_COMM" \
+ "|_EXE" \
+ "|_SYSTEMD_CGROUP" \
+ "|_SYSTEMD_USER_UNIT" \
+ "|USER_UNIT" \
+ "|UNIT" \
+ ""
+
+static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
+static bool plugin_should_exit = false;
+
+DICTIONARY *uids = NULL;
+DICTIONARY *gids = NULL;
+
+
+// ----------------------------------------------------------------------------
+
+int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t stop_monotonic_ut) {
+ sd_journal *j;
+ int r;
+
+ // Open the system journal for reading
+ r = sd_journal_open(&j, SD_JOURNAL_ALL_NAMESPACES);
+ if (r < 0)
+ return HTTP_RESP_INTERNAL_SERVER_ERROR;
+
+ facets_rows_begin(facets);
+
+ bool timed_out = false;
+ size_t row_counter = 0;
+ sd_journal_seek_realtime_usec(j, before_ut);
+ SD_JOURNAL_FOREACH_BACKWARDS(j) {
+ row_counter++;
+
+ uint64_t msg_ut;
+ sd_journal_get_realtime_usec(j, &msg_ut);
+ if (msg_ut < after_ut)
+ break;
+
+ const void *data;
+ size_t length;
+ SD_JOURNAL_FOREACH_DATA(j, data, length) {
+ const char *key = data;
+ const char *equal = strchr(key, '=');
+ if(unlikely(!equal))
+ continue;
+
+ const char *value = ++equal;
+ size_t key_length = value - key; // including '\0'
+
+ char key_copy[key_length];
+ memcpy(key_copy, key, key_length - 1);
+ key_copy[key_length - 1] = '\0';
+
+ size_t value_length = length - key_length; // without '\0'
+ facets_add_key_value_length(facets, key_copy, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);
+ }
+
+ facets_row_finished(facets, msg_ut);
+
+ if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {
+ timed_out = true;
+ break;
+ }
+ }
+
+ sd_journal_close(j);
+
+ buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
+ buffer_json_member_add_boolean(wb, "partial", timed_out);
+ buffer_json_member_add_string(wb, "type", "table");
+ buffer_json_member_add_time_t(wb, "update_every", 1);
+ buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
+
+ facets_report(facets, wb);
+
+ buffer_json_member_add_time_t(wb, "expires", now_realtime_sec());
+ buffer_json_finalize(wb);
+
+ return HTTP_RESP_OK;
+}
+
+static void systemd_journal_function_help(const char *transaction) {
+ pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
+ fprintf(stdout,
+ "%s / %s\n"
+ "\n"
+ "%s\n"
+ "\n"
+ "The following filters are supported:\n"
+ "\n"
+ " help\n"
+ " Shows this help message.\n"
+ "\n"
+ " before:TIMESTAMP\n"
+ " Absolute or relative (to now) timestamp in seconds, to start the query.\n"
+ " The query is always executed from the most recent to the oldest log entry.\n"
+ " If not given the default is: now.\n"
+ "\n"
+ " after:TIMESTAMP\n"
+ " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n"
+ " If not given, the default is %d.\n"
+ "\n"
+ " last:ITEMS\n"
+ " The number of items to return.\n"
+ " The default is %d.\n"
+ "\n"
+ " anchor:NUMBER\n"
+ " The `timestamp` of the item last received, to return log entries after that.\n"
+ " If not given, the query will return the top `ITEMS` from the most recent.\n"
+ "\n"
+ " facet_id:value_id1,value_id2,value_id3,...\n"
+ " Apply filters to the query, based on the facet IDs returned.\n"
+ " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n"
+ "\n"
+ "Filters can be combined. Each filter can be given only one time.\n"
+ , program_name
+ , SYSTEMD_JOURNAL_FUNCTION_NAME
+ , SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION
+ , -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION
+ , SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY
+ );
+ pluginsd_function_result_end_to_stdout();
+}
+
+static const char *syslog_facility_to_name(int facility) {
+ switch (facility) {
+ case LOG_FAC(LOG_KERN): return "kern";
+ case LOG_FAC(LOG_USER): return "user";
+ case LOG_FAC(LOG_MAIL): return "mail";
+ case LOG_FAC(LOG_DAEMON): return "daemon";
+ case LOG_FAC(LOG_AUTH): return "auth";
+ case LOG_FAC(LOG_SYSLOG): return "syslog";
+ case LOG_FAC(LOG_LPR): return "lpr";
+ case LOG_FAC(LOG_NEWS): return "news";
+ case LOG_FAC(LOG_UUCP): return "uucp";
+ case LOG_FAC(LOG_CRON): return "cron";
+ case LOG_FAC(LOG_AUTHPRIV): return "authpriv";
+ case LOG_FAC(LOG_FTP): return "ftp";
+ case LOG_FAC(LOG_LOCAL0): return "local0";
+ case LOG_FAC(LOG_LOCAL1): return "local1";
+ case LOG_FAC(LOG_LOCAL2): return "local2";
+ case LOG_FAC(LOG_LOCAL3): return "local3";
+ case LOG_FAC(LOG_LOCAL4): return "local4";
+ case LOG_FAC(LOG_LOCAL5): return "local5";
+ case LOG_FAC(LOG_LOCAL6): return "local6";
+ case LOG_FAC(LOG_LOCAL7): return "local7";
+ default: return NULL;
+ }
+}
+
+static const char *syslog_priority_to_name(int priority) {
+ switch (priority) {
+ case LOG_ALERT: return "alert";
+ case LOG_CRIT: return "critical";
+ case LOG_DEBUG: return "debug";
+ case LOG_EMERG: return "panic";
+ case LOG_ERR: return "error";
+ case LOG_INFO: return "info";
+ case LOG_NOTICE: return "notice";
+ case LOG_WARNING: return "warning";
+ default: return NULL;
+ }
+}
+
+static char *uid_to_username(uid_t uid, char *buffer, size_t buffer_size) {
+ struct passwd pw, *result;
+ char tmp[1024 + 1];
+
+ if (getpwuid_r(uid, &pw, tmp, 1024, &result) != 0 || result == NULL)
+ return NULL;
+
+ strncpy(buffer, pw.pw_name, buffer_size - 1);
+ buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
+ return buffer;
+}
+
+static char *gid_to_groupname(gid_t gid, char* buffer, size_t buffer_size) {
+ struct group grp, *result;
+ char tmp[1024 + 1];
+
+ if (getgrgid_r(gid, &grp, tmp, 1024, &result) != 0 || result == NULL)
+ return NULL;
+
+ strncpy(buffer, grp.gr_name, buffer_size - 1);
+ buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
+ return buffer;
+}
+
+static void systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
+ const char *v = buffer_tostring(wb);
+ if(*v && isdigit(*v)) {
+ int facility = str2i(buffer_tostring(wb));
+ const char *name = syslog_facility_to_name(facility);
+ if (name) {
+ buffer_flush(wb);
+ buffer_json_add_array_item_string(wb, name);
+ }
+ }
+}
+
+static void systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
+ const char *v = buffer_tostring(wb);
+ if(*v && isdigit(*v)) {
+ int priority = str2i(buffer_tostring(wb));
+ const char *name = syslog_priority_to_name(priority);
+ if (name) {
+ buffer_flush(wb);
+ buffer_json_add_array_item_string(wb, name);
+ }
+ }
+}
+
+static void systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
+ DICTIONARY *cache = data;
+ const char *v = buffer_tostring(wb);
+ if(*v && isdigit(*v)) {
+ const char *sv = dictionary_get(cache, v);
+ if(!sv) {
+ char buf[1024 + 1];
+ int uid = str2i(buffer_tostring(wb));
+ const char *name = uid_to_username(uid, buf, 1024);
+ if (!name)
+ name = v;
+
+ sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
+ }
+
+ buffer_flush(wb);
+ buffer_strcat(wb, sv);
+ }
+}
+
+static void systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
+ DICTIONARY *cache = data;
+ const char *v = buffer_tostring(wb);
+ if(*v && isdigit(*v)) {
+ const char *sv = dictionary_get(cache, v);
+ if(!sv) {
+ char buf[1024 + 1];
+ int gid = str2i(buffer_tostring(wb));
+ const char *name = gid_to_groupname(gid, buf, 1024);
+ if (!name)
+ name = v;
+
+ sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
+ }
+
+ buffer_flush(wb);
+ buffer_strcat(wb, sv);
+ }
+}
+
+static void systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {
+ FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER");
+ FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID");
+
+ const char *identifier = syslog_identifier_rkv ? buffer_tostring(syslog_identifier_rkv->wb) : "UNKNOWN";
+ const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : "UNKNOWN";
+
+ buffer_flush(rkv->wb);
+ buffer_sprintf(rkv->wb, "%s[%s]", identifier, pid);
+
+ buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
+}
+
+static void function_systemd_journal(const char *transaction, char *function, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
+ char *words[SYSTEMD_JOURNAL_MAX_PARAMS] = { NULL };
+ size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_JOURNAL_MAX_PARAMS);
+
+ BUFFER *wb = buffer_create(0, NULL);
+ buffer_flush(wb);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
+
+ FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,
+ SYSTEMD_ALWAYS_VISIBLE_KEYS,
+ SYSTEMD_KEYS_INCLUDED_IN_FACETS,
+ SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);
+
+ facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);
+ facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);
+ facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
+ facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
+ facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
+
+ // register the fields in the order you want them on the dashboard
+
+ facets_register_dynamic_key(facets, "ND_JOURNAL_PROCESS", FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS,
+ systemd_journal_dynamic_row_id, NULL);
+
+ facets_register_key(facets, "MESSAGE",
+ FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_MAIN_TEXT|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS);
+
+ facets_register_key_transformation(facets, "PRIORITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
+ systemd_journal_transform_priority, NULL);
+
+ facets_register_key_transformation(facets, "SYSLOG_FACILITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
+ systemd_journal_transform_syslog_facility, NULL);
+
+ facets_register_key(facets, "SYSLOG_IDENTIFIER", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
+ facets_register_key(facets, "UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
+ facets_register_key(facets, "USER_UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
+
+ facets_register_key_transformation(facets, "_UID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
+ systemd_journal_transform_uid, uids);
+
+ facets_register_key_transformation(facets, "_GID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
+ systemd_journal_transform_gid, gids);
+
+ time_t after_s = 0, before_s = 0;
+ usec_t anchor = 0;
+ size_t last = 0;
+ const char *query = NULL;
+
+ buffer_json_member_add_object(wb, "request");
+ buffer_json_member_add_object(wb, "filters");
+
+ for(int i = 1; i < SYSTEMD_JOURNAL_MAX_PARAMS ;i++) {
+ const char *keyword = get_word(words, num_words, i);
+ if(!keyword) break;
+
+ if(strcmp(keyword, JOURNAL_PARAMETER_HELP) == 0) {
+ systemd_journal_function_help(transaction);
+ goto cleanup;
+ }
+ else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", strlen(JOURNAL_PARAMETER_AFTER ":")) == 0) {
+ after_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_AFTER ":")]);
+ }
+ else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", strlen(JOURNAL_PARAMETER_BEFORE ":")) == 0) {
+ before_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_BEFORE ":")]);
+ }
+ else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", strlen(JOURNAL_PARAMETER_ANCHOR ":")) == 0) {
+ anchor = str2ull(&keyword[strlen(JOURNAL_PARAMETER_ANCHOR ":")], NULL);
+ }
+ else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", strlen(JOURNAL_PARAMETER_LAST ":")) == 0) {
+ last = str2ul(&keyword[strlen(JOURNAL_PARAMETER_LAST ":")]);
+ }
+ else if(strncmp(keyword, JOURNAL_PARAMETER_QUERY ":", strlen(JOURNAL_PARAMETER_QUERY ":")) == 0) {
+ query= &keyword[strlen(JOURNAL_PARAMETER_QUERY ":")];
+ }
+ else {
+ char *value = strchr(keyword, ':');
+ if(value) {
+ *value++ = '\0';
+
+ buffer_json_member_add_array(wb, keyword);
+
+ while(value) {
+ char *sep = strchr(value, ',');
+ if(sep)
+ *sep++ = '\0';
+
+ facets_register_facet_filter(facets, keyword, value, FACET_KEY_OPTION_REORDER);
+ buffer_json_add_array_item_string(wb, value);
+
+ value = sep;
+ }
+
+ buffer_json_array_close(wb); // keyword
+ }
+ }
+ }
+
+ buffer_json_object_close(wb); // filters
+
+ time_t expires = now_realtime_sec() + 1;
+ time_t now_s;
+
+ if(!after_s && !before_s) {
+ now_s = now_realtime_sec();
+ before_s = now_s;
+ after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
+ }
+ else
+ rrdr_relative_window_to_absolute(&after_s, &before_s, &now_s, false);
+
+ if(after_s > before_s) {
+ time_t tmp = after_s;
+ after_s = before_s;
+ before_s = tmp;
+ }
+
+ if(after_s == before_s)
+ after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
+
+ if(!last)
+ last = SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY;
+
+ buffer_json_member_add_time_t(wb, "after", after_s);
+ buffer_json_member_add_time_t(wb, "before", before_s);
+ buffer_json_member_add_uint64(wb, "anchor", anchor);
+ buffer_json_member_add_uint64(wb, "last", last);
+ buffer_json_member_add_string(wb, "query", query);
+ buffer_json_member_add_time_t(wb, "timeout", timeout);
+ buffer_json_object_close(wb); // request
+
+ facets_set_items(facets, last);
+ facets_set_anchor(facets, anchor);
+ facets_set_query(facets, query);
+ int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,
+ now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
+
+ if(response != HTTP_RESP_OK) {
+ pluginsd_function_json_error(transaction, response, "failed");
+ goto cleanup;
+ }
+
+ pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);
+ fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
+
+ pluginsd_function_result_end_to_stdout();
+
+cleanup:
+ facets_destroy(facets);
+ buffer_free(wb);
+}
+
+static void *reader_main(void *arg __maybe_unused) {
+ char buffer[PLUGINSD_LINE_MAX + 1];
+
+ char *s = NULL;
+ while(!plugin_should_exit && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
+
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
+ size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, PLUGINSD_MAX_WORDS);
+
+ const char *keyword = get_word(words, num_words, 0);
+
+ if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
+ char *transaction = get_word(words, num_words, 1);
+ char *timeout_s = get_word(words, num_words, 2);
+ char *function = get_word(words, num_words, 3);
+
+ if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
+ netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
+ keyword,
+ transaction?transaction:"(unset)",
+ timeout_s?timeout_s:"(unset)",
+ function?function:"(unset)");
+ }
+ else {
+ int timeout = str2i(timeout_s);
+ if(timeout <= 0) timeout = SYSTEMD_JOURNAL_DEFAULT_TIMEOUT;
+
+ netdata_mutex_lock(&mutex);
+
+ if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)
+ function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
+ else
+ pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in systemd-journal.plugin.");
+
+ fflush(stdout);
+ netdata_mutex_unlock(&mutex);
+ }
+ }
+ else
+ netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)");
+ }
+
+ if(!s || feof(stdin) || ferror(stdin)) {
+ plugin_should_exit = true;
+ netdata_log_error("Received error on stdin.");
+ }
+
+ exit(1);
+}
+
+int main(int argc __maybe_unused, char **argv __maybe_unused) {
+ stderror = stderr;
+ clocks_init();
+
+ program_name = "systemd-journal.plugin";
+
+ // disable syslog
+ error_log_syslog = 0;
+
+ // set errors flood protection to 100 logs per hour
+ error_log_errors_per_period = 100;
+ error_log_throttle_period = 3600;
+
+ uids = dictionary_create(0);
+ gids = dictionary_create(0);
+
+ // ------------------------------------------------------------------------
+ // debug
+
+ if(argc == 2 && strcmp(argv[1], "debug") == 0) {
+ char buf[] = "systemd-journal after:-86400 before:0 last:500";
+ function_systemd_journal("123", buf, "", 0, 30);
+ exit(1);
+ }
+
+ // ------------------------------------------------------------------------
+
+ netdata_thread_t reader_thread;
+ netdata_thread_create(&reader_thread, "SDJ_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
+
+ // ------------------------------------------------------------------------
+
+ time_t started_t = now_monotonic_sec();
+
+ size_t iteration;
+ usec_t step = 1000 * USEC_PER_MS;
+ bool tty = isatty(fileno(stderr)) == 1;
+
+ netdata_mutex_lock(&mutex);
+ fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n",
+ SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
+
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(iteration = 0; 1 ; iteration++) {
+ netdata_mutex_unlock(&mutex);
+ heartbeat_next(&hb, step);
+ netdata_mutex_lock(&mutex);
+
+ if(!tty)
+ fprintf(stdout, "\n");
+
+ fflush(stdout);
+
+ time_t now = now_monotonic_sec();
+ if(now - started_t > 86400)
+ break;
+ }
+
+ dictionary_destroy(uids);
+ dictionary_destroy(gids);
+
+ exit(0);
+}
diff --git a/configure.ac b/configure.ac
index d2f047e61b..ac0d7bff6f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -69,6 +69,12 @@ AC_ARG_ENABLE(
,
[enable_plugin_freeipmi="detect"]
)
+AC_ARG_ENABLE(
+ [plugin-systemd-journal],
+ [AS_HELP_STRING([--enable-plugin-systemd-journal], [enable systemd-journal plugin @<:@default autodetect@:>@])],
+ ,
+ [enable_plugin_systemd_journal="detect"]
+)
AC_ARG_ENABLE(
[plugin-cups],
[AS_HELP_STRING([--enable-plugin-cups], [enable cups plugin @<:@default autodetect@:>@])],
@@ -1106,6 +1112,39 @@ AC_MSG_RESULT([${enable_plugin_freeipmi}])
AM_CONDITIONAL([ENABLE_PLUGIN_FREEIPMI], [test "${enable_plugin_freeipmi}" = "yes"])
+# -----------------------------------------------------------------------------
+# systemd-journal.plugin - systemd
+
+LIBS_BAK="${LIBS}"
+
+AC_CHECK_LIB([systemd], [sd_journal_open], [have_systemd_libs=yes], [have_systemd_libs=no])
+AC_CHECK_HEADERS([systemd/sd-journal.h], [have_systemd_journal_header=yes], [have_systemd_journal_header=no])
+
+if test "${have_systemd_libs}" = "yes" -a "${have_systemd_journal_header}" = "yes"; then
+ have_systemd="yes"
+else
+ have_systemd="no"
+fi
+
+test "${enable_plugin_systemd_journal}" = "yes" -a "${have_systemd}" != "yes" && \
+ AC_MSG_ERROR([systemd is required but not found. Try installing 'libsystemd-dev' or 'libsystemd-devel'])
+
+AC_MSG_CHECKING([if systemd-journal.plugin should be enabled])
+if test "${enable_plugin_systemd_journal}" != "no" -a "${have_systemd}" = "yes"; then
+ enable_plugin_systemd_journal="yes"
+ AC_DEFINE([HAVE_SYSTEMD], [1], [systemd usability])
+ OPTIONAL_SYSTEMD_CFLAGS="-I/usr/include"
+ OPTIONAL_SYSTEMD_LIBS="-lsystemd"
+else
+ enable_plugin_systemd_journal="no"
+fi
+AC_MSG_RESULT([${enable_plugin_systemd_journal}])
+AM_CONDITIONAL([ENABLE_PLUGIN_SYSTEMD_JOURNAL], [test "${enable_plugin_systemd_journal}" = "yes"])
+
+AC_MSG_NOTICE([OPTIONAL_SYSTEMD_LIBS is set to: ${OPTIONAL_SYSTEMD_LIBS}])
+
+LIBS="${LIBS_BAK}"
+
# -----------------------------------------------------------------------------
# cups.plugin - libcups
@@ -1874,6 +1913,7 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
AC_SUBST([OPTIONAL_GTEST_LIBS])
AC_SUBST([OPTIONAL_ML_CFLAGS])
AC_SUBST([OPTIONAL_ML_LIBS])
+AC_SUBST(OPTIONAL_SYSTEMD_LIBS)
# -----------------------------------------------------------------------------
# Check if cmocka is available - needed for unit testing
@@ -1937,6 +1977,7 @@ AC_CONFIG_FILES([
collectors/tc.plugin/Makefile
collectors/xenstat.plugin/Makefile
collectors/perf.plugin/Makefile
+ collectors/systemd-journal.plugin/Makefile
daemon/Makefile
database/Makefile
database/contexts/Makefile
@@ -1968,6 +2009,7 @@ AC_CONFIG_FILES([
libnetdata/dictionary/Makefile
libnetdata/ebpf/Makefile
libnetdata/eval/Makefile
+ libnetdata/facets/Makefile
libnetdata/july/Makefile
libnetdata/locks/Makefile
libnetdata/log/Makefile
diff --git a/contrib/debian/rules b/contrib/debian/rules
index 3fe4e63fac..d41b289b78 100755
--- a/contrib/debian/rules
+++ b/contrib/debian/rules
@@ -213,6 +213,9 @@ override_dh_fixperms:
# local-listeners
chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/local-listeners
+ # systemd-journal
+ # chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/systemd-journal.plugin
+
override_dh_installlogrotate:
cp system/logrotate/netdata debian/netdata.logrotate
dh_installlogrotate
diff --git a/daemon/buildinfo.c b/daemon/buildinfo.c
index 56cde84fc2..4bc1e72a4e 100644
--- a/daemon/buildinfo.c
+++ b/daemon/buildinfo.c
@@ -1469,7 +1469,7 @@ void print_build_info_json(void) {
populate_directories();
BUFFER *b = buffer_create(0, NULL);
- buffer_json_initialize(b, "\"", "\"", 0, true, false);
+ buffer_json_initialize(b, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
build_info_to_json_object(b);
diff --git a/daemon/main.c b/daemon/main.c
index 9ed481cc5e..6ddf57aa17 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -1332,6 +1332,7 @@ int mrg_unittest(void);
int julytest(void);
int pluginsd_parser_unittest(void);
void replication_initialize(void);
+void bearer_tokens_init(void);
int main(int argc, char **argv) {
// initialize the system clocks
diff --git a/database/contexts/api_v1.c b/database/contexts/api_v1.c
index b4bcfe4ae3..bc7fee496d 100644
--- a/database/contexts/api_v1.c
+++ b/database/contexts/api_v1.c
@@ -148,7 +148,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void
if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
- buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false);
+ buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
struct rrdcontext_to_json t_metrics = {
.wb = wb_metrics,
@@ -268,7 +268,7 @@ static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void
|| t_parent->chart_dimensions) {
wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
- buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false);
+ buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
struct rrdcontext_to_json t_instances = {
.wb = wb_instances,
@@ -366,9 +366,9 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
if(after != 0 && before != 0)
- rrdr_relative_window_to_absolute(&after, &before, NULL);
+ rrdr_relative_window_to_absolute(&after, &before, NULL, false);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
struct rrdcontext_to_json t_contexts = {
.wb = wb,
.options = options|RRDCONTEXT_OPTION_SKIP_ID,
@@ -403,9 +403,9 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
uuid_unparse(*host->node_id, node_uuid);
if(after != 0 && before != 0)
- rrdr_relative_window_to_absolute(&after, &before, NULL);
+ rrdr_relative_window_to_absolute(&after, &before, NULL, false);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
buffer_json_member_add_string(wb, "node_id", node_uuid);
diff --git a/database/contexts/api_v2.c b/database/contexts/api_v2.c
index 88b55ad20a..08739160d9 100644
--- a/database/contexts/api_v2.c
+++ b/database/contexts/api_v2.c
@@ -1298,7 +1298,7 @@ int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_ha
buffer_flush(w->response.data);
- buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
+ buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false);
buffer_json_finalize(w->response.data);
@@ -1934,14 +1934,14 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
}
if(req->after || req->before) {
- ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now);
+ ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false);
ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
}
else
ctl.now = now_realtime_sec();
- buffer_json_initialize(wb, "\"", "\"", 0,
- true, (req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG));
+ buffer_json_initialize(wb, "\"", "\"", 0, true,
+ ((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_uint64(wb, "api", 2);
diff --git a/database/contexts/query_target.c b/database/contexts/query_target.c
index 16e1a73c97..829640b909 100644
--- a/database/contexts/query_target.c
+++ b/database/contexts/query_target.c
@@ -1052,7 +1052,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
if(query_target_has_percentage_of_group(qt))
qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
- qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
+ qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now,
+ unittest_running);
// prepare our local variables - we need these across all these functions
QUERY_TARGET_LOCALS qtl = {
diff --git a/database/rrdfunctions.c b/database/rrdfunctions.c
index cee1ac89ee..d32a4b8c91 100644
--- a/database/rrdfunctions.c
+++ b/database/rrdfunctions.c
@@ -270,7 +270,7 @@ static inline size_t sanitize_function_text(char *dst, const char *src, size_t d
// we keep a dictionary per RRDSET with these functions
// the dictionary is created on demand (only when a function is added to an RRDSET)
-typedef enum {
+typedef enum __attribute__((packed)) {
RRD_FUNCTION_LOCAL = (1 << 0),
RRD_FUNCTION_GLOBAL = (1 << 1),
@@ -279,7 +279,7 @@ typedef enum {
struct rrd_collector_function {
bool sync; // when true, the function is called synchronously
- uint8_t options; // RRD_FUNCTION_OPTIONS
+ RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
STRING *help;
int timeout; // the default timeout of the function
@@ -814,7 +814,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
buffer_flush(wb);
wb->content_type = CT_APPLICATION_JSON;
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
@@ -858,8 +858,8 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
// retention
buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node
- buffer_json_add_array_item_uint64(wb, s.db.first_time_s * 1000); // dbFrom
- buffer_json_add_array_item_uint64(wb, s.db.last_time_s * 1000); // dbTo
+ buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom
+ buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo
if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s)
buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration
@@ -877,7 +877,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
// collection
if(s.ingest.since) {
- buffer_json_add_array_item_uint64(wb, s.ingest.since * 1000); // InSince
+ buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince
buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge
}
else {
@@ -897,7 +897,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
// streaming
if(s.stream.since) {
- buffer_json_add_array_item_uint64(wb, s.stream.since * 1000); // OutSince
+ buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince
buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge
}
else {
@@ -990,19 +990,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
NULL);
buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From",
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To",
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration",
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_NONE, NULL);
@@ -1049,13 +1049,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
// --- collection ---
buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change",
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_NONE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age",
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -1124,13 +1124,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
// --- streaming ---
buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change",
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_NONE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age",
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
@@ -1242,14 +1242,14 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince",
"Last Outbound Connection Attempt Status Change Time",
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_NONE, NULL);
buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge",
"Last Outbound Connection Attempt Status Change Age",
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_OPTS_VISIBLE, NULL);
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am
index b81d620ba6..e85f4abe1b 100644
--- a/libnetdata/Makefile.am
+++ b/libnetdata/Makefile.am
@@ -14,6 +14,7 @@ SUBDIRS = \
dictionary \
ebpf \
eval \
+ facets \
json \
july \
health \
diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c
index e775b95176..2d09bb1ff6 100644
--- a/libnetdata/buffer/buffer.c
+++ b/libnetdata/buffer/buffer.c
@@ -307,11 +307,11 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
// ----------------------------------------------------------------------------
void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
- bool add_anonymous_object, bool minify) {
+ bool add_anonymous_object, BUFFER_JSON_OPTIONS options) {
strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
strncpyz(wb->json.value_quote, value_quote, BUFFER_QUOTE_MAX_SIZE);
- wb->json.minify = minify;
+ wb->json.options = options;
wb->json.depth = (int8_t)(depth - 1);
_buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
@@ -339,7 +339,7 @@ void buffer_json_finalize(BUFFER *wb) {
}
}
- if(!wb->json.minify)
+ if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY))
buffer_fast_strcat(wb, "\n", 1);
}
@@ -490,13 +490,13 @@ int buffer_unittest(void) {
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_finalize(wb);
errors += buffer_expect(wb, "{\n}\n");
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(wb, "hello", "world");
buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
buffer_json_member_add_object(wb, "object1");
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index d0078a521d..02134c8b26 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -68,6 +68,12 @@ typedef enum __attribute__ ((__packed__)) {
CT_APPLICATION_ZIP,
} HTTP_CONTENT_TYPE;
+typedef enum __attribute__ ((__packed__)) {
+ BUFFER_JSON_OPTIONS_DEFAULT = 0,
+ BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),
+ BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS = (1 << 1),
+} BUFFER_JSON_OPTIONS;
+
typedef struct web_buffer {
size_t size; // allocation size of buffer, in bytes
size_t len; // current data length in buffer, in bytes
@@ -82,7 +88,7 @@ typedef struct web_buffer {
char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
int8_t depth;
- bool minify;
+ BUFFER_JSON_OPTIONS options;
BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
} json;
} BUFFER;
@@ -148,7 +154,7 @@ static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
}
void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
- bool add_anonymous_object, bool minify);
+ bool add_anonymous_object, BUFFER_JSON_OPTIONS options);
void buffer_json_finalize(BUFFER *wb);
@@ -668,7 +674,8 @@ static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {
if(wb->json.stack[wb->json.depth].count)
buffer_fast_strcat(wb, ",", 1);
- if(wb->json.minify)
+ if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||
+ (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS)))
return;
buffer_fast_strcat(wb, "\n", 1);
@@ -715,7 +722,7 @@ static inline void buffer_json_object_close(BUFFER *wb) {
assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");
#endif
- if(!wb->json.minify) {
+ if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY)) {
buffer_fast_strcat(wb, "\n", 1);
buffer_print_spaces(wb, wb->json.depth);
}
@@ -801,48 +808,42 @@ static inline void buffer_json_add_array_item_array(BUFFER *wb) {
}
static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_json_add_string_value(wb, value);
wb->json.stack[wb->json.depth].count++;
}
static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_netdata_double(wb, value);
wb->json.stack[wb->json.depth].count++;
}
static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_int64(wb, value);
wb->json.stack[wb->json.depth].count++;
}
static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_uint64(wb, value);
wb->json.stack[wb->json.depth].count++;
}
static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_int64(wb, value);
wb->json.stack[wb->json.depth].count++;
}
static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_int64(wb, value);
buffer_fast_strcat(wb, "000", 3);
@@ -850,8 +851,7 @@ static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value)
}
static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_print_int64(wb, value);
buffer_fast_strcat(wb, "000", 3);
@@ -859,8 +859,7 @@ static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value
}
static inline void buffer_json_add_array_item_object(BUFFER *wb) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
+ buffer_print_json_comma_newline_spacing(wb);
buffer_fast_strcat(wb, "{", 1);
wb->json.stack[wb->json.depth].count++;
@@ -919,6 +918,11 @@ static inline void buffer_json_array_close(BUFFER *wb) {
assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");
#endif
+ if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS) {
+ buffer_fast_strcat(wb, "\n", 1);
+ buffer_print_spaces(wb, wb->json.depth);
+ }
+
buffer_fast_strcat(wb, "]", 1);
_buffer_json_depth_pop(wb);
}
@@ -928,6 +932,8 @@ typedef enum __attribute__((packed)) {
RRDF_FIELD_OPTS_UNIQUE_KEY = (1 << 0), // the field is the unique key of the row
RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default
RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky
+ RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width
+ RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should get full width
} RRDF_FIELD_OPTIONS;
typedef enum __attribute__((packed)) {
@@ -969,7 +975,8 @@ static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {
typedef enum __attribute__((packed)) {
RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation
RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%
- RRDF_FIELD_VISUAL_PILL, // array of values (transformation is respected)
+ RRDF_FIELD_VISUAL_PILL, //
+ RRDF_FIELD_VISUAL_MARKDOC, //
} RRDF_FIELD_VISUAL;
static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) {
@@ -983,14 +990,18 @@ static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual)
case RRDF_FIELD_VISUAL_PILL:
return "pill";
+
+ case RRDF_FIELD_VISUAL_MARKDOC:
+ return "markdoc";
}
}
typedef enum __attribute__((packed)) {
RRDF_FIELD_TRANSFORM_NONE, // show the value as-is
- RRDF_FIELD_TRANSFORM_NUMBER, // show the value repsecting the decimal_points
- RRDF_FIELD_TRANSFORM_DURATION, // transform as duration in second to a human readable duration
- RRDF_FIELD_TRANSFORM_DATETIME, // UNIX epoch timestamp in ms
+ RRDF_FIELD_TRANSFORM_NUMBER, // show the value respecting the decimal_points
+ RRDF_FIELD_TRANSFORM_DURATION_S, // transform as duration in second to a human-readable duration
+ RRDF_FIELD_TRANSFORM_DATETIME_MS, // UNIX epoch timestamp in ms
+ RRDF_FIELD_TRANSFORM_DATETIME_USEC, // UNIX epoch timestamp in usec
} RRDF_FIELD_TRANSFORM;
static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) {
@@ -1002,11 +1013,14 @@ static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM tr
case RRDF_FIELD_TRANSFORM_NUMBER:
return "number";
- case RRDF_FIELD_TRANSFORM_DURATION:
+ case RRDF_FIELD_TRANSFORM_DURATION_S:
return "duration";
- case RRDF_FIELD_TRANSFORM_DATETIME:
+ case RRDF_FIELD_TRANSFORM_DATETIME_MS:
return "datetime";
+
+ case RRDF_FIELD_TRANSFORM_DATETIME_USEC:
+ return "datetime_usec";
}
}
@@ -1064,18 +1078,26 @@ static inline const char *rrdf_field_summary_to_string(RRDF_FIELD_SUMMARY summar
}
typedef enum __attribute__((packed)) {
+ RRDF_FIELD_FILTER_NONE,
RRDF_FIELD_FILTER_RANGE,
RRDF_FIELD_FILTER_MULTISELECT,
+ RRDF_FIELD_FILTER_FACET,
} RRDF_FIELD_FILTER;
static inline const char *rrdf_field_filter_to_string(RRDF_FIELD_FILTER filter) {
switch(filter) {
- default:
case RRDF_FIELD_FILTER_RANGE:
return "range";
case RRDF_FIELD_FILTER_MULTISELECT:
return "multiselect";
+
+ case RRDF_FIELD_FILTER_FACET:
+ return "facet";
+
+ default:
+ case RRDF_FIELD_FILTER_NONE:
+ return "none";
}
}
@@ -1114,6 +1136,9 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const
buffer_json_member_add_boolean(wb, "sticky", options & RRDF_FIELD_OPTS_STICKY);
buffer_json_member_add_string(wb, "summary", rrdf_field_summary_to_string(summary));
buffer_json_member_add_string(wb, "filter", rrdf_field_filter_to_string(filter));
+
+ buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH);
+ buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP);
}
buffer_json_object_close(wb);
}
diff --git a/libnetdata/facets/Makefile.am b/libnetdata/facets/Makefile.am
new file mode 100644
index 0000000000..161784b8f6
--- /dev/null
+++ b/libnetdata/facets/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/facets/README.md b/libnetdata/facets/README.md
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c
new file mode 100644
index 0000000000..5a0ea30f76
--- /dev/null
+++ b/libnetdata/facets/facets.c
@@ -0,0 +1,827 @@
+#include "facets.h"
+
+#define FACET_VALUE_UNSET "-"
+#define HISTOGRAM_COLUMNS 60
+
+static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
+
+// ----------------------------------------------------------------------------
+
+time_t calculate_bar_width(time_t before, time_t after) {
+ // Array of valid durations in seconds
+ static time_t valid_durations[] = {
+ 1,
+ 15,
+ 30,
+ 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
+ 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
+ 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
+ 1 * (30*86400) // months
+ };
+ static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
+
+ time_t duration = before - after;
+ time_t bar_width = 1;
+
+ for (int i = array_size - 1; i >= 0; --i) {
+ if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) {
+ bar_width = valid_durations[i];
+ break;
+ }
+ }
+
+ return bar_width;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void uint32_to_char(uint32_t num, char *out) {
+ static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
+
+ int i;
+ for(i = 5; i >= 0; --i) {
+ out[i] = id_encoding_characters[num & 63];
+ num >>= 6;
+ }
+ out[6] = '\0';
+}
+
+inline void facets_string_hash(const char *src, char *out) {
+ uint32_t hash1 = fnv1a_hash32(src);
+ uint32_t hash2 = djb2_hash32(src);
+ uint32_t hash3 = larson_hash32(src);
+
+ uint32_to_char(hash1, out);
+ uint32_to_char(hash2, &out[6]);
+ uint32_to_char(hash3, &out[12]);
+
+ out[18] = '\0';
+}
+
+// ----------------------------------------------------------------------------
+
+typedef struct facet_value {
+ const char *name;
+
+ bool selected;
+
+ uint32_t rows_matching_facet_value;
+ uint32_t final_facet_value_counter;
+} FACET_VALUE;
+
+struct facet_key {
+ const char *name;
+
+ DICTIONARY *values;
+
+ FACET_KEY_OPTIONS options;
+
+ bool default_selected_for_values; // the default "selected" for all values in the dictionary
+
+ // members about the current row
+ uint32_t key_found_in_row;
+ uint32_t key_values_selected_in_row;
+ BUFFER *current_value;
+
+ uint32_t order;
+
+ struct {
+ facet_dynamic_row_t cb;
+ void *data;
+ } dynamic;
+
+ struct {
+ facets_key_transformer_t cb;
+ void *data;
+
+ } transform;
+};
+
+struct facets {
+ SIMPLE_PATTERN *visible_keys;
+ SIMPLE_PATTERN *excluded_keys;
+ SIMPLE_PATTERN *included_keys;
+
+ FACETS_OPTIONS options;
+ usec_t anchor;
+
+ SIMPLE_PATTERN *query; // the full text search pattern
+ size_t keys_filtered_by_query; // the number of fields we do full text search (constant)
+ size_t keys_matched_by_query; // the number of fields matched the full text search (per row)
+
+ DICTIONARY *accepted_params;
+
+ DICTIONARY *keys;
+ FACET_ROW *base; // double linked list of the selected facets rows
+
+ uint32_t items_to_return;
+ uint32_t max_items_to_return;
+ uint32_t order;
+
+ struct {
+ FACET_ROW *last_added;
+
+ size_t evaluated;
+ size_t matched;
+
+ size_t first;
+ size_t forwards;
+ size_t backwards;
+ size_t skips_before;
+ size_t skips_after;
+ size_t prepends;
+ size_t appends;
+ size_t shifts;
+ } operations;
+};
+
+// ----------------------------------------------------------------------------
+
+static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
+ if(!k->key_found_in_row)
+ v->rows_matching_facet_value++;
+
+ k->key_found_in_row++;
+
+ if(v->selected)
+ k->key_values_selected_in_row++;
+}
+
+static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
+ bool included = true, excluded = false;
+
+ if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) {
+ if(k->options & FACET_KEY_OPTION_FACET) {
+ included = true;
+ excluded = false;
+ }
+ else if(k->options & FACET_KEY_OPTION_NO_FACET) {
+ included = false;
+ excluded = true;
+ }
+ }
+ else {
+ if (facets->included_keys) {
+ if (!simple_pattern_matches(facets->included_keys, k->name))
+ included = false;
+ }
+
+ if (facets->excluded_keys) {
+ if (simple_pattern_matches(facets->excluded_keys, k->name))
+ excluded = true;
+ }
+ }
+
+ if(included && !excluded) {
+ k->options |= FACET_KEY_OPTION_FACET;
+ k->options &= ~FACET_KEY_OPTION_NO_FACET;
+ return true;
+ }
+
+ k->options |= FACET_KEY_OPTION_NO_FACET;
+ k->options &= ~FACET_KEY_OPTION_FACET;
+ return false;
+}
+
+// ----------------------------------------------------------------------------
+// FACET_VALUE dictionary hooks
+
+static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_VALUE *v = value;
+ FACET_KEY *k = data;
+
+ if(v->name) {
+ // an actual value, not a filter
+ v->name = strdupz(v->name);
+ facet_value_is_used(k, v);
+ }
+
+ if(!v->selected)
+ v->selected = k->default_selected_for_values;
+}
+
+static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
+ FACET_VALUE *v = old_value;
+ FACET_VALUE *nv = new_value;
+ FACET_KEY *k = data;
+
+ if(!v->name && nv->name)
+ // an actual value, not a filter
+ v->name = strdupz(nv->name);
+
+ if(v->name)
+ facet_value_is_used(k, v);
+
+ return false;
+}
+
+static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+ FACET_VALUE *v = value;
+ freez((char *)v->name);
+}
+
+// ----------------------------------------------------------------------------
+// FACET_KEY dictionary hooks
+
+static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) {
+ if(k->values)
+ return;
+
+ if(facets_key_is_facet(facets, k)) {
+ k->values = dictionary_create_advanced(
+ DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
+ NULL, sizeof(FACET_VALUE));
+ dictionary_register_insert_callback(k->values, facet_value_insert_callback, k);
+ dictionary_register_conflict_callback(k->values, facet_value_conflict_callback, k);
+ dictionary_register_delete_callback(k->values, facet_value_delete_callback, k);
+ }
+}
+
+static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_KEY *k = value;
+ FACETS *facets = data;
+
+ if(!(k->options & FACET_KEY_OPTION_REORDER))
+ k->order = facets->order++;
+
+ if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS))
+ facets->keys_filtered_by_query++;
+
+ if(k->name) {
+ // an actual value, not a filter
+ k->name = strdupz(k->name);
+ facet_key_late_init(facets, k);
+ }
+
+ k->current_value = buffer_create(0, NULL);
+}
+
+static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
+ FACET_KEY *k = old_value;
+ FACET_KEY *nk = new_value;
+ FACETS *facets = data;
+
+ if(!k->name && nk->name) {
+ // an actual value, not a filter
+ k->name = strdupz(nk->name);
+ facet_key_late_init(facets, k);
+ }
+
+ if(k->options & FACET_KEY_OPTION_REORDER) {
+ k->order = facets->order++;
+ k->options &= ~FACET_KEY_OPTION_REORDER;
+ }
+
+ return false;
+}
+
+static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+ FACET_KEY *k = value;
+ dictionary_destroy(k->values);
+ buffer_free(k->current_value);
+ freez((char *)k->name);
+}
+
+// ----------------------------------------------------------------------------
+
+FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
+ FACETS *facets = callocz(1, sizeof(FACETS));
+ facets->options = options;
+ facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY));
+ dictionary_register_insert_callback(facets->keys, facet_key_insert_callback, facets);
+ dictionary_register_conflict_callback(facets->keys, facet_key_conflict_callback, facets);
+ dictionary_register_delete_callback(facets->keys, facet_key_delete_callback, facets);
+
+ if(facet_keys && *facet_keys)
+ facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ if(non_facet_keys && *non_facet_keys)
+ facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ if(visible_keys && *visible_keys)
+ facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ facets->max_items_to_return = items_to_return;
+ facets->anchor = anchor;
+ facets->order = 1;
+
+ return facets;
+}
+
+void facets_destroy(FACETS *facets) {
+ dictionary_destroy(facets->accepted_params);
+ dictionary_destroy(facets->keys);
+ simple_pattern_free(facets->visible_keys);
+ simple_pattern_free(facets->included_keys);
+ simple_pattern_free(facets->excluded_keys);
+
+ while(facets->base) {
+ FACET_ROW *r = facets->base;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next);
+
+ facets_row_free(facets, r);
+ }
+
+ freez(facets);
+}
+
+void facets_accepted_param(FACETS *facets, const char *param) {
+ if(!facets->accepted_params)
+ facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+
+ dictionary_set(facets->accepted_params, param, NULL, 0);
+}
+
+inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) {
+ FACET_KEY tk = {
+ .name = key,
+ .options = options,
+ .default_selected_for_values = true,
+ };
+ char hash[FACET_STRING_HASH_SIZE];
+ facets_string_hash(tk.name, hash);
+ return dictionary_set(facets->keys, hash, &tk, sizeof(tk));
+}
+
+inline FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) {
+ FACET_KEY *k = facets_register_key(facets, key, options);
+ k->transform.cb = cb;
+ k->transform.data = data;
+ return k;
+}
+
+inline FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) {
+ FACET_KEY *k = facets_register_key(facets, key, options);
+ k->dynamic.cb = cb;
+ k->dynamic.data = data;
+ return k;
+}
+
+void facets_set_query(FACETS *facets, const char *query) {
+ if(!query)
+ return;
+
+ facets->query = simple_pattern_create(query, " \t", SIMPLE_PATTERN_SUBSTRING, false);
+}
+
+void facets_set_items(FACETS *facets, uint32_t items) {
+ facets->max_items_to_return = items;
+}
+
+void facets_set_anchor(FACETS *facets, usec_t anchor) {
+ facets->anchor = anchor;
+}
+
+void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options) {
+ FACET_KEY tk = {
+ .options = options,
+ };
+ FACET_KEY *k = dictionary_set(facets->keys, key_id, &tk, sizeof(tk));
+
+ k->default_selected_for_values = false;
+ k->options |= FACET_KEY_OPTION_FACET;
+ k->options &= ~FACET_KEY_OPTION_NO_FACET;
+ facet_key_late_init(facets, k);
+
+ FACET_VALUE tv = {
+ .selected = true,
+ };
+ dictionary_set(k->values, value_ids, &tv, sizeof(tv));
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) {
+ if(k->transform.cb)
+ k->transform.cb(facets, k->current_value, k->transform.data);
+
+ if(buffer_strlen(k->current_value) == 0)
+ buffer_strcat(k->current_value, FACET_VALUE_UNSET);
+
+// bool found = false;
+// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL)
+// found = true;
+
+ if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {
+ if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value)))
+ facets->keys_matched_by_query++;
+ }
+
+ if(k->values) {
+ FACET_VALUE tk = {
+ .name = buffer_tostring(k->current_value),
+ };
+ char hash[FACET_STRING_HASH_SIZE];
+ facets_string_hash(tk.name, hash);
+ dictionary_set(k->values, hash, &tk, sizeof(tk));
+ }
+ else {
+ k->key_found_in_row++;
+ k->key_values_selected_in_row++;
+ }
+}
+
+void facets_add_key_value(FACETS *facets, const char *key, const char *value) {
+ FACET_KEY *k = facets_register_key(facets, key, 0);
+ buffer_flush(k->current_value);
+ buffer_strcat(k->current_value, value);
+
+ facets_check_value(facets, k);
+}
+
+void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) {
+ FACET_KEY *k = facets_register_key(facets, key, 0);
+ buffer_flush(k->current_value);
+ buffer_strncat(k->current_value, value, value_len);
+
+ facets_check_value(facets, k);
+}
+
+// ----------------------------------------------------------------------------
+// FACET_ROW dictionary hooks
+
+static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = value;
+ FACET_ROW *row = data; (void)row;
+
+ rkv->wb = buffer_create(0, NULL);
+ buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET);
+}
+
+static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = old_value;
+ FACET_ROW_KEY_VALUE *n_rkv = new_value;
+ FACET_ROW *row = data; (void)row;
+
+ buffer_flush(rkv->wb);
+ buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET);
+
+ return false;
+}
+
+static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = value;
+ FACET_ROW *row = data; (void)row;
+
+ buffer_free(rkv->wb);
+}
+
+// ----------------------------------------------------------------------------
+// FACET_ROW management
+
+static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
+ dictionary_destroy(row->dict);
+ freez(row);
+}
+
+static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) {
+ FACET_ROW *row;
+
+ if(into)
+ row = into;
+ else {
+ row = callocz(1, sizeof(FACET_ROW));
+ row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE));
+ dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row);
+ dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row);
+ dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row);
+ }
+
+ row->usec = usec;
+
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ FACET_ROW_KEY_VALUE t = {
+ .tmp = buffer_strlen(k->current_value) ? buffer_tostring(k->current_value) : FACET_VALUE_UNSET,
+ .wb = NULL,
+ };
+ dictionary_set(row->dict, k->name, &t, sizeof(t));
+ }
+ dfe_done(k);
+
+ return row;
+}
+
+// ----------------------------------------------------------------------------
+
+static void facets_row_keep(FACETS *facets, usec_t usec) {
+ facets->operations.matched++;
+
+ if(usec < facets->anchor) {
+ facets->operations.skips_before++;
+ return;
+ }
+
+ if(unlikely(!facets->base)) {
+ facets->operations.last_added = facets_row_create(facets, usec, NULL);
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
+ facets->items_to_return++;
+ facets->operations.first++;
+ return;
+ }
+
+ if(likely(usec > facets->base->prev->usec))
+ facets->operations.last_added = facets->base->prev;
+
+ FACET_ROW *last = facets->operations.last_added;
+ while(last->prev != facets->base->prev && usec > last->prev->usec) {
+ last = last->prev;
+ facets->operations.backwards++;
+ }
+
+ while(last->next && usec < last->next->usec) {
+ last = last->next;
+ facets->operations.forwards++;
+ }
+
+ if(facets->items_to_return >= facets->max_items_to_return) {
+ if(last == facets->base->prev && usec < last->usec) {
+ facets->operations.skips_after++;
+ return;
+ }
+ }
+
+ facets->items_to_return++;
+
+ if(usec > last->usec) {
+ if(facets->items_to_return > facets->max_items_to_return) {
+ facets->items_to_return--;
+ facets->operations.shifts++;
+ facets->operations.last_added = facets->base->prev;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
+ facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added);
+ }
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
+ facets->operations.prepends++;
+ }
+ else {
+ facets->operations.last_added = facets_row_create(facets, usec, NULL);
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
+ facets->operations.appends++;
+ }
+
+ while(facets->items_to_return > facets->max_items_to_return) {
+ // we have to remove something
+
+ FACET_ROW *tmp = facets->base->prev;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next);
+ facets->items_to_return--;
+
+ if(unlikely(facets->operations.last_added == tmp))
+ facets->operations.last_added = facets->base->prev;
+
+ facets_row_free(facets, tmp);
+ facets->operations.shifts++;
+ }
+}
+
+void facets_rows_begin(FACETS *facets) {
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ k->key_found_in_row = 0;
+ k->key_values_selected_in_row = 0;
+ buffer_flush(k->current_value);
+ }
+ dfe_done(k);
+
+ facets->keys_matched_by_query = 0;
+}
+
+void facets_row_finished(FACETS *facets, usec_t usec) {
+ if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query)
+ goto cleanup;
+
+ facets->operations.evaluated++;
+
+ uint32_t total_keys = 0;
+ uint32_t selected_by = 0;
+
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ if(!k->key_found_in_row) {
+ internal_fatal(buffer_strlen(k->current_value), "key is not found in row but it has a current value");
+ // put the FACET_VALUE_UNSET value into it
+ facets_check_value(facets, k);
+ }
+
+ internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point");
+ internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point");
+ internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once");
+
+ k->key_found_in_row = 1;
+
+ total_keys += k->key_found_in_row;
+ selected_by += (k->key_values_selected_in_row) ? 1 : 0;
+ }
+ dfe_done(k);
+
+ if(selected_by >= total_keys - 1) {
+ uint32_t found = 0;
+
+ dfe_start_read(facets->keys, k){
+ uint32_t counted_by = selected_by;
+
+ if (counted_by != total_keys && !k->key_values_selected_in_row)
+ counted_by++;
+
+ if(counted_by == total_keys) {
+ if(k->values) {
+ char hash[FACET_STRING_HASH_SIZE];
+ facets_string_hash(buffer_tostring(k->current_value), hash);
+ FACET_VALUE *v = dictionary_get(k->values, hash);
+ v->final_facet_value_counter++;
+ }
+
+ found++;
+ }
+ }
+ dfe_done(k);
+
+ internal_fatal(!found, "We should find at least one facet to count this row");
+ (void)found;
+ }
+
+ if(selected_by == total_keys)
+ facets_row_keep(facets, usec);
+
+cleanup:
+ facets_rows_begin(facets);
+}
+
+// ----------------------------------------------------------------------------
+// output
+
+void facets_report(FACETS *facets, BUFFER *wb) {
+ buffer_json_member_add_boolean(wb, "show_ids", false);
+ buffer_json_member_add_boolean(wb, "has_history", true);
+
+ buffer_json_member_add_object(wb, "pagination");
+ buffer_json_member_add_boolean(wb, "enabled", true);
+ buffer_json_member_add_string(wb, "key", "anchor");
+ buffer_json_member_add_string(wb, "column", "timestamp");
+ buffer_json_object_close(wb);
+
+ buffer_json_member_add_array(wb, "accepted_params");
+ {
+ if(facets->accepted_params) {
+ void *t;
+ dfe_start_read(facets->accepted_params, t) {
+ buffer_json_add_array_item_string(wb, t_dfe.name);
+ }
+ dfe_done(t);
+ }
+
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ if(!k->values)
+ continue;
+
+ buffer_json_add_array_item_string(wb, k_dfe.name);
+ }
+ dfe_done(k);
+ }
+ buffer_json_array_close(wb); // accepted_params
+
+ buffer_json_member_add_array(wb, "facets");
+ {
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ if(!k->values)
+ continue;
+
+ buffer_json_add_array_item_object(wb); // key
+ {
+ buffer_json_member_add_string(wb, "id", k_dfe.name);
+ buffer_json_member_add_string(wb, "name", k->name);
+
+ if(!k->order)
+ k->order = facets->order++;
+
+ buffer_json_member_add_uint64(wb, "order", k->order);
+ buffer_json_member_add_array(wb, "options");
+ {
+ FACET_VALUE *v;
+ dfe_start_read(k->values, v) {
+ buffer_json_add_array_item_object(wb);
+ {
+ buffer_json_member_add_string(wb, "id", v_dfe.name);
+ buffer_json_member_add_string(wb, "name", v->name);
+ buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);
+ }
+ buffer_json_object_close(wb);
+ }
+ dfe_done(v);
+ }
+ buffer_json_array_close(wb); // options
+ }
+ buffer_json_object_close(wb); // key
+ }
+ dfe_done(k);
+ }
+ buffer_json_array_close(wb); // facets
+
+ buffer_json_member_add_object(wb, "columns");
+ {
+ size_t field_id = 0;
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ "timestamp", "Timestamp",
+ RRDF_FIELD_TYPE_TIMESTAMP,
+ RRDF_FIELD_VISUAL_VALUE,
+ RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
+ RRDF_FIELD_SORT_DESCENDING,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ RRDF_FIELD_FILTER_RANGE,
+ RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,
+ NULL);
+
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k) {
+ RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE;
+ bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY);
+
+ if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values))
+ visible = true;
+
+ if(!visible)
+ visible = simple_pattern_matches(facets->visible_keys, k->name);
+
+ if(visible)
+ options |= RRDF_FIELD_OPTS_VISIBLE;
+
+ if(k->options & FACET_KEY_OPTION_MAIN_TEXT)
+ options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
+
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ k_dfe.name, k->name ? k->name : k_dfe.name,
+ RRDF_FIELD_TYPE_STRING,
+ RRDF_FIELD_VISUAL_VALUE,
+ RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
+ RRDF_FIELD_SORT_ASCENDING,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE,
+ options,
+ FACET_VALUE_UNSET);
+ }
+ dfe_done(k);
+ }
+ buffer_json_object_close(wb); // columns
+
+ buffer_json_member_add_array(wb, "data");
+ {
+ for(FACET_ROW *row = facets->base ; row ;row = row->next) {
+ buffer_json_add_array_item_array(wb); // each row
+ buffer_json_add_array_item_uint64(wb, row->usec);
+
+ FACET_KEY *k;
+ dfe_start_read(facets->keys, k)
+ {
+ FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
+
+ if(unlikely(k->dynamic.cb)) {
+ if(unlikely(!rkv))
+ rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv));
+
+ k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data);
+ }
+ else
+ buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET);
+ }
+ dfe_done(k);
+ buffer_json_array_close(wb); // each row
+ }
+ }
+ buffer_json_array_close(wb); // data
+
+ buffer_json_member_add_string(wb, "default_sort_column", "timestamp");
+ buffer_json_member_add_array(wb, "default_charts");
+ buffer_json_array_close(wb);
+
+ buffer_json_member_add_object(wb, "items");
+ {
+ buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated);
+ buffer_json_member_add_uint64(wb, "matched", facets->operations.matched);
+ buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);
+ buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);
+ buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);
+ buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);
+ }
+ buffer_json_object_close(wb); // items
+
+ buffer_json_member_add_object(wb, "stats");
+ {
+ buffer_json_member_add_uint64(wb, "first", facets->operations.first);
+ buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards);
+ buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards);
+ buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before);
+ buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after);
+ buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends);
+ buffer_json_member_add_uint64(wb, "appends", facets->operations.appends);
+ buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts);
+ }
+ buffer_json_object_close(wb); // items
+
+}
diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h
new file mode 100644
index 0000000000..310b8cdc98
--- /dev/null
+++ b/libnetdata/facets/facets.h
@@ -0,0 +1,62 @@
+#ifndef FACETS_H
+#define FACETS_H 1
+
+#include "../libnetdata.h"
+
+typedef enum __attribute__((packed)) {
+ FACET_KEY_OPTION_FACET = (1 << 0), // filterable values
+ FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value
+ FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table
+ FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table
+ FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS)
+ FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap
+ FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter
+} FACET_KEY_OPTIONS;
+
+typedef struct facet_row_key_value {
+ const char *tmp;
+ BUFFER *wb;
+} FACET_ROW_KEY_VALUE;
+
+typedef struct facet_row {
+ usec_t usec;
+ DICTIONARY *dict;
+ struct facet_row *prev, *next;
+} FACET_ROW;
+
+typedef struct facets FACETS;
+typedef struct facet_key FACET_KEY;
+
+#define FACET_STRING_HASH_SIZE 19
+void facets_string_hash(const char *src, char *out);
+
+typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data);
+typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
+FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data);
+FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data);
+
+typedef enum __attribute__((packed)) {
+ FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table
+ FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
+} FACETS_OPTIONS;
+
+FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);
+void facets_destroy(FACETS *facets);
+
+void facets_accepted_param(FACETS *facets, const char *param);
+
+void facets_rows_begin(FACETS *facets);
+void facets_row_finished(FACETS *facets, usec_t usec);
+
+FACET_KEY *facets_register_key(FACETS *facets, const char *param, FACET_KEY_OPTIONS options);
+void facets_set_query(FACETS *facets, const char *query);
+void facets_set_items(FACETS *facets, uint32_t items);
+void facets_set_anchor(FACETS *facets, usec_t anchor);
+void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options);
+
+void facets_add_key_value(FACETS *facets, const char *key, const char *value);
+void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len);
+
+void facets_report(FACETS *facets, BUFFER *wb);
+
+#endif
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index d0a0b98a23..26582ffe2e 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -1966,3 +1966,135 @@ int hash256_string(const unsigned char *string, size_t size, char *hash) {
EVP_MD_CTX_destroy(ctx);
return 1;
}
+
+// Returns 1 if an absolute period was requested or 0 if it was a relative period
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) {
+ time_t now = now_realtime_sec() - 1;
+
+ if(now_ptr)
+ *now_ptr = now;
+
+ int absolute_period_requested = -1;
+ long long after_requested, before_requested;
+
+ before_requested = *before;
+ after_requested = *after;
+
+ // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
+ if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
+ // if the user asked for a positive relative time,
+ // flip it to a negative
+ if(before_requested > 0)
+ before_requested = -before_requested;
+
+ before_requested = now + before_requested;
+ absolute_period_requested = 0;
+ }
+
+ // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
+ if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
+ if(after_requested > 0)
+ after_requested = -after_requested;
+
+ // if the user didn't give an after, use the number of points
+ // to give a sane default
+ if(after_requested == 0)
+ after_requested = -600;
+
+ // since the query engine now returns inclusive timestamps
+ // it is awkward to return 6 points when after=-5 is given
+ // so for relative queries we add 1 second, to give
+ // more predictable results to users.
+ after_requested = before_requested + after_requested + 1;
+ absolute_period_requested = 0;
+ }
+
+ if(absolute_period_requested == -1)
+ absolute_period_requested = 1;
+
+ // check if the parameters are flipped
+ if(after_requested > before_requested) {
+ long long t = before_requested;
+ before_requested = after_requested;
+ after_requested = t;
+ }
+
+ // if the query requests future data
+ // shift the query back to be in the present time
+ // (this may also happen because of the rules above)
+ if(before_requested > now) {
+ long long delta = before_requested - now;
+ before_requested -= delta;
+ after_requested -= delta;
+ }
+
+ time_t absolute_minimum_time = now - (10 * 365 * 86400);
+ time_t absolute_maximum_time = now + (1 * 365 * 86400);
+
+ if (after_requested < absolute_minimum_time && !unittest_running)
+ after_requested = absolute_minimum_time;
+
+ if (after_requested > absolute_maximum_time && !unittest_running)
+ after_requested = absolute_maximum_time;
+
+ if (before_requested < absolute_minimum_time && !unittest_running)
+ before_requested = absolute_minimum_time;
+
+ if (before_requested > absolute_maximum_time && !unittest_running)
+ before_requested = absolute_maximum_time;
+
+ *before = before_requested;
+ *after = after_requested;
+
+ return (absolute_period_requested != 1);
+}
+
+int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size) {
+ static const unsigned char base64_table[256] = {
+ ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
+ ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
+ ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
+ ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
+ ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
+ ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
+ ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
+ ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
+ [0 ... '+' - 1] = 255,
+ ['+' + 1 ... '/' - 1] = 255,
+ ['9' + 1 ... 'A' - 1] = 255,
+ ['Z' + 1 ... 'a' - 1] = 255,
+ ['z' + 1 ... 255] = 255
+ };
+
+ size_t count = 0;
+ unsigned int tmp = 0;
+ int i, bit;
+
+ if (decoded_size < 1)
+ return 0; // Buffer size must be at least 1 for null termination
+
+ for (i = 0, bit = 0; encoded[i]; i++) {
+ unsigned char value = base64_table[(unsigned char)encoded[i]];
+ if (value > 63)
+ return -1; // Invalid character in input
+
+ tmp = tmp << 6 | value;
+ if (++bit == 4) {
+ if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full
+ decoded[count++] = (tmp >> 16) & 0xFF;
+ decoded[count++] = (tmp >> 8) & 0xFF;
+ decoded[count++] = tmp & 0xFF;
+ tmp = 0;
+ bit = 0;
+ }
+ }
+
+ if (bit > 0 && count + 1 < decoded_size) {
+ tmp <<= 6 * (4 - bit);
+ if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF;
+ if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF;
+ }
+
+ decoded[count] = '\0'; // Null terminate the output string
+ return count;
+}
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index 53b93e1e1a..b8dedf5156 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -836,6 +836,7 @@ extern char *netdata_configured_host_prefix;
#include "yaml.h"
#include "http/http_defs.h"
#include "gorilla/gorilla.h"
+#include "facets/facets.h"
#include "dyn_conf/dyn_conf.h"
// BEWARE: this exists in alarm-notify.sh
@@ -979,6 +980,14 @@ typedef enum {
void timing_action(TIMING_ACTION action, TIMING_STEP step);
int hash256_string(const unsigned char *string, size_t size, char *hash);
+
+extern bool unittest_running;
+#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
+
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running);
+
+int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size);
+
# ifdef __cplusplus
}
# endif
diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h
index 8331ecbbe3..c4bd473609 100644
--- a/libnetdata/socket/socket.h
+++ b/libnetdata/socket/socket.h
@@ -11,27 +11,27 @@
typedef enum web_client_acl {
WEB_CLIENT_ACL_NONE = (0),
- WEB_CLIENT_ACL_NOCHECK = (0), // Don't check anything - this should work on all channels
- WEB_CLIENT_ACL_DASHBOARD = (1 << 0),
- WEB_CLIENT_ACL_REGISTRY = (1 << 1),
- WEB_CLIENT_ACL_BADGE = (1 << 2),
- WEB_CLIENT_ACL_MGMT = (1 << 3),
- WEB_CLIENT_ACL_STREAMING = (1 << 4),
- WEB_CLIENT_ACL_NETDATACONF = (1 << 5),
- WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 6),
- WEB_CLIENT_ACL_SSL_FORCE = (1 << 7),
- WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8),
- WEB_CLIENT_ACL_ACLK = (1 << 9),
- WEB_CLIENT_ACL_WEBRTC = (1 << 10),
- WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 11), // allow unprotected access if bearer is not enabled in netdata
- WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 12), // allow access only if a valid bearer is used
+ WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
+ WEB_CLIENT_ACL_DASHBOARD = (1 << 1),
+ WEB_CLIENT_ACL_REGISTRY = (1 << 2),
+ WEB_CLIENT_ACL_BADGE = (1 << 3),
+ WEB_CLIENT_ACL_MGMT = (1 << 4),
+ WEB_CLIENT_ACL_STREAMING = (1 << 5),
+ WEB_CLIENT_ACL_NETDATACONF = (1 << 6),
+ WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7),
+ WEB_CLIENT_ACL_SSL_FORCE = (1 << 8),
+ WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9),
+ WEB_CLIENT_ACL_ACLK = (1 << 10),
+ WEB_CLIENT_ACL_WEBRTC = (1 << 11),
+ WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
+ WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
} WEB_CLIENT_ACL;
#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)
#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED)
#ifdef NETDATA_DEV_MODE
-#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD
+#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK
#else
#define ACL_DEV_OPEN_ACCESS 0
#endif
diff --git a/libnetdata/worker_utilization/worker_utilization.h b/libnetdata/worker_utilization/worker_utilization.h
index 6745a010bc..cc3f826886 100644
--- a/libnetdata/worker_utilization/worker_utilization.h
+++ b/libnetdata/worker_utilization/worker_utilization.h
@@ -7,7 +7,7 @@
#define WORKER_UTILIZATION_MAX_JOB_TYPES 50
-typedef enum {
+typedef enum __attribute__((packed)) {
WORKER_METRIC_EMPTY = 0,
WORKER_METRIC_IDLE_BUSY = 1,
WORKER_METRIC_ABSOLUTE = 2,
diff --git a/netdata-installer.sh b/netdata-installer.sh
index 6161ca4bad..414ce7cd51 100755
--- a/netdata-installer.sh
+++ b/netdata-installer.sh
@@ -1290,6 +1290,11 @@ if [ "$(id -u)" -eq 0 ]; then
run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
fi
+
+ if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin" ]; then
+ run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
+ run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
+ fi
else
# non-privileged user installation
run chown "${NETDATA_USER}:${NETDATA_GROUP}" "${NETDATA_LOG_DIR}"
diff --git a/registry/registry.c b/registry/registry.c
index 4b6cb2bd1a..0393389ea3 100644
--- a/registry/registry.c
+++ b/registry/registry.c
@@ -58,7 +58,7 @@ static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PER
static inline void registry_json_header(RRDHOST *host, struct web_client *w, const char *action, const char *status) {
buffer_flush(w->response.data);
w->response.data->content_type = CT_APPLICATION_JSON;
- buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
+ buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(w->response.data, "action", action);
buffer_json_member_add_string(w->response.data, "status", status);
buffer_json_member_add_string(w->response.data, "hostname", rrdhost_registry_hostname(host));
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 6a66cbcca6..52025c9fc2 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -874,7 +874,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
sq[0] = '"';
}
- buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_uint64(wb, "api", 1);
buffer_json_member_add_string(wb, "id", qt->id);
@@ -1289,7 +1289,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
sq[0] = '\'';
}
- buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_uint64(wb, "api", 2);
if(options & RRDR_OPTION_DEBUG) {
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index ca3a41aae2..f0c0c39ba7 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -38,8 +38,6 @@ typedef enum {
#define HOSTNAME_MAX 1024
-#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
-
#define DATASOURCE_FORMAT_JSON "json"
#define DATASOURCE_FORMAT_JSON2 "json2"
#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 8e1af4d63b..388dacee9a 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -2075,88 +2075,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
}
#endif // NETDATA_INTERNAL_CHECKS
-// Returns 1 if an absolute period was requested or 0 if it was a relative period
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) {
- time_t now = now_realtime_sec() - 1;
-
- if(now_ptr)
- *now_ptr = now;
-
- int absolute_period_requested = -1;
- long long after_requested, before_requested;
-
- before_requested = *before;
- after_requested = *after;
-
- // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
- if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
- // if the user asked for a positive relative time,
- // flip it to a negative
- if(before_requested > 0)
- before_requested = -before_requested;
-
- before_requested = now + before_requested;
- absolute_period_requested = 0;
- }
-
- // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
- if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
- if(after_requested > 0)
- after_requested = -after_requested;
-
- // if the user didn't give an after, use the number of points
- // to give a sane default
- if(after_requested == 0)
- after_requested = -600;
-
- // since the query engine now returns inclusive timestamps
- // it is awkward to return 6 points when after=-5 is given
- // so for relative queries we add 1 second, to give
- // more predictable results to users.
- after_requested = before_requested + after_requested + 1;
- absolute_period_requested = 0;
- }
-
- if(absolute_period_requested == -1)
- absolute_period_requested = 1;
-
- // check if the parameters are flipped
- if(after_requested > before_requested) {
- long long t = before_requested;
- before_requested = after_requested;
- after_requested = t;
- }
-
- // if the query requests future data
- // shift the query back to be in the present time
- // (this may also happen because of the rules above)
- if(before_requested > now) {
- long long delta = before_requested - now;
- before_requested -= delta;
- after_requested -= delta;
- }
-
- time_t absolute_minimum_time = now - (10 * 365 * 86400);
- time_t absolute_maximum_time = now + (1 * 365 * 86400);
-
- if (after_requested < absolute_minimum_time && !unittest_running)
- after_requested = absolute_minimum_time;
-
- if (after_requested > absolute_maximum_time && !unittest_running)
- after_requested = absolute_maximum_time;
-
- if (before_requested < absolute_minimum_time && !unittest_running)
- before_requested = absolute_minimum_time;
-
- if (before_requested > absolute_maximum_time && !unittest_running)
- before_requested = absolute_maximum_time;
-
- *before = before_requested;
- *after = after_requested;
-
- return (absolute_period_requested != 1);
-}
-
// #define DEBUG_QUERY_LOGIC 1
#ifdef DEBUG_QUERY_LOGIC
@@ -2283,7 +2201,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
}
// convert our before_wanted and after_wanted to absolute
- rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
+ rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL, unittest_running);
query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index c4a1f83f23..e02e006752 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -206,8 +206,6 @@ RRDR *rrd2rrdr_legacy(
RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
bool query_target_calculate_window(struct query_target *qt);
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr);
-
#ifdef __cplusplus
}
#endif
diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c
index 26d4555fa7..2782aef604 100644
--- a/web/api/queries/weights.c
+++ b/web/api/queries/weights.c
@@ -169,7 +169,7 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
size_t examined_dimensions, usec_t duration,
WEIGHTS_STATS *stats) {
- buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -221,7 +221,7 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
size_t examined_dimensions, usec_t duration,
WEIGHTS_STATS *stats) {
- buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -739,7 +739,7 @@ static size_t registered_results_to_json_multinode_no_group_by(
size_t examined_dimensions, struct query_weights_data *qwd,
WEIGHTS_STATS *stats,
struct query_versions *versions) {
- buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_uint64(wb, "api", 2);
results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -958,7 +958,7 @@ static size_t registered_results_to_json_multinode_group_by(
size_t examined_dimensions, struct query_weights_data *qwd,
WEIGHTS_STATS *stats,
struct query_versions *versions) {
- buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_uint64(wb, "api", 2);
results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -1806,7 +1806,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
}
};
- if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))
+ if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL, false))
buffer_no_cacheable(wb);
else
buffer_cacheable(wb);
@@ -1823,7 +1823,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
if(qwr->baseline_before <= API_RELATIVE_TIME_MAX)
qwr->baseline_before += qwr->after;
- rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL);
+ rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL, false);
if (qwr->baseline_before <= qwr->baseline_after) {
resp = HTTP_RESP_BAD_REQUEST;
diff --git a/web/api/web_api.c b/web/api/web_api.c
index e2558b47c9..7a4704bd58 100644
--- a/web/api/web_api.c
+++ b/web/api/web_api.c
@@ -6,7 +6,7 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at
DICTIONARY *netdata_authorized_bearers = NULL;
static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {
- if(endpoint_acl == WEB_CLIENT_ACL_NOCHECK)
+ if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK))
// the endpoint is totally public
return true;
@@ -23,11 +23,24 @@ static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL
// endpoint does not require a bearer
return true;
- if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)) || api_check_bearer_token(w))
+ if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)))
// the request is coming from ACLK or WEBRTC (authorized already),
- // or we have a valid bearer on the request
return true;
+ // at this point we need a bearer to serve the request
+ // either because:
+ //
+ // 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or
+ // 2. netdata_is_protected_by_bearer == true
+ //
+
+ BEARER_STATUS t = api_check_bearer_token(w);
+ if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED)
+ // we have a valid bearer on the request
+ return true;
+
+ netdata_log_info("BEARER: bearer is required for request: code %d", t);
+
return false;
}
@@ -60,7 +73,7 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat
return HTTP_RESP_BAD_REQUEST;
}
- if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
+ if(unlikely(!web_client_check_acl_and_bearer(w, api_commands[i].acl)))
return web_client_permission_denied(w);
char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
diff --git a/web/api/web_api.h b/web/api/web_api.h
index 53bebfd269..f7ae45ad0c 100644
--- a/web/api/web_api.h
+++ b/web/api/web_api.h
@@ -11,9 +11,19 @@
extern bool netdata_is_protected_by_bearer;
extern DICTIONARY *netdata_authorized_bearers;
-bool api_check_bearer_token(struct web_client *w);
-bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
-void bearer_tokens_init(void);
+typedef enum __attribute__((packed)) {
+ BEARER_STATUS_NO_BEARER_IN_HEADERS,
+ BEARER_STATUS_BEARER_DOES_NOT_FIT,
+ BEARER_STATUS_NOT_PARSABLE,
+ BEARER_STATUS_EXTRACTED_FROM_HEADER,
+ BEARER_STATUS_NO_BEARERS_DICTIONARY,
+ BEARER_STATUS_NOT_FOUND_IN_DICTIONARY,
+ BEARER_STATUS_EXPIRED,
+ BEARER_STATUS_AVAILABLE_AND_VALIDATED,
+} BEARER_STATUS;
+
+BEARER_STATUS api_check_bearer_token(struct web_client *w);
+BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
struct web_api_command {
const char *command;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index d730e42d78..60962413d6 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -941,7 +941,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");
if(cookie)
strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);
- else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))
+ else if(extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)) != BEARER_STATUS_EXTRACTED_FROM_HEADER)
person_guid[0] = '\0';
char action = '\0';
@@ -1199,7 +1199,7 @@ static void host_collectors(RRDHOST *host, BUFFER *wb) {
extern int aclk_connected;
inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) {
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
buffer_json_member_add_string(wb, "uid", host->machine_guid);
@@ -1319,7 +1319,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
buffer_flush(wb);
wb->content_type = CT_APPLICATION_JSON;
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
ml_host_get_detection_info(host, wb);
buffer_json_finalize(wb);
@@ -1424,7 +1424,7 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
wb->content_type = CT_APPLICATION_JSON;
buffer_no_cacheable(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
host_functions2json(host, wb);
buffer_json_finalize(wb);
diff --git a/web/api/web_api_v2.c b/web/api/web_api_v2.c
index e9b8d801d0..850282121e 100644
--- a/web/api/web_api_v2.c
+++ b/web/api/web_api_v2.c
@@ -54,13 +54,13 @@ static time_t bearer_get_token(uuid_t *uuid) {
#define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "
-bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
+BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
const char *req = buffer_tostring(w->response.data);
size_t req_len = buffer_strlen(w->response.data);
const char *bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);
if(!bearer)
- return false;
+ return BEARER_STATUS_NO_BEARER_IN_HEADERS;
const char *token_start = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;
@@ -69,26 +69,33 @@ bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t d
const char *token_end = token_start + UUID_STR_LEN - 1 + 2;
if (token_end > req + req_len)
- return false;
+ return BEARER_STATUS_BEARER_DOES_NOT_FIT;
strncpyz(dst, token_start, dst_len - 1);
uuid_t uuid;
if (uuid_parse(dst, uuid) != 0)
- return false;
+ return BEARER_STATUS_NOT_PARSABLE;
- return true;
+ return BEARER_STATUS_EXTRACTED_FROM_HEADER;
}
-bool api_check_bearer_token(struct web_client *w) {
+BEARER_STATUS api_check_bearer_token(struct web_client *w) {
if(!netdata_authorized_bearers)
- return false;
+ return BEARER_STATUS_NO_BEARERS_DICTIONARY;
char token[UUID_STR_LEN];
- if(!extract_bearer_token_from_request(w, token, sizeof(token)))
- return false;
+ BEARER_STATUS t = extract_bearer_token_from_request(w, token, sizeof(token));
+ if(t != BEARER_STATUS_EXTRACTED_FROM_HEADER)
+ return t;
struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);
- return z && z->expires_s > now_monotonic_sec();
+ if(!z)
+ return BEARER_STATUS_NOT_FOUND_IN_DICTIONARY;
+
+ if(z->expires_s < now_monotonic_sec())
+ return BEARER_STATUS_EXPIRED;
+
+ return BEARER_STATUS_AVAILABLE_AND_VALIDATED;
}
static bool verify_agent_uuids(const char *machine_guid, const char *node_id, const char *claim_id) {
@@ -153,7 +160,7 @@ int api_v2_bearer_protection(RRDHOST *host __maybe_unused, struct web_client *w
BUFFER *wb = w->response.data;
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
buffer_json_finalize(wb);
@@ -192,7 +199,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
BUFFER *wb = w->response.data;
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
buffer_json_member_add_uuid(wb, "token", &uuid);
diff --git a/web/rtc/webrtc.c b/web/rtc/webrtc.c
index 331326c84f..45e5e0dac6 100644
--- a/web/rtc/webrtc.c
+++ b/web/rtc/webrtc.c
@@ -304,7 +304,7 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz
web_client_timeout_checkpoint_set(w, 0);
web_client_decode_path_and_query_string(w, path);
path = (char *)buffer_tostring(w->url_path_decoded);
- w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path);
+ w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
web_client_timeout_checkpoint_response_ready(w, NULL);
size_t sent_bytes = 0;
@@ -643,7 +643,7 @@ int webrtc_new_connection(const char *sdp, BUFFER *wb) {
}
buffer_flush(wb);
- buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
wb->content_type = CT_APPLICATION_JSON;
WEBRTC_CONN *conn = webrtc_create_connection();