mirror of
https://github.com/netdata/netdata.git
synced 2025-04-28 06:32:30 +00:00
systemd-journal plugin (#15363)
This commit is contained in:
Costa Tsaousis
GitHub
parent
a833f6674f
commit
ce75313de0
40 changed files with 1879 additions and 224 deletions
Makefile.am
claim
collectors
configure.accontrib/debian
daemon
database
libnetdata
netdata-installer.shregistry
web
16
Makefile.am
16
Makefile.am
|
|
@ -154,6 +154,8 @@ LIBNETDATA_FILES = \
|
|||
libnetdata/dictionary/dictionary.h \
|
||||
libnetdata/eval/eval.c \
|
||||
libnetdata/eval/eval.h \
|
||||
libnetdata/facets/facets.c \
|
||||
libnetdata/facets/facets.h \
|
||||
libnetdata/gorilla/gorilla.h \
|
||||
libnetdata/gorilla/gorilla.cc \
|
||||
libnetdata/inlined.h \
|
||||
|
|
@ -302,6 +304,11 @@ FREEIPMI_PLUGIN_FILES = \
|
|||
$(LIBNETDATA_FILES) \
|
||||
$(NULL)
|
||||
|
||||
SYSTEMD_JOURNAL_PLUGIN_FILES = \
|
||||
collectors/systemd-journal.plugin/systemd-journal.c \
|
||||
$(LIBNETDATA_FILES) \
|
||||
$(NULL)
|
||||
|
||||
CUPS_PLUGIN_FILES = \
|
||||
collectors/cups.plugin/cups_plugin.c \
|
||||
$(LIBNETDATA_FILES) \
|
||||
|
|
@ -1232,6 +1239,15 @@ if ENABLE_PLUGIN_FREEIPMI
|
|||
$(NULL)
|
||||
endif
|
||||
|
||||
if ENABLE_PLUGIN_SYSTEMD_JOURNAL
|
||||
plugins_PROGRAMS += systemd-journal.plugin
|
||||
systemd_journal_plugin_SOURCES = $(SYSTEMD_JOURNAL_PLUGIN_FILES)
|
||||
systemd_journal_plugin_LDADD = \
|
||||
$(NETDATA_COMMON_LIBS) \
|
||||
$(OPTIONAL_SYSTEMD_LIBS) \
|
||||
$(NULL)
|
||||
endif
|
||||
|
||||
if ENABLE_PLUGIN_EBPF
|
||||
plugins_PROGRAMS += ebpf.plugin
|
||||
ebpf_plugin_SOURCES = $(EBPF_PLUGIN_FILES)
|
||||
|
|
|
|||
|
|
@ -356,7 +356,7 @@ int api_v2_claim(struct web_client *w, char *url) {
|
|||
|
||||
BUFFER *wb = w->response.data;
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
time_t now_s = now_realtime_sec();
|
||||
CLOUD_STATUS status = buffer_json_cloud_status(wb, now_s);
|
||||
|
|
@ -462,7 +462,7 @@ int api_v2_claim(struct web_client *w, char *url) {
|
|||
// our status may have changed
|
||||
// refresh the status in our output
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
now_s = now_realtime_sec();
|
||||
buffer_json_cloud_status(wb, now_s);
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ SUBDIRS = \
|
|||
statsd.plugin \
|
||||
ebpf.plugin \
|
||||
tc.plugin \
|
||||
systemd-journal.plugin \
|
||||
$(NULL)
|
||||
|
||||
usercustompluginsconfigdir=$(configdir)/custom-plugins.d
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
#define APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION "Detailed information on the currently running processes."
|
||||
|
||||
#define APPS_PLUGIN_FUNCTIONS() do { \
|
||||
fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" 10 \"%s\"\n", APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
|
||||
fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
|
||||
} while(0)
|
||||
|
||||
|
||||
|
|
@ -4572,7 +4572,7 @@ static int check_capabilities() {
|
|||
}
|
||||
#endif
|
||||
|
||||
netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
|
||||
static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
|
||||
|
||||
#define PROCESS_FILTER_CATEGORY "category:"
|
||||
#define PROCESS_FILTER_USER "user:"
|
||||
|
|
@ -4625,15 +4625,6 @@ static void get_MemTotal(void) {
|
|||
#endif
|
||||
}
|
||||
|
||||
static void apps_plugin_function_error(const char *transaction, int code, const char *msg) {
|
||||
char buffer[PLUGINSD_LINE_MAX + 1];
|
||||
json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
|
||||
|
||||
pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
|
||||
fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
|
||||
pluginsd_function_result_end_to_stdout();
|
||||
}
|
||||
|
||||
static void apps_plugin_function_processes_help(const char *transaction) {
|
||||
pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
|
||||
fprintf(stdout, "%s",
|
||||
|
|
@ -4681,7 +4672,7 @@ static void apps_plugin_function_processes_help(const char *transaction) {
|
|||
buffer_json_add_array_item_double(wb, _tmp); \
|
||||
} while(0)
|
||||
|
||||
static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
|
||||
static void function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
|
||||
struct pid_stat *p;
|
||||
|
||||
char *words[PLUGINSD_MAX_WORDS] = { NULL };
|
||||
|
|
@ -4702,21 +4693,21 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
|
|||
if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {
|
||||
category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);
|
||||
if(!category) {
|
||||
apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
|
||||
user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
|
||||
if(!user) {
|
||||
apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {
|
||||
group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);
|
||||
if(!group) {
|
||||
apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -4742,7 +4733,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
|
|||
else {
|
||||
char msg[PLUGINSD_LINE_MAX];
|
||||
snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
|
||||
apps_plugin_function_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -4755,7 +4746,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
|
|||
unsigned int io_divisor = 1024 * RATES_DETAIL;
|
||||
|
||||
BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
|
||||
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
|
||||
buffer_json_member_add_string(wb, "type", "table");
|
||||
buffer_json_member_add_time_t(wb, "update_every", update_every);
|
||||
|
|
@ -5232,7 +5223,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
|
|||
RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "Uptime", "Uptime in seconds", RRDF_FIELD_TYPE_DURATION,
|
||||
RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION, 2,
|
||||
RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_DURATION_S, 2,
|
||||
"seconds", Uptime_max, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MAX,
|
||||
RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
|
@ -5532,9 +5523,9 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
|
|||
pluginsd_function_result_end_to_stdout();
|
||||
}
|
||||
|
||||
bool apps_plugin_exit = false;
|
||||
static bool apps_plugin_exit = false;
|
||||
|
||||
void *reader_main(void *arg __maybe_unused) {
|
||||
static void *reader_main(void *arg __maybe_unused) {
|
||||
char buffer[PLUGINSD_LINE_MAX + 1];
|
||||
|
||||
char *s = NULL;
|
||||
|
|
@ -5566,9 +5557,9 @@ void *reader_main(void *arg __maybe_unused) {
|
|||
netdata_mutex_lock(&mutex);
|
||||
|
||||
if(strncmp(function, "processes", strlen("processes")) == 0)
|
||||
apps_plugin_function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
|
||||
function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
|
||||
else
|
||||
apps_plugin_function_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");
|
||||
|
||||
fflush(stdout);
|
||||
netdata_mutex_unlock(&mutex);
|
||||
|
|
@ -5696,6 +5687,8 @@ int main(int argc, char **argv) {
|
|||
netdata_thread_create(&reader_thread, "APPS_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
|
||||
netdata_mutex_lock(&mutex);
|
||||
|
||||
APPS_PLUGIN_FUNCTIONS();
|
||||
|
||||
usec_t step = update_every * USEC_PER_SEC;
|
||||
global_iterations_counter = 1;
|
||||
heartbeat_t hb;
|
||||
|
|
|
|||
|
|
@ -206,7 +206,7 @@ static void ebpf_function_thread_manipulation(const char *transaction,
|
|||
time_t expires = now_realtime_sec() + em->update_every;
|
||||
|
||||
BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
|
||||
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
|
||||
buffer_json_member_add_string(wb, "type", "table");
|
||||
buffer_json_member_add_time_t(wb, "update_every", em->update_every);
|
||||
|
|
|
|||
|
|
@ -99,8 +99,6 @@ void pluginsd_process_thread_cleanup(void *ptr);
|
|||
|
||||
size_t pluginsd_initialize_plugin_directories();
|
||||
|
||||
|
||||
|
||||
#define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
|
||||
buffer_sprintf(wb \
|
||||
, PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
|
||||
|
|
@ -125,4 +123,13 @@ size_t pluginsd_initialize_plugin_directories();
|
|||
#define pluginsd_function_result_end_to_stdout() \
|
||||
fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
|
||||
|
||||
static inline void pluginsd_function_json_error(const char *transaction, int code, const char *msg) {
|
||||
char buffer[PLUGINSD_LINE_MAX + 1];
|
||||
json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
|
||||
|
||||
pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
|
||||
fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
|
||||
pluginsd_function_result_end_to_stdout();
|
||||
}
|
||||
|
||||
#endif /* NETDATA_PLUGINS_D_H */
|
||||
|
|
|
|||
0
collectors/systemd-journal.plugin/Makefile.am
Normal file
0
collectors/systemd-journal.plugin/Makefile.am
Normal file
0
collectors/systemd-journal.plugin/README.md
Normal file
0
collectors/systemd-journal.plugin/README.md
Normal file
578
collectors/systemd-journal.plugin/systemd-journal.c
Normal file
578
collectors/systemd-journal.plugin/systemd-journal.c
Normal file
|
|
@ -0,0 +1,578 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
/*
|
||||
* netdata systemd-journal.plugin
|
||||
* Copyright (C) 2023 Netdata Inc.
|
||||
* GPL v3+
|
||||
*/
|
||||
|
||||
// TODO - 1) MARKDOC
|
||||
|
||||
#include "collectors/all.h"
|
||||
#include "libnetdata/libnetdata.h"
|
||||
#include "libnetdata/required_dummies.h"
|
||||
|
||||
#include <systemd/sd-journal.h>
|
||||
#include <syslog.h>
|
||||
|
||||
#define FACET_MAX_VALUE_LENGTH 8192
|
||||
|
||||
#define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
|
||||
#define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal"
|
||||
#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 30
|
||||
#define SYSTEMD_JOURNAL_MAX_PARAMS 100
|
||||
#define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (3 * 3600)
|
||||
#define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200
|
||||
|
||||
#define JOURNAL_PARAMETER_HELP "help"
|
||||
#define JOURNAL_PARAMETER_AFTER "after"
|
||||
#define JOURNAL_PARAMETER_BEFORE "before"
|
||||
#define JOURNAL_PARAMETER_ANCHOR "anchor"
|
||||
#define JOURNAL_PARAMETER_LAST "last"
|
||||
#define JOURNAL_PARAMETER_QUERY "query"
|
||||
|
||||
#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
|
||||
#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL
|
||||
#define SYSTEMD_KEYS_INCLUDED_IN_FACETS \
|
||||
"_TRANSPORT" \
|
||||
"|SYSLOG_IDENTIFIER" \
|
||||
"|SYSLOG_FACILITY" \
|
||||
"|PRIORITY" \
|
||||
"|_HOSTNAME" \
|
||||
"|_RUNTIME_SCOPE" \
|
||||
"|_PID" \
|
||||
"|_UID" \
|
||||
"|_GID" \
|
||||
"|_SYSTEMD_UNIT" \
|
||||
"|_SYSTEMD_SLICE" \
|
||||
"|_SYSTEMD_USER_SLICE" \
|
||||
"|_COMM" \
|
||||
"|_EXE" \
|
||||
"|_SYSTEMD_CGROUP" \
|
||||
"|_SYSTEMD_USER_UNIT" \
|
||||
"|USER_UNIT" \
|
||||
"|UNIT" \
|
||||
""
|
||||
|
||||
static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;
|
||||
static bool plugin_should_exit = false;
|
||||
|
||||
DICTIONARY *uids = NULL;
|
||||
DICTIONARY *gids = NULL;
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t stop_monotonic_ut) {
|
||||
sd_journal *j;
|
||||
int r;
|
||||
|
||||
// Open the system journal for reading
|
||||
r = sd_journal_open(&j, SD_JOURNAL_ALL_NAMESPACES);
|
||||
if (r < 0)
|
||||
return HTTP_RESP_INTERNAL_SERVER_ERROR;
|
||||
|
||||
facets_rows_begin(facets);
|
||||
|
||||
bool timed_out = false;
|
||||
size_t row_counter = 0;
|
||||
sd_journal_seek_realtime_usec(j, before_ut);
|
||||
SD_JOURNAL_FOREACH_BACKWARDS(j) {
|
||||
row_counter++;
|
||||
|
||||
uint64_t msg_ut;
|
||||
sd_journal_get_realtime_usec(j, &msg_ut);
|
||||
if (msg_ut < after_ut)
|
||||
break;
|
||||
|
||||
const void *data;
|
||||
size_t length;
|
||||
SD_JOURNAL_FOREACH_DATA(j, data, length) {
|
||||
const char *key = data;
|
||||
const char *equal = strchr(key, '=');
|
||||
if(unlikely(!equal))
|
||||
continue;
|
||||
|
||||
const char *value = ++equal;
|
||||
size_t key_length = value - key; // including '\0'
|
||||
|
||||
char key_copy[key_length];
|
||||
memcpy(key_copy, key, key_length - 1);
|
||||
key_copy[key_length - 1] = '\0';
|
||||
|
||||
size_t value_length = length - key_length; // without '\0'
|
||||
facets_add_key_value_length(facets, key_copy, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);
|
||||
}
|
||||
|
||||
facets_row_finished(facets, msg_ut);
|
||||
|
||||
if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {
|
||||
timed_out = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sd_journal_close(j);
|
||||
|
||||
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
|
||||
buffer_json_member_add_boolean(wb, "partial", timed_out);
|
||||
buffer_json_member_add_string(wb, "type", "table");
|
||||
buffer_json_member_add_time_t(wb, "update_every", 1);
|
||||
buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
|
||||
|
||||
facets_report(facets, wb);
|
||||
|
||||
buffer_json_member_add_time_t(wb, "expires", now_realtime_sec());
|
||||
buffer_json_finalize(wb);
|
||||
|
||||
return HTTP_RESP_OK;
|
||||
}
|
||||
|
||||
static void systemd_journal_function_help(const char *transaction) {
|
||||
pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);
|
||||
fprintf(stdout,
|
||||
"%s / %s\n"
|
||||
"\n"
|
||||
"%s\n"
|
||||
"\n"
|
||||
"The following filters are supported:\n"
|
||||
"\n"
|
||||
" help\n"
|
||||
" Shows this help message.\n"
|
||||
"\n"
|
||||
" before:TIMESTAMP\n"
|
||||
" Absolute or relative (to now) timestamp in seconds, to start the query.\n"
|
||||
" The query is always executed from the most recent to the oldest log entry.\n"
|
||||
" If not given the default is: now.\n"
|
||||
"\n"
|
||||
" after:TIMESTAMP\n"
|
||||
" Absolute or relative (to `before`) timestamp in seconds, to end the query.\n"
|
||||
" If not given, the default is %d.\n"
|
||||
"\n"
|
||||
" last:ITEMS\n"
|
||||
" The number of items to return.\n"
|
||||
" The default is %d.\n"
|
||||
"\n"
|
||||
" anchor:NUMBER\n"
|
||||
" The `timestamp` of the item last received, to return log entries after that.\n"
|
||||
" If not given, the query will return the top `ITEMS` from the most recent.\n"
|
||||
"\n"
|
||||
" facet_id:value_id1,value_id2,value_id3,...\n"
|
||||
" Apply filters to the query, based on the facet IDs returned.\n"
|
||||
" Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n"
|
||||
"\n"
|
||||
"Filters can be combined. Each filter can be given only one time.\n"
|
||||
, program_name
|
||||
, SYSTEMD_JOURNAL_FUNCTION_NAME
|
||||
, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION
|
||||
, -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION
|
||||
, SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY
|
||||
);
|
||||
pluginsd_function_result_end_to_stdout();
|
||||
}
|
||||
|
||||
static const char *syslog_facility_to_name(int facility) {
|
||||
switch (facility) {
|
||||
case LOG_FAC(LOG_KERN): return "kern";
|
||||
case LOG_FAC(LOG_USER): return "user";
|
||||
case LOG_FAC(LOG_MAIL): return "mail";
|
||||
case LOG_FAC(LOG_DAEMON): return "daemon";
|
||||
case LOG_FAC(LOG_AUTH): return "auth";
|
||||
case LOG_FAC(LOG_SYSLOG): return "syslog";
|
||||
case LOG_FAC(LOG_LPR): return "lpr";
|
||||
case LOG_FAC(LOG_NEWS): return "news";
|
||||
case LOG_FAC(LOG_UUCP): return "uucp";
|
||||
case LOG_FAC(LOG_CRON): return "cron";
|
||||
case LOG_FAC(LOG_AUTHPRIV): return "authpriv";
|
||||
case LOG_FAC(LOG_FTP): return "ftp";
|
||||
case LOG_FAC(LOG_LOCAL0): return "local0";
|
||||
case LOG_FAC(LOG_LOCAL1): return "local1";
|
||||
case LOG_FAC(LOG_LOCAL2): return "local2";
|
||||
case LOG_FAC(LOG_LOCAL3): return "local3";
|
||||
case LOG_FAC(LOG_LOCAL4): return "local4";
|
||||
case LOG_FAC(LOG_LOCAL5): return "local5";
|
||||
case LOG_FAC(LOG_LOCAL6): return "local6";
|
||||
case LOG_FAC(LOG_LOCAL7): return "local7";
|
||||
default: return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static const char *syslog_priority_to_name(int priority) {
|
||||
switch (priority) {
|
||||
case LOG_ALERT: return "alert";
|
||||
case LOG_CRIT: return "critical";
|
||||
case LOG_DEBUG: return "debug";
|
||||
case LOG_EMERG: return "panic";
|
||||
case LOG_ERR: return "error";
|
||||
case LOG_INFO: return "info";
|
||||
case LOG_NOTICE: return "notice";
|
||||
case LOG_WARNING: return "warning";
|
||||
default: return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static char *uid_to_username(uid_t uid, char *buffer, size_t buffer_size) {
|
||||
struct passwd pw, *result;
|
||||
char tmp[1024 + 1];
|
||||
|
||||
if (getpwuid_r(uid, &pw, tmp, 1024, &result) != 0 || result == NULL)
|
||||
return NULL;
|
||||
|
||||
strncpy(buffer, pw.pw_name, buffer_size - 1);
|
||||
buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static char *gid_to_groupname(gid_t gid, char* buffer, size_t buffer_size) {
|
||||
struct group grp, *result;
|
||||
char tmp[1024 + 1];
|
||||
|
||||
if (getgrgid_r(gid, &grp, tmp, 1024, &result) != 0 || result == NULL)
|
||||
return NULL;
|
||||
|
||||
strncpy(buffer, grp.gr_name, buffer_size - 1);
|
||||
buffer[buffer_size - 1] = '\0'; // Null-terminate just in case
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static void systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
|
||||
const char *v = buffer_tostring(wb);
|
||||
if(*v && isdigit(*v)) {
|
||||
int facility = str2i(buffer_tostring(wb));
|
||||
const char *name = syslog_facility_to_name(facility);
|
||||
if (name) {
|
||||
buffer_flush(wb);
|
||||
buffer_json_add_array_item_string(wb, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {
|
||||
const char *v = buffer_tostring(wb);
|
||||
if(*v && isdigit(*v)) {
|
||||
int priority = str2i(buffer_tostring(wb));
|
||||
const char *name = syslog_priority_to_name(priority);
|
||||
if (name) {
|
||||
buffer_flush(wb);
|
||||
buffer_json_add_array_item_string(wb, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
|
||||
DICTIONARY *cache = data;
|
||||
const char *v = buffer_tostring(wb);
|
||||
if(*v && isdigit(*v)) {
|
||||
const char *sv = dictionary_get(cache, v);
|
||||
if(!sv) {
|
||||
char buf[1024 + 1];
|
||||
int uid = str2i(buffer_tostring(wb));
|
||||
const char *name = uid_to_username(uid, buf, 1024);
|
||||
if (!name)
|
||||
name = v;
|
||||
|
||||
sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
|
||||
}
|
||||
|
||||
buffer_flush(wb);
|
||||
buffer_strcat(wb, sv);
|
||||
}
|
||||
}
|
||||
|
||||
static void systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {
|
||||
DICTIONARY *cache = data;
|
||||
const char *v = buffer_tostring(wb);
|
||||
if(*v && isdigit(*v)) {
|
||||
const char *sv = dictionary_get(cache, v);
|
||||
if(!sv) {
|
||||
char buf[1024 + 1];
|
||||
int gid = str2i(buffer_tostring(wb));
|
||||
const char *name = gid_to_groupname(gid, buf, 1024);
|
||||
if (!name)
|
||||
name = v;
|
||||
|
||||
sv = dictionary_set(cache, v, (void *)name, strlen(name) + 1);
|
||||
}
|
||||
|
||||
buffer_flush(wb);
|
||||
buffer_strcat(wb, sv);
|
||||
}
|
||||
}
|
||||
|
||||
static void systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {
|
||||
FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER");
|
||||
FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID");
|
||||
|
||||
const char *identifier = syslog_identifier_rkv ? buffer_tostring(syslog_identifier_rkv->wb) : "UNKNOWN";
|
||||
const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : "UNKNOWN";
|
||||
|
||||
buffer_flush(rkv->wb);
|
||||
buffer_sprintf(rkv->wb, "%s[%s]", identifier, pid);
|
||||
|
||||
buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
|
||||
}
|
||||
|
||||
static void function_systemd_journal(const char *transaction, char *function, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
|
||||
char *words[SYSTEMD_JOURNAL_MAX_PARAMS] = { NULL };
|
||||
size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_JOURNAL_MAX_PARAMS);
|
||||
|
||||
BUFFER *wb = buffer_create(0, NULL);
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);
|
||||
|
||||
FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,
|
||||
SYSTEMD_ALWAYS_VISIBLE_KEYS,
|
||||
SYSTEMD_KEYS_INCLUDED_IN_FACETS,
|
||||
SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);
|
||||
|
||||
facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);
|
||||
facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);
|
||||
facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
|
||||
facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
|
||||
facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
|
||||
|
||||
// register the fields in the order you want them on the dashboard
|
||||
|
||||
facets_register_dynamic_key(facets, "ND_JOURNAL_PROCESS", FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS,
|
||||
systemd_journal_dynamic_row_id, NULL);
|
||||
|
||||
facets_register_key(facets, "MESSAGE",
|
||||
FACET_KEY_OPTION_NO_FACET|FACET_KEY_OPTION_MAIN_TEXT|FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_FTS);
|
||||
|
||||
facets_register_key_transformation(facets, "PRIORITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
|
||||
systemd_journal_transform_priority, NULL);
|
||||
|
||||
facets_register_key_transformation(facets, "SYSLOG_FACILITY", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
|
||||
systemd_journal_transform_syslog_facility, NULL);
|
||||
|
||||
facets_register_key(facets, "SYSLOG_IDENTIFIER", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
|
||||
facets_register_key(facets, "UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
|
||||
facets_register_key(facets, "USER_UNIT", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS);
|
||||
|
||||
facets_register_key_transformation(facets, "_UID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
|
||||
systemd_journal_transform_uid, uids);
|
||||
|
||||
facets_register_key_transformation(facets, "_GID", FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS,
|
||||
systemd_journal_transform_gid, gids);
|
||||
|
||||
time_t after_s = 0, before_s = 0;
|
||||
usec_t anchor = 0;
|
||||
size_t last = 0;
|
||||
const char *query = NULL;
|
||||
|
||||
buffer_json_member_add_object(wb, "request");
|
||||
buffer_json_member_add_object(wb, "filters");
|
||||
|
||||
for(int i = 1; i < SYSTEMD_JOURNAL_MAX_PARAMS ;i++) {
|
||||
const char *keyword = get_word(words, num_words, i);
|
||||
if(!keyword) break;
|
||||
|
||||
if(strcmp(keyword, JOURNAL_PARAMETER_HELP) == 0) {
|
||||
systemd_journal_function_help(transaction);
|
||||
goto cleanup;
|
||||
}
|
||||
else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", strlen(JOURNAL_PARAMETER_AFTER ":")) == 0) {
|
||||
after_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_AFTER ":")]);
|
||||
}
|
||||
else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", strlen(JOURNAL_PARAMETER_BEFORE ":")) == 0) {
|
||||
before_s = str2l(&keyword[strlen(JOURNAL_PARAMETER_BEFORE ":")]);
|
||||
}
|
||||
else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", strlen(JOURNAL_PARAMETER_ANCHOR ":")) == 0) {
|
||||
anchor = str2ull(&keyword[strlen(JOURNAL_PARAMETER_ANCHOR ":")], NULL);
|
||||
}
|
||||
else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", strlen(JOURNAL_PARAMETER_LAST ":")) == 0) {
|
||||
last = str2ul(&keyword[strlen(JOURNAL_PARAMETER_LAST ":")]);
|
||||
}
|
||||
else if(strncmp(keyword, JOURNAL_PARAMETER_QUERY ":", strlen(JOURNAL_PARAMETER_QUERY ":")) == 0) {
|
||||
query= &keyword[strlen(JOURNAL_PARAMETER_QUERY ":")];
|
||||
}
|
||||
else {
|
||||
char *value = strchr(keyword, ':');
|
||||
if(value) {
|
||||
*value++ = '\0';
|
||||
|
||||
buffer_json_member_add_array(wb, keyword);
|
||||
|
||||
while(value) {
|
||||
char *sep = strchr(value, ',');
|
||||
if(sep)
|
||||
*sep++ = '\0';
|
||||
|
||||
facets_register_facet_filter(facets, keyword, value, FACET_KEY_OPTION_REORDER);
|
||||
buffer_json_add_array_item_string(wb, value);
|
||||
|
||||
value = sep;
|
||||
}
|
||||
|
||||
buffer_json_array_close(wb); // keyword
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer_json_object_close(wb); // filters
|
||||
|
||||
time_t expires = now_realtime_sec() + 1;
|
||||
time_t now_s;
|
||||
|
||||
if(!after_s && !before_s) {
|
||||
now_s = now_realtime_sec();
|
||||
before_s = now_s;
|
||||
after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
|
||||
}
|
||||
else
|
||||
rrdr_relative_window_to_absolute(&after_s, &before_s, &now_s, false);
|
||||
|
||||
if(after_s > before_s) {
|
||||
time_t tmp = after_s;
|
||||
after_s = before_s;
|
||||
before_s = tmp;
|
||||
}
|
||||
|
||||
if(after_s == before_s)
|
||||
after_s = before_s - SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION;
|
||||
|
||||
if(!last)
|
||||
last = SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY;
|
||||
|
||||
buffer_json_member_add_time_t(wb, "after", after_s);
|
||||
buffer_json_member_add_time_t(wb, "before", before_s);
|
||||
buffer_json_member_add_uint64(wb, "anchor", anchor);
|
||||
buffer_json_member_add_uint64(wb, "last", last);
|
||||
buffer_json_member_add_string(wb, "query", query);
|
||||
buffer_json_member_add_time_t(wb, "timeout", timeout);
|
||||
buffer_json_object_close(wb); // request
|
||||
|
||||
facets_set_items(facets, last);
|
||||
facets_set_anchor(facets, anchor);
|
||||
facets_set_query(facets, query);
|
||||
int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,
|
||||
now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
|
||||
|
||||
if(response != HTTP_RESP_OK) {
|
||||
pluginsd_function_json_error(transaction, response, "failed");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);
|
||||
fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
|
||||
|
||||
pluginsd_function_result_end_to_stdout();
|
||||
|
||||
cleanup:
|
||||
facets_destroy(facets);
|
||||
buffer_free(wb);
|
||||
}
|
||||
|
||||
static void *reader_main(void *arg __maybe_unused) {
|
||||
char buffer[PLUGINSD_LINE_MAX + 1];
|
||||
|
||||
char *s = NULL;
|
||||
while(!plugin_should_exit && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
|
||||
|
||||
char *words[PLUGINSD_MAX_WORDS] = { NULL };
|
||||
size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, PLUGINSD_MAX_WORDS);
|
||||
|
||||
const char *keyword = get_word(words, num_words, 0);
|
||||
|
||||
if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
|
||||
char *transaction = get_word(words, num_words, 1);
|
||||
char *timeout_s = get_word(words, num_words, 2);
|
||||
char *function = get_word(words, num_words, 3);
|
||||
|
||||
if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
|
||||
netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
|
||||
keyword,
|
||||
transaction?transaction:"(unset)",
|
||||
timeout_s?timeout_s:"(unset)",
|
||||
function?function:"(unset)");
|
||||
}
|
||||
else {
|
||||
int timeout = str2i(timeout_s);
|
||||
if(timeout <= 0) timeout = SYSTEMD_JOURNAL_DEFAULT_TIMEOUT;
|
||||
|
||||
netdata_mutex_lock(&mutex);
|
||||
|
||||
if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)
|
||||
function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);
|
||||
else
|
||||
pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in systemd-journal.plugin.");
|
||||
|
||||
fflush(stdout);
|
||||
netdata_mutex_unlock(&mutex);
|
||||
}
|
||||
}
|
||||
else
|
||||
netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)");
|
||||
}
|
||||
|
||||
if(!s || feof(stdin) || ferror(stdin)) {
|
||||
plugin_should_exit = true;
|
||||
netdata_log_error("Received error on stdin.");
|
||||
}
|
||||
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int main(int argc __maybe_unused, char **argv __maybe_unused) {
|
||||
stderror = stderr;
|
||||
clocks_init();
|
||||
|
||||
program_name = "systemd-journal.plugin";
|
||||
|
||||
// disable syslog
|
||||
error_log_syslog = 0;
|
||||
|
||||
// set errors flood protection to 100 logs per hour
|
||||
error_log_errors_per_period = 100;
|
||||
error_log_throttle_period = 3600;
|
||||
|
||||
uids = dictionary_create(0);
|
||||
gids = dictionary_create(0);
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// debug
|
||||
|
||||
if(argc == 2 && strcmp(argv[1], "debug") == 0) {
|
||||
char buf[] = "systemd-journal after:-86400 before:0 last:500";
|
||||
function_systemd_journal("123", buf, "", 0, 30);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
netdata_thread_t reader_thread;
|
||||
netdata_thread_create(&reader_thread, "SDJ_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
time_t started_t = now_monotonic_sec();
|
||||
|
||||
size_t iteration;
|
||||
usec_t step = 1000 * USEC_PER_MS;
|
||||
bool tty = isatty(fileno(stderr)) == 1;
|
||||
|
||||
netdata_mutex_lock(&mutex);
|
||||
fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n",
|
||||
SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
|
||||
|
||||
heartbeat_t hb;
|
||||
heartbeat_init(&hb);
|
||||
for(iteration = 0; 1 ; iteration++) {
|
||||
netdata_mutex_unlock(&mutex);
|
||||
heartbeat_next(&hb, step);
|
||||
netdata_mutex_lock(&mutex);
|
||||
|
||||
if(!tty)
|
||||
fprintf(stdout, "\n");
|
||||
|
||||
fflush(stdout);
|
||||
|
||||
time_t now = now_monotonic_sec();
|
||||
if(now - started_t > 86400)
|
||||
break;
|
||||
}
|
||||
|
||||
dictionary_destroy(uids);
|
||||
dictionary_destroy(gids);
|
||||
|
||||
exit(0);
|
||||
}
|
||||
42
configure.ac
42
configure.ac
|
|
@ -69,6 +69,12 @@ AC_ARG_ENABLE(
|
|||
,
|
||||
[enable_plugin_freeipmi="detect"]
|
||||
)
|
||||
AC_ARG_ENABLE(
|
||||
[plugin-systemd-journal],
|
||||
[AS_HELP_STRING([--enable-plugin-systemd-journal], [enable systemd-journal plugin @<:@default autodetect@:>@])],
|
||||
,
|
||||
[enable_plugin_systemd_journal="detect"]
|
||||
)
|
||||
AC_ARG_ENABLE(
|
||||
[plugin-cups],
|
||||
[AS_HELP_STRING([--enable-plugin-cups], [enable cups plugin @<:@default autodetect@:>@])],
|
||||
|
|
@ -1106,6 +1112,39 @@ AC_MSG_RESULT([${enable_plugin_freeipmi}])
|
|||
AM_CONDITIONAL([ENABLE_PLUGIN_FREEIPMI], [test "${enable_plugin_freeipmi}" = "yes"])
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# systemd-journal.plugin - systemd
|
||||
|
||||
LIBS_BAK="${LIBS}"
|
||||
|
||||
AC_CHECK_LIB([systemd], [sd_journal_open], [have_systemd_libs=yes], [have_systemd_libs=no])
|
||||
AC_CHECK_HEADERS([systemd/sd-journal.h], [have_systemd_journal_header=yes], [have_systemd_journal_header=no])
|
||||
|
||||
if test "${have_systemd_libs}" = "yes" -a "${have_systemd_journal_header}" = "yes"; then
|
||||
have_systemd="yes"
|
||||
else
|
||||
have_systemd="no"
|
||||
fi
|
||||
|
||||
test "${enable_plugin_systemd_journal}" = "yes" -a "${have_systemd}" != "yes" && \
|
||||
AC_MSG_ERROR([systemd is required but not found. Try installing 'libsystemd-dev' or 'libsystemd-devel'])
|
||||
|
||||
AC_MSG_CHECKING([if systemd-journal.plugin should be enabled])
|
||||
if test "${enable_plugin_systemd_journal}" != "no" -a "${have_systemd}" = "yes"; then
|
||||
enable_plugin_systemd_journal="yes"
|
||||
AC_DEFINE([HAVE_SYSTEMD], [1], [systemd usability])
|
||||
OPTIONAL_SYSTEMD_CFLAGS="-I/usr/include"
|
||||
OPTIONAL_SYSTEMD_LIBS="-lsystemd"
|
||||
else
|
||||
enable_plugin_systemd_journal="no"
|
||||
fi
|
||||
AC_MSG_RESULT([${enable_plugin_systemd_journal}])
|
||||
AM_CONDITIONAL([ENABLE_PLUGIN_SYSTEMD_JOURNAL], [test "${enable_plugin_systemd_journal}" = "yes"])
|
||||
|
||||
AC_MSG_NOTICE([OPTIONAL_SYSTEMD_LIBS is set to: ${OPTIONAL_SYSTEMD_LIBS}])
|
||||
|
||||
LIBS="${LIBS_BAK}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# cups.plugin - libcups
|
||||
|
||||
|
|
@ -1874,6 +1913,7 @@ AC_SUBST([OPTIONAL_GTEST_CFLAGS])
|
|||
AC_SUBST([OPTIONAL_GTEST_LIBS])
|
||||
AC_SUBST([OPTIONAL_ML_CFLAGS])
|
||||
AC_SUBST([OPTIONAL_ML_LIBS])
|
||||
AC_SUBST(OPTIONAL_SYSTEMD_LIBS)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Check if cmocka is available - needed for unit testing
|
||||
|
|
@ -1937,6 +1977,7 @@ AC_CONFIG_FILES([
|
|||
collectors/tc.plugin/Makefile
|
||||
collectors/xenstat.plugin/Makefile
|
||||
collectors/perf.plugin/Makefile
|
||||
collectors/systemd-journal.plugin/Makefile
|
||||
daemon/Makefile
|
||||
database/Makefile
|
||||
database/contexts/Makefile
|
||||
|
|
@ -1968,6 +2009,7 @@ AC_CONFIG_FILES([
|
|||
libnetdata/dictionary/Makefile
|
||||
libnetdata/ebpf/Makefile
|
||||
libnetdata/eval/Makefile
|
||||
libnetdata/facets/Makefile
|
||||
libnetdata/july/Makefile
|
||||
libnetdata/locks/Makefile
|
||||
libnetdata/log/Makefile
|
||||
|
|
|
|||
|
|
@ -213,6 +213,9 @@ override_dh_fixperms:
|
|||
# local-listeners
|
||||
chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/local-listeners
|
||||
|
||||
# systemd-journal
|
||||
# chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/systemd-journal.plugin
|
||||
|
||||
override_dh_installlogrotate:
|
||||
cp system/logrotate/netdata debian/netdata.logrotate
|
||||
dh_installlogrotate
|
||||
|
|
|
|||
|
|
@ -1469,7 +1469,7 @@ void print_build_info_json(void) {
|
|||
populate_directories();
|
||||
|
||||
BUFFER *b = buffer_create(0, NULL);
|
||||
buffer_json_initialize(b, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(b, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
build_info_to_json_object(b);
|
||||
|
||||
|
|
|
|||
|
|
@ -1332,6 +1332,7 @@ int mrg_unittest(void);
|
|||
int julytest(void);
|
||||
int pluginsd_parser_unittest(void);
|
||||
void replication_initialize(void);
|
||||
void bearer_tokens_init(void);
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
// initialize the system clocks
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void
|
|||
if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
|
||||
|
||||
wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
|
||||
buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false);
|
||||
buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
struct rrdcontext_to_json t_metrics = {
|
||||
.wb = wb_metrics,
|
||||
|
|
@ -268,7 +268,7 @@ static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void
|
|||
|| t_parent->chart_dimensions) {
|
||||
|
||||
wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
|
||||
buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false);
|
||||
buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
struct rrdcontext_to_json t_instances = {
|
||||
.wb = wb_instances,
|
||||
|
|
@ -366,9 +366,9 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
|
|||
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
|
||||
|
||||
if(after != 0 && before != 0)
|
||||
rrdr_relative_window_to_absolute(&after, &before, NULL);
|
||||
rrdr_relative_window_to_absolute(&after, &before, NULL, false);
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
struct rrdcontext_to_json t_contexts = {
|
||||
.wb = wb,
|
||||
.options = options|RRDCONTEXT_OPTION_SKIP_ID,
|
||||
|
|
@ -403,9 +403,9 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
|
|||
uuid_unparse(*host->node_id, node_uuid);
|
||||
|
||||
if(after != 0 && before != 0)
|
||||
rrdr_relative_window_to_absolute(&after, &before, NULL);
|
||||
rrdr_relative_window_to_absolute(&after, &before, NULL, false);
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
|
||||
buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
|
||||
buffer_json_member_add_string(wb, "node_id", node_uuid);
|
||||
|
|
|
|||
|
|
@ -1298,7 +1298,7 @@ int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_ha
|
|||
|
||||
buffer_flush(w->response.data);
|
||||
|
||||
buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false);
|
||||
buffer_json_finalize(w->response.data);
|
||||
|
|
@ -1934,14 +1934,14 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
|
|||
}
|
||||
|
||||
if(req->after || req->before) {
|
||||
ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now);
|
||||
ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false);
|
||||
ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
|
||||
}
|
||||
else
|
||||
ctl.now = now_realtime_sec();
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0,
|
||||
true, (req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG));
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true,
|
||||
((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
buffer_json_member_add_uint64(wb, "api", 2);
|
||||
|
||||
|
|
|
|||
|
|
@ -1052,7 +1052,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
|
|||
if(query_target_has_percentage_of_group(qt))
|
||||
qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
|
||||
|
||||
qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
|
||||
qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now,
|
||||
unittest_running);
|
||||
|
||||
// prepare our local variables - we need these across all these functions
|
||||
QUERY_TARGET_LOCALS qtl = {
|
||||
|
|
|
|||
|
|
@ -270,7 +270,7 @@ static inline size_t sanitize_function_text(char *dst, const char *src, size_t d
|
|||
// we keep a dictionary per RRDSET with these functions
|
||||
// the dictionary is created on demand (only when a function is added to an RRDSET)
|
||||
|
||||
typedef enum {
|
||||
typedef enum __attribute__((packed)) {
|
||||
RRD_FUNCTION_LOCAL = (1 << 0),
|
||||
RRD_FUNCTION_GLOBAL = (1 << 1),
|
||||
|
||||
|
|
@ -279,7 +279,7 @@ typedef enum {
|
|||
|
||||
struct rrd_collector_function {
|
||||
bool sync; // when true, the function is called synchronously
|
||||
uint8_t options; // RRD_FUNCTION_OPTIONS
|
||||
RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
|
||||
STRING *help;
|
||||
int timeout; // the default timeout of the function
|
||||
|
||||
|
|
@ -814,7 +814,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
|
||||
buffer_flush(wb);
|
||||
wb->content_type = CT_APPLICATION_JSON;
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
|
||||
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
|
||||
|
|
@ -858,8 +858,8 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
|
||||
// retention
|
||||
buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node
|
||||
buffer_json_add_array_item_uint64(wb, s.db.first_time_s * 1000); // dbFrom
|
||||
buffer_json_add_array_item_uint64(wb, s.db.last_time_s * 1000); // dbTo
|
||||
buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom
|
||||
buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo
|
||||
|
||||
if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s)
|
||||
buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration
|
||||
|
|
@ -877,7 +877,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
|
||||
// collection
|
||||
if(s.ingest.since) {
|
||||
buffer_json_add_array_item_uint64(wb, s.ingest.since * 1000); // InSince
|
||||
buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince
|
||||
buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge
|
||||
}
|
||||
else {
|
||||
|
|
@ -897,7 +897,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
|
||||
// streaming
|
||||
if(s.stream.since) {
|
||||
buffer_json_add_array_item_uint64(wb, s.stream.since * 1000); // OutSince
|
||||
buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince
|
||||
buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge
|
||||
}
|
||||
else {
|
||||
|
|
@ -990,19 +990,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration",
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_NONE, NULL);
|
||||
|
|
@ -1049,13 +1049,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
// --- collection ---
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_NONE, NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age",
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
|
@ -1124,13 +1124,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
// --- streaming ---
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_NONE, NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age",
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
|
@ -1242,14 +1242,14 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
|
|||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince",
|
||||
"Last Outbound Connection Attempt Status Change Time",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME,
|
||||
RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_NONE, NULL);
|
||||
|
||||
buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge",
|
||||
"Last Outbound Connection Attempt Status Change Age",
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION,
|
||||
RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
|
||||
0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
|
||||
RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE, NULL);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ SUBDIRS = \
|
|||
dictionary \
|
||||
ebpf \
|
||||
eval \
|
||||
facets \
|
||||
json \
|
||||
july \
|
||||
health \
|
||||
|
|
|
|||
|
|
@ -307,11 +307,11 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {
|
|||
// ----------------------------------------------------------------------------
|
||||
|
||||
void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
|
||||
bool add_anonymous_object, bool minify) {
|
||||
bool add_anonymous_object, BUFFER_JSON_OPTIONS options) {
|
||||
strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
|
||||
strncpyz(wb->json.value_quote, value_quote, BUFFER_QUOTE_MAX_SIZE);
|
||||
|
||||
wb->json.minify = minify;
|
||||
wb->json.options = options;
|
||||
wb->json.depth = (int8_t)(depth - 1);
|
||||
_buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
|
||||
|
||||
|
|
@ -339,7 +339,7 @@ void buffer_json_finalize(BUFFER *wb) {
|
|||
}
|
||||
}
|
||||
|
||||
if(!wb->json.minify)
|
||||
if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY))
|
||||
buffer_fast_strcat(wb, "\n", 1);
|
||||
}
|
||||
|
||||
|
|
@ -490,13 +490,13 @@ int buffer_unittest(void) {
|
|||
|
||||
buffer_flush(wb);
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_finalize(wb);
|
||||
errors += buffer_expect(wb, "{\n}\n");
|
||||
|
||||
buffer_flush(wb);
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_string(wb, "hello", "world");
|
||||
buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
|
||||
buffer_json_member_add_object(wb, "object1");
|
||||
|
|
|
|||
|
|
@ -68,6 +68,12 @@ typedef enum __attribute__ ((__packed__)) {
|
|||
CT_APPLICATION_ZIP,
|
||||
} HTTP_CONTENT_TYPE;
|
||||
|
||||
typedef enum __attribute__ ((__packed__)) {
|
||||
BUFFER_JSON_OPTIONS_DEFAULT = 0,
|
||||
BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),
|
||||
BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS = (1 << 1),
|
||||
} BUFFER_JSON_OPTIONS;
|
||||
|
||||
typedef struct web_buffer {
|
||||
size_t size; // allocation size of buffer, in bytes
|
||||
size_t len; // current data length in buffer, in bytes
|
||||
|
|
@ -82,7 +88,7 @@ typedef struct web_buffer {
|
|||
char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
|
||||
char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
|
||||
int8_t depth;
|
||||
bool minify;
|
||||
BUFFER_JSON_OPTIONS options;
|
||||
BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
|
||||
} json;
|
||||
} BUFFER;
|
||||
|
|
@ -148,7 +154,7 @@ static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
|
|||
}
|
||||
|
||||
void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
|
||||
bool add_anonymous_object, bool minify);
|
||||
bool add_anonymous_object, BUFFER_JSON_OPTIONS options);
|
||||
|
||||
void buffer_json_finalize(BUFFER *wb);
|
||||
|
||||
|
|
@ -668,7 +674,8 @@ static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {
|
|||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
|
||||
if(wb->json.minify)
|
||||
if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||
|
||||
(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS)))
|
||||
return;
|
||||
|
||||
buffer_fast_strcat(wb, "\n", 1);
|
||||
|
|
@ -715,7 +722,7 @@ static inline void buffer_json_object_close(BUFFER *wb) {
|
|||
assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
|
||||
assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");
|
||||
#endif
|
||||
if(!wb->json.minify) {
|
||||
if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY)) {
|
||||
buffer_fast_strcat(wb, "\n", 1);
|
||||
buffer_print_spaces(wb, wb->json.depth);
|
||||
}
|
||||
|
|
@ -801,48 +808,42 @@ static inline void buffer_json_add_array_item_array(BUFFER *wb) {
|
|||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_json_add_string_value(wb, value);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_netdata_double(wb, value);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_int64(wb, value);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_uint64(wb, value);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_int64(wb, value);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_int64(wb, value);
|
||||
buffer_fast_strcat(wb, "000", 3);
|
||||
|
|
@ -850,8 +851,7 @@ static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value)
|
|||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_print_int64(wb, value);
|
||||
buffer_fast_strcat(wb, "000", 3);
|
||||
|
|
@ -859,8 +859,7 @@ static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value
|
|||
}
|
||||
|
||||
static inline void buffer_json_add_array_item_object(BUFFER *wb) {
|
||||
if(wb->json.stack[wb->json.depth].count)
|
||||
buffer_fast_strcat(wb, ",", 1);
|
||||
buffer_print_json_comma_newline_spacing(wb);
|
||||
|
||||
buffer_fast_strcat(wb, "{", 1);
|
||||
wb->json.stack[wb->json.depth].count++;
|
||||
|
|
@ -919,6 +918,11 @@ static inline void buffer_json_array_close(BUFFER *wb) {
|
|||
assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
|
||||
assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");
|
||||
#endif
|
||||
if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS) {
|
||||
buffer_fast_strcat(wb, "\n", 1);
|
||||
buffer_print_spaces(wb, wb->json.depth);
|
||||
}
|
||||
|
||||
buffer_fast_strcat(wb, "]", 1);
|
||||
_buffer_json_depth_pop(wb);
|
||||
}
|
||||
|
|
@ -928,6 +932,8 @@ typedef enum __attribute__((packed)) {
|
|||
RRDF_FIELD_OPTS_UNIQUE_KEY = (1 << 0), // the field is the unique key of the row
|
||||
RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default
|
||||
RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky
|
||||
RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width
|
||||
RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should get full width
|
||||
} RRDF_FIELD_OPTIONS;
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
|
|
@ -969,7 +975,8 @@ static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {
|
|||
typedef enum __attribute__((packed)) {
|
||||
RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation
|
||||
RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%
|
||||
RRDF_FIELD_VISUAL_PILL, // array of values (transformation is respected)
|
||||
RRDF_FIELD_VISUAL_PILL, //
|
||||
RRDF_FIELD_VISUAL_MARKDOC, //
|
||||
} RRDF_FIELD_VISUAL;
|
||||
|
||||
static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) {
|
||||
|
|
@ -983,14 +990,18 @@ static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual)
|
|||
|
||||
case RRDF_FIELD_VISUAL_PILL:
|
||||
return "pill";
|
||||
|
||||
case RRDF_FIELD_VISUAL_MARKDOC:
|
||||
return "markdoc";
|
||||
}
|
||||
}
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
RRDF_FIELD_TRANSFORM_NONE, // show the value as-is
|
||||
RRDF_FIELD_TRANSFORM_NUMBER, // show the value repsecting the decimal_points
|
||||
RRDF_FIELD_TRANSFORM_DURATION, // transform as duration in second to a human readable duration
|
||||
RRDF_FIELD_TRANSFORM_DATETIME, // UNIX epoch timestamp in ms
|
||||
RRDF_FIELD_TRANSFORM_NUMBER, // show the value respecting the decimal_points
|
||||
RRDF_FIELD_TRANSFORM_DURATION_S, // transform as duration in second to a human-readable duration
|
||||
RRDF_FIELD_TRANSFORM_DATETIME_MS, // UNIX epoch timestamp in ms
|
||||
RRDF_FIELD_TRANSFORM_DATETIME_USEC, // UNIX epoch timestamp in usec
|
||||
} RRDF_FIELD_TRANSFORM;
|
||||
|
||||
static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) {
|
||||
|
|
@ -1002,11 +1013,14 @@ static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM tr
|
|||
case RRDF_FIELD_TRANSFORM_NUMBER:
|
||||
return "number";
|
||||
|
||||
case RRDF_FIELD_TRANSFORM_DURATION:
|
||||
case RRDF_FIELD_TRANSFORM_DURATION_S:
|
||||
return "duration";
|
||||
|
||||
case RRDF_FIELD_TRANSFORM_DATETIME:
|
||||
case RRDF_FIELD_TRANSFORM_DATETIME_MS:
|
||||
return "datetime";
|
||||
|
||||
case RRDF_FIELD_TRANSFORM_DATETIME_USEC:
|
||||
return "datetime_usec";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1064,18 +1078,26 @@ static inline const char *rrdf_field_summary_to_string(RRDF_FIELD_SUMMARY summar
|
|||
}
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
RRDF_FIELD_FILTER_NONE,
|
||||
RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_FILTER_MULTISELECT,
|
||||
RRDF_FIELD_FILTER_FACET,
|
||||
} RRDF_FIELD_FILTER;
|
||||
|
||||
static inline const char *rrdf_field_filter_to_string(RRDF_FIELD_FILTER filter) {
|
||||
switch(filter) {
|
||||
default:
|
||||
case RRDF_FIELD_FILTER_RANGE:
|
||||
return "range";
|
||||
|
||||
case RRDF_FIELD_FILTER_MULTISELECT:
|
||||
return "multiselect";
|
||||
|
||||
case RRDF_FIELD_FILTER_FACET:
|
||||
return "facet";
|
||||
|
||||
default:
|
||||
case RRDF_FIELD_FILTER_NONE:
|
||||
return "none";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1114,6 +1136,9 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const
|
|||
buffer_json_member_add_boolean(wb, "sticky", options & RRDF_FIELD_OPTS_STICKY);
|
||||
buffer_json_member_add_string(wb, "summary", rrdf_field_summary_to_string(summary));
|
||||
buffer_json_member_add_string(wb, "filter", rrdf_field_filter_to_string(filter));
|
||||
|
||||
buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH);
|
||||
buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP);
|
||||
}
|
||||
buffer_json_object_close(wb);
|
||||
}
|
||||
|
|
|
|||
8
libnetdata/facets/Makefile.am
Normal file
8
libnetdata/facets/Makefile.am
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
AUTOMAKE_OPTIONS = subdir-objects
|
||||
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
|
||||
|
||||
dist_noinst_DATA = \
|
||||
README.md \
|
||||
$(NULL)
|
||||
0
libnetdata/facets/README.md
Normal file
0
libnetdata/facets/README.md
Normal file
827
libnetdata/facets/facets.c
Normal file
827
libnetdata/facets/facets.c
Normal file
|
|
@ -0,0 +1,827 @@
|
|||
#include "facets.h"
|
||||
|
||||
#define FACET_VALUE_UNSET "-"
|
||||
#define HISTOGRAM_COLUMNS 60
|
||||
|
||||
static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
time_t calculate_bar_width(time_t before, time_t after) {
|
||||
// Array of valid durations in seconds
|
||||
static time_t valid_durations[] = {
|
||||
1,
|
||||
15,
|
||||
30,
|
||||
1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
|
||||
1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
|
||||
1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
|
||||
1 * (30*86400) // months
|
||||
};
|
||||
static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
|
||||
|
||||
time_t duration = before - after;
|
||||
time_t bar_width = 1;
|
||||
|
||||
for (int i = array_size - 1; i >= 0; --i) {
|
||||
if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) {
|
||||
bar_width = valid_durations[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return bar_width;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
static inline void uint32_to_char(uint32_t num, char *out) {
|
||||
static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
|
||||
|
||||
int i;
|
||||
for(i = 5; i >= 0; --i) {
|
||||
out[i] = id_encoding_characters[num & 63];
|
||||
num >>= 6;
|
||||
}
|
||||
out[6] = '\0';
|
||||
}
|
||||
|
||||
inline void facets_string_hash(const char *src, char *out) {
|
||||
uint32_t hash1 = fnv1a_hash32(src);
|
||||
uint32_t hash2 = djb2_hash32(src);
|
||||
uint32_t hash3 = larson_hash32(src);
|
||||
|
||||
uint32_to_char(hash1, out);
|
||||
uint32_to_char(hash2, &out[6]);
|
||||
uint32_to_char(hash3, &out[12]);
|
||||
|
||||
out[18] = '\0';
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
typedef struct facet_value {
|
||||
const char *name;
|
||||
|
||||
bool selected;
|
||||
|
||||
uint32_t rows_matching_facet_value;
|
||||
uint32_t final_facet_value_counter;
|
||||
} FACET_VALUE;
|
||||
|
||||
struct facet_key {
|
||||
const char *name;
|
||||
|
||||
DICTIONARY *values;
|
||||
|
||||
FACET_KEY_OPTIONS options;
|
||||
|
||||
bool default_selected_for_values; // the default "selected" for all values in the dictionary
|
||||
|
||||
// members about the current row
|
||||
uint32_t key_found_in_row;
|
||||
uint32_t key_values_selected_in_row;
|
||||
BUFFER *current_value;
|
||||
|
||||
uint32_t order;
|
||||
|
||||
struct {
|
||||
facet_dynamic_row_t cb;
|
||||
void *data;
|
||||
} dynamic;
|
||||
|
||||
struct {
|
||||
facets_key_transformer_t cb;
|
||||
void *data;
|
||||
|
||||
} transform;
|
||||
};
|
||||
|
||||
struct facets {
|
||||
SIMPLE_PATTERN *visible_keys;
|
||||
SIMPLE_PATTERN *excluded_keys;
|
||||
SIMPLE_PATTERN *included_keys;
|
||||
|
||||
FACETS_OPTIONS options;
|
||||
usec_t anchor;
|
||||
|
||||
SIMPLE_PATTERN *query; // the full text search pattern
|
||||
size_t keys_filtered_by_query; // the number of fields we do full text search (constant)
|
||||
size_t keys_matched_by_query; // the number of fields matched the full text search (per row)
|
||||
|
||||
DICTIONARY *accepted_params;
|
||||
|
||||
DICTIONARY *keys;
|
||||
FACET_ROW *base; // double linked list of the selected facets rows
|
||||
|
||||
uint32_t items_to_return;
|
||||
uint32_t max_items_to_return;
|
||||
uint32_t order;
|
||||
|
||||
struct {
|
||||
FACET_ROW *last_added;
|
||||
|
||||
size_t evaluated;
|
||||
size_t matched;
|
||||
|
||||
size_t first;
|
||||
size_t forwards;
|
||||
size_t backwards;
|
||||
size_t skips_before;
|
||||
size_t skips_after;
|
||||
size_t prepends;
|
||||
size_t appends;
|
||||
size_t shifts;
|
||||
} operations;
|
||||
};
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
|
||||
if(!k->key_found_in_row)
|
||||
v->rows_matching_facet_value++;
|
||||
|
||||
k->key_found_in_row++;
|
||||
|
||||
if(v->selected)
|
||||
k->key_values_selected_in_row++;
|
||||
}
|
||||
|
||||
static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
|
||||
bool included = true, excluded = false;
|
||||
|
||||
if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) {
|
||||
if(k->options & FACET_KEY_OPTION_FACET) {
|
||||
included = true;
|
||||
excluded = false;
|
||||
}
|
||||
else if(k->options & FACET_KEY_OPTION_NO_FACET) {
|
||||
included = false;
|
||||
excluded = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (facets->included_keys) {
|
||||
if (!simple_pattern_matches(facets->included_keys, k->name))
|
||||
included = false;
|
||||
}
|
||||
|
||||
if (facets->excluded_keys) {
|
||||
if (simple_pattern_matches(facets->excluded_keys, k->name))
|
||||
excluded = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(included && !excluded) {
|
||||
k->options |= FACET_KEY_OPTION_FACET;
|
||||
k->options &= ~FACET_KEY_OPTION_NO_FACET;
|
||||
return true;
|
||||
}
|
||||
|
||||
k->options |= FACET_KEY_OPTION_NO_FACET;
|
||||
k->options &= ~FACET_KEY_OPTION_FACET;
|
||||
return false;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// FACET_VALUE dictionary hooks
|
||||
|
||||
static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
|
||||
FACET_VALUE *v = value;
|
||||
FACET_KEY *k = data;
|
||||
|
||||
if(v->name) {
|
||||
// an actual value, not a filter
|
||||
v->name = strdupz(v->name);
|
||||
facet_value_is_used(k, v);
|
||||
}
|
||||
|
||||
if(!v->selected)
|
||||
v->selected = k->default_selected_for_values;
|
||||
}
|
||||
|
||||
static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
|
||||
FACET_VALUE *v = old_value;
|
||||
FACET_VALUE *nv = new_value;
|
||||
FACET_KEY *k = data;
|
||||
|
||||
if(!v->name && nv->name)
|
||||
// an actual value, not a filter
|
||||
v->name = strdupz(nv->name);
|
||||
|
||||
if(v->name)
|
||||
facet_value_is_used(k, v);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
|
||||
FACET_VALUE *v = value;
|
||||
freez((char *)v->name);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// FACET_KEY dictionary hooks
|
||||
|
||||
static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) {
|
||||
if(k->values)
|
||||
return;
|
||||
|
||||
if(facets_key_is_facet(facets, k)) {
|
||||
k->values = dictionary_create_advanced(
|
||||
DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
|
||||
NULL, sizeof(FACET_VALUE));
|
||||
dictionary_register_insert_callback(k->values, facet_value_insert_callback, k);
|
||||
dictionary_register_conflict_callback(k->values, facet_value_conflict_callback, k);
|
||||
dictionary_register_delete_callback(k->values, facet_value_delete_callback, k);
|
||||
}
|
||||
}
|
||||
|
||||
static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
|
||||
FACET_KEY *k = value;
|
||||
FACETS *facets = data;
|
||||
|
||||
if(!(k->options & FACET_KEY_OPTION_REORDER))
|
||||
k->order = facets->order++;
|
||||
|
||||
if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS))
|
||||
facets->keys_filtered_by_query++;
|
||||
|
||||
if(k->name) {
|
||||
// an actual value, not a filter
|
||||
k->name = strdupz(k->name);
|
||||
facet_key_late_init(facets, k);
|
||||
}
|
||||
|
||||
k->current_value = buffer_create(0, NULL);
|
||||
}
|
||||
|
||||
static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
|
||||
FACET_KEY *k = old_value;
|
||||
FACET_KEY *nk = new_value;
|
||||
FACETS *facets = data;
|
||||
|
||||
if(!k->name && nk->name) {
|
||||
// an actual value, not a filter
|
||||
k->name = strdupz(nk->name);
|
||||
facet_key_late_init(facets, k);
|
||||
}
|
||||
|
||||
if(k->options & FACET_KEY_OPTION_REORDER) {
|
||||
k->order = facets->order++;
|
||||
k->options &= ~FACET_KEY_OPTION_REORDER;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
|
||||
FACET_KEY *k = value;
|
||||
dictionary_destroy(k->values);
|
||||
buffer_free(k->current_value);
|
||||
freez((char *)k->name);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
|
||||
FACETS *facets = callocz(1, sizeof(FACETS));
|
||||
facets->options = options;
|
||||
facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY));
|
||||
dictionary_register_insert_callback(facets->keys, facet_key_insert_callback, facets);
|
||||
dictionary_register_conflict_callback(facets->keys, facet_key_conflict_callback, facets);
|
||||
dictionary_register_delete_callback(facets->keys, facet_key_delete_callback, facets);
|
||||
|
||||
if(facet_keys && *facet_keys)
|
||||
facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
|
||||
|
||||
if(non_facet_keys && *non_facet_keys)
|
||||
facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
|
||||
|
||||
if(visible_keys && *visible_keys)
|
||||
facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
|
||||
|
||||
facets->max_items_to_return = items_to_return;
|
||||
facets->anchor = anchor;
|
||||
facets->order = 1;
|
||||
|
||||
return facets;
|
||||
}
|
||||
|
||||
void facets_destroy(FACETS *facets) {
|
||||
dictionary_destroy(facets->accepted_params);
|
||||
dictionary_destroy(facets->keys);
|
||||
simple_pattern_free(facets->visible_keys);
|
||||
simple_pattern_free(facets->included_keys);
|
||||
simple_pattern_free(facets->excluded_keys);
|
||||
|
||||
while(facets->base) {
|
||||
FACET_ROW *r = facets->base;
|
||||
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next);
|
||||
|
||||
facets_row_free(facets, r);
|
||||
}
|
||||
|
||||
freez(facets);
|
||||
}
|
||||
|
||||
void facets_accepted_param(FACETS *facets, const char *param) {
|
||||
if(!facets->accepted_params)
|
||||
facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
|
||||
|
||||
dictionary_set(facets->accepted_params, param, NULL, 0);
|
||||
}
|
||||
|
||||
inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) {
|
||||
FACET_KEY tk = {
|
||||
.name = key,
|
||||
.options = options,
|
||||
.default_selected_for_values = true,
|
||||
};
|
||||
char hash[FACET_STRING_HASH_SIZE];
|
||||
facets_string_hash(tk.name, hash);
|
||||
return dictionary_set(facets->keys, hash, &tk, sizeof(tk));
|
||||
}
|
||||
|
||||
inline FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) {
|
||||
FACET_KEY *k = facets_register_key(facets, key, options);
|
||||
k->transform.cb = cb;
|
||||
k->transform.data = data;
|
||||
return k;
|
||||
}
|
||||
|
||||
inline FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) {
|
||||
FACET_KEY *k = facets_register_key(facets, key, options);
|
||||
k->dynamic.cb = cb;
|
||||
k->dynamic.data = data;
|
||||
return k;
|
||||
}
|
||||
|
||||
void facets_set_query(FACETS *facets, const char *query) {
|
||||
if(!query)
|
||||
return;
|
||||
|
||||
facets->query = simple_pattern_create(query, " \t", SIMPLE_PATTERN_SUBSTRING, false);
|
||||
}
|
||||
|
||||
void facets_set_items(FACETS *facets, uint32_t items) {
|
||||
facets->max_items_to_return = items;
|
||||
}
|
||||
|
||||
void facets_set_anchor(FACETS *facets, usec_t anchor) {
|
||||
facets->anchor = anchor;
|
||||
}
|
||||
|
||||
void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options) {
|
||||
FACET_KEY tk = {
|
||||
.options = options,
|
||||
};
|
||||
FACET_KEY *k = dictionary_set(facets->keys, key_id, &tk, sizeof(tk));
|
||||
|
||||
k->default_selected_for_values = false;
|
||||
k->options |= FACET_KEY_OPTION_FACET;
|
||||
k->options &= ~FACET_KEY_OPTION_NO_FACET;
|
||||
facet_key_late_init(facets, k);
|
||||
|
||||
FACET_VALUE tv = {
|
||||
.selected = true,
|
||||
};
|
||||
dictionary_set(k->values, value_ids, &tv, sizeof(tv));
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) {
|
||||
if(k->transform.cb)
|
||||
k->transform.cb(facets, k->current_value, k->transform.data);
|
||||
|
||||
if(buffer_strlen(k->current_value) == 0)
|
||||
buffer_strcat(k->current_value, FACET_VALUE_UNSET);
|
||||
|
||||
// bool found = false;
|
||||
// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL)
|
||||
// found = true;
|
||||
|
||||
if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {
|
||||
if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value)))
|
||||
facets->keys_matched_by_query++;
|
||||
}
|
||||
|
||||
if(k->values) {
|
||||
FACET_VALUE tk = {
|
||||
.name = buffer_tostring(k->current_value),
|
||||
};
|
||||
char hash[FACET_STRING_HASH_SIZE];
|
||||
facets_string_hash(tk.name, hash);
|
||||
dictionary_set(k->values, hash, &tk, sizeof(tk));
|
||||
}
|
||||
else {
|
||||
k->key_found_in_row++;
|
||||
k->key_values_selected_in_row++;
|
||||
}
|
||||
}
|
||||
|
||||
void facets_add_key_value(FACETS *facets, const char *key, const char *value) {
|
||||
FACET_KEY *k = facets_register_key(facets, key, 0);
|
||||
buffer_flush(k->current_value);
|
||||
buffer_strcat(k->current_value, value);
|
||||
|
||||
facets_check_value(facets, k);
|
||||
}
|
||||
|
||||
void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) {
|
||||
FACET_KEY *k = facets_register_key(facets, key, 0);
|
||||
buffer_flush(k->current_value);
|
||||
buffer_strncat(k->current_value, value, value_len);
|
||||
|
||||
facets_check_value(facets, k);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// FACET_ROW dictionary hooks
|
||||
|
||||
static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
|
||||
FACET_ROW_KEY_VALUE *rkv = value;
|
||||
FACET_ROW *row = data; (void)row;
|
||||
|
||||
rkv->wb = buffer_create(0, NULL);
|
||||
buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET);
|
||||
}
|
||||
|
||||
static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
|
||||
FACET_ROW_KEY_VALUE *rkv = old_value;
|
||||
FACET_ROW_KEY_VALUE *n_rkv = new_value;
|
||||
FACET_ROW *row = data; (void)row;
|
||||
|
||||
buffer_flush(rkv->wb);
|
||||
buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
|
||||
FACET_ROW_KEY_VALUE *rkv = value;
|
||||
FACET_ROW *row = data; (void)row;
|
||||
|
||||
buffer_free(rkv->wb);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// FACET_ROW management
|
||||
|
||||
static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
|
||||
dictionary_destroy(row->dict);
|
||||
freez(row);
|
||||
}
|
||||
|
||||
static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) {
|
||||
FACET_ROW *row;
|
||||
|
||||
if(into)
|
||||
row = into;
|
||||
else {
|
||||
row = callocz(1, sizeof(FACET_ROW));
|
||||
row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE));
|
||||
dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row);
|
||||
dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row);
|
||||
dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row);
|
||||
}
|
||||
|
||||
row->usec = usec;
|
||||
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
FACET_ROW_KEY_VALUE t = {
|
||||
.tmp = buffer_strlen(k->current_value) ? buffer_tostring(k->current_value) : FACET_VALUE_UNSET,
|
||||
.wb = NULL,
|
||||
};
|
||||
dictionary_set(row->dict, k->name, &t, sizeof(t));
|
||||
}
|
||||
dfe_done(k);
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
static void facets_row_keep(FACETS *facets, usec_t usec) {
|
||||
facets->operations.matched++;
|
||||
|
||||
if(usec < facets->anchor) {
|
||||
facets->operations.skips_before++;
|
||||
return;
|
||||
}
|
||||
|
||||
if(unlikely(!facets->base)) {
|
||||
facets->operations.last_added = facets_row_create(facets, usec, NULL);
|
||||
DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
|
||||
facets->items_to_return++;
|
||||
facets->operations.first++;
|
||||
return;
|
||||
}
|
||||
|
||||
if(likely(usec > facets->base->prev->usec))
|
||||
facets->operations.last_added = facets->base->prev;
|
||||
|
||||
FACET_ROW *last = facets->operations.last_added;
|
||||
while(last->prev != facets->base->prev && usec > last->prev->usec) {
|
||||
last = last->prev;
|
||||
facets->operations.backwards++;
|
||||
}
|
||||
|
||||
while(last->next && usec < last->next->usec) {
|
||||
last = last->next;
|
||||
facets->operations.forwards++;
|
||||
}
|
||||
|
||||
if(facets->items_to_return >= facets->max_items_to_return) {
|
||||
if(last == facets->base->prev && usec < last->usec) {
|
||||
facets->operations.skips_after++;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
facets->items_to_return++;
|
||||
|
||||
if(usec > last->usec) {
|
||||
if(facets->items_to_return > facets->max_items_to_return) {
|
||||
facets->items_to_return--;
|
||||
facets->operations.shifts++;
|
||||
facets->operations.last_added = facets->base->prev;
|
||||
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
|
||||
facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added);
|
||||
}
|
||||
DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
|
||||
facets->operations.prepends++;
|
||||
}
|
||||
else {
|
||||
facets->operations.last_added = facets_row_create(facets, usec, NULL);
|
||||
DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
|
||||
facets->operations.appends++;
|
||||
}
|
||||
|
||||
while(facets->items_to_return > facets->max_items_to_return) {
|
||||
// we have to remove something
|
||||
|
||||
FACET_ROW *tmp = facets->base->prev;
|
||||
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next);
|
||||
facets->items_to_return--;
|
||||
|
||||
if(unlikely(facets->operations.last_added == tmp))
|
||||
facets->operations.last_added = facets->base->prev;
|
||||
|
||||
facets_row_free(facets, tmp);
|
||||
facets->operations.shifts++;
|
||||
}
|
||||
}
|
||||
|
||||
void facets_rows_begin(FACETS *facets) {
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
k->key_found_in_row = 0;
|
||||
k->key_values_selected_in_row = 0;
|
||||
buffer_flush(k->current_value);
|
||||
}
|
||||
dfe_done(k);
|
||||
|
||||
facets->keys_matched_by_query = 0;
|
||||
}
|
||||
|
||||
void facets_row_finished(FACETS *facets, usec_t usec) {
|
||||
if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query)
|
||||
goto cleanup;
|
||||
|
||||
facets->operations.evaluated++;
|
||||
|
||||
uint32_t total_keys = 0;
|
||||
uint32_t selected_by = 0;
|
||||
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
if(!k->key_found_in_row) {
|
||||
internal_fatal(buffer_strlen(k->current_value), "key is not found in row but it has a current value");
|
||||
// put the FACET_VALUE_UNSET value into it
|
||||
facets_check_value(facets, k);
|
||||
}
|
||||
|
||||
internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point");
|
||||
internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point");
|
||||
internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once");
|
||||
|
||||
k->key_found_in_row = 1;
|
||||
|
||||
total_keys += k->key_found_in_row;
|
||||
selected_by += (k->key_values_selected_in_row) ? 1 : 0;
|
||||
}
|
||||
dfe_done(k);
|
||||
|
||||
if(selected_by >= total_keys - 1) {
|
||||
uint32_t found = 0;
|
||||
|
||||
dfe_start_read(facets->keys, k){
|
||||
uint32_t counted_by = selected_by;
|
||||
|
||||
if (counted_by != total_keys && !k->key_values_selected_in_row)
|
||||
counted_by++;
|
||||
|
||||
if(counted_by == total_keys) {
|
||||
if(k->values) {
|
||||
char hash[FACET_STRING_HASH_SIZE];
|
||||
facets_string_hash(buffer_tostring(k->current_value), hash);
|
||||
FACET_VALUE *v = dictionary_get(k->values, hash);
|
||||
v->final_facet_value_counter++;
|
||||
}
|
||||
|
||||
found++;
|
||||
}
|
||||
}
|
||||
dfe_done(k);
|
||||
|
||||
internal_fatal(!found, "We should find at least one facet to count this row");
|
||||
(void)found;
|
||||
}
|
||||
|
||||
if(selected_by == total_keys)
|
||||
facets_row_keep(facets, usec);
|
||||
|
||||
cleanup:
|
||||
facets_rows_begin(facets);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// output
|
||||
|
||||
void facets_report(FACETS *facets, BUFFER *wb) {
|
||||
buffer_json_member_add_boolean(wb, "show_ids", false);
|
||||
buffer_json_member_add_boolean(wb, "has_history", true);
|
||||
|
||||
buffer_json_member_add_object(wb, "pagination");
|
||||
buffer_json_member_add_boolean(wb, "enabled", true);
|
||||
buffer_json_member_add_string(wb, "key", "anchor");
|
||||
buffer_json_member_add_string(wb, "column", "timestamp");
|
||||
buffer_json_object_close(wb);
|
||||
|
||||
buffer_json_member_add_array(wb, "accepted_params");
|
||||
{
|
||||
if(facets->accepted_params) {
|
||||
void *t;
|
||||
dfe_start_read(facets->accepted_params, t) {
|
||||
buffer_json_add_array_item_string(wb, t_dfe.name);
|
||||
}
|
||||
dfe_done(t);
|
||||
}
|
||||
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
if(!k->values)
|
||||
continue;
|
||||
|
||||
buffer_json_add_array_item_string(wb, k_dfe.name);
|
||||
}
|
||||
dfe_done(k);
|
||||
}
|
||||
buffer_json_array_close(wb); // accepted_params
|
||||
|
||||
buffer_json_member_add_array(wb, "facets");
|
||||
{
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
if(!k->values)
|
||||
continue;
|
||||
|
||||
buffer_json_add_array_item_object(wb); // key
|
||||
{
|
||||
buffer_json_member_add_string(wb, "id", k_dfe.name);
|
||||
buffer_json_member_add_string(wb, "name", k->name);
|
||||
|
||||
if(!k->order)
|
||||
k->order = facets->order++;
|
||||
|
||||
buffer_json_member_add_uint64(wb, "order", k->order);
|
||||
buffer_json_member_add_array(wb, "options");
|
||||
{
|
||||
FACET_VALUE *v;
|
||||
dfe_start_read(k->values, v) {
|
||||
buffer_json_add_array_item_object(wb);
|
||||
{
|
||||
buffer_json_member_add_string(wb, "id", v_dfe.name);
|
||||
buffer_json_member_add_string(wb, "name", v->name);
|
||||
buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);
|
||||
}
|
||||
buffer_json_object_close(wb);
|
||||
}
|
||||
dfe_done(v);
|
||||
}
|
||||
buffer_json_array_close(wb); // options
|
||||
}
|
||||
buffer_json_object_close(wb); // key
|
||||
}
|
||||
dfe_done(k);
|
||||
}
|
||||
buffer_json_array_close(wb); // facets
|
||||
|
||||
buffer_json_member_add_object(wb, "columns");
|
||||
{
|
||||
size_t field_id = 0;
|
||||
buffer_rrdf_table_add_field(
|
||||
wb, field_id++,
|
||||
"timestamp", "Timestamp",
|
||||
RRDF_FIELD_TYPE_TIMESTAMP,
|
||||
RRDF_FIELD_VISUAL_VALUE,
|
||||
RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
|
||||
RRDF_FIELD_SORT_DESCENDING,
|
||||
NULL,
|
||||
RRDF_FIELD_SUMMARY_COUNT,
|
||||
RRDF_FIELD_FILTER_RANGE,
|
||||
RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,
|
||||
NULL);
|
||||
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k) {
|
||||
RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE;
|
||||
bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY);
|
||||
|
||||
if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values))
|
||||
visible = true;
|
||||
|
||||
if(!visible)
|
||||
visible = simple_pattern_matches(facets->visible_keys, k->name);
|
||||
|
||||
if(visible)
|
||||
options |= RRDF_FIELD_OPTS_VISIBLE;
|
||||
|
||||
if(k->options & FACET_KEY_OPTION_MAIN_TEXT)
|
||||
options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
|
||||
|
||||
buffer_rrdf_table_add_field(
|
||||
wb, field_id++,
|
||||
k_dfe.name, k->name ? k->name : k_dfe.name,
|
||||
RRDF_FIELD_TYPE_STRING,
|
||||
RRDF_FIELD_VISUAL_VALUE,
|
||||
RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
|
||||
RRDF_FIELD_SORT_ASCENDING,
|
||||
NULL,
|
||||
RRDF_FIELD_SUMMARY_COUNT,
|
||||
k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE,
|
||||
options,
|
||||
FACET_VALUE_UNSET);
|
||||
}
|
||||
dfe_done(k);
|
||||
}
|
||||
buffer_json_object_close(wb); // columns
|
||||
|
||||
buffer_json_member_add_array(wb, "data");
|
||||
{
|
||||
for(FACET_ROW *row = facets->base ; row ;row = row->next) {
|
||||
buffer_json_add_array_item_array(wb); // each row
|
||||
buffer_json_add_array_item_uint64(wb, row->usec);
|
||||
|
||||
FACET_KEY *k;
|
||||
dfe_start_read(facets->keys, k)
|
||||
{
|
||||
FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
|
||||
|
||||
if(unlikely(k->dynamic.cb)) {
|
||||
if(unlikely(!rkv))
|
||||
rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv));
|
||||
|
||||
k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data);
|
||||
}
|
||||
else
|
||||
buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET);
|
||||
}
|
||||
dfe_done(k);
|
||||
buffer_json_array_close(wb); // each row
|
||||
}
|
||||
}
|
||||
buffer_json_array_close(wb); // data
|
||||
|
||||
buffer_json_member_add_string(wb, "default_sort_column", "timestamp");
|
||||
buffer_json_member_add_array(wb, "default_charts");
|
||||
buffer_json_array_close(wb);
|
||||
|
||||
buffer_json_member_add_object(wb, "items");
|
||||
{
|
||||
buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated);
|
||||
buffer_json_member_add_uint64(wb, "matched", facets->operations.matched);
|
||||
buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);
|
||||
buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);
|
||||
buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);
|
||||
buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);
|
||||
}
|
||||
buffer_json_object_close(wb); // items
|
||||
|
||||
buffer_json_member_add_object(wb, "stats");
|
||||
{
|
||||
buffer_json_member_add_uint64(wb, "first", facets->operations.first);
|
||||
buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards);
|
||||
buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards);
|
||||
buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before);
|
||||
buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after);
|
||||
buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends);
|
||||
buffer_json_member_add_uint64(wb, "appends", facets->operations.appends);
|
||||
buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts);
|
||||
}
|
||||
buffer_json_object_close(wb); // items
|
||||
|
||||
}
|
||||
62
libnetdata/facets/facets.h
Normal file
62
libnetdata/facets/facets.h
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
#ifndef FACETS_H
|
||||
#define FACETS_H 1
|
||||
|
||||
#include "../libnetdata.h"
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
FACET_KEY_OPTION_FACET = (1 << 0), // filterable values
|
||||
FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value
|
||||
FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table
|
||||
FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table
|
||||
FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS)
|
||||
FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap
|
||||
FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter
|
||||
} FACET_KEY_OPTIONS;
|
||||
|
||||
typedef struct facet_row_key_value {
|
||||
const char *tmp;
|
||||
BUFFER *wb;
|
||||
} FACET_ROW_KEY_VALUE;
|
||||
|
||||
typedef struct facet_row {
|
||||
usec_t usec;
|
||||
DICTIONARY *dict;
|
||||
struct facet_row *prev, *next;
|
||||
} FACET_ROW;
|
||||
|
||||
typedef struct facets FACETS;
|
||||
typedef struct facet_key FACET_KEY;
|
||||
|
||||
#define FACET_STRING_HASH_SIZE 19
|
||||
void facets_string_hash(const char *src, char *out);
|
||||
|
||||
typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data);
|
||||
typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *wb, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
|
||||
FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data);
|
||||
FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data);
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table
|
||||
FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
|
||||
} FACETS_OPTIONS;
|
||||
|
||||
FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);
|
||||
void facets_destroy(FACETS *facets);
|
||||
|
||||
void facets_accepted_param(FACETS *facets, const char *param);
|
||||
|
||||
void facets_rows_begin(FACETS *facets);
|
||||
void facets_row_finished(FACETS *facets, usec_t usec);
|
||||
|
||||
FACET_KEY *facets_register_key(FACETS *facets, const char *param, FACET_KEY_OPTIONS options);
|
||||
void facets_set_query(FACETS *facets, const char *query);
|
||||
void facets_set_items(FACETS *facets, uint32_t items);
|
||||
void facets_set_anchor(FACETS *facets, usec_t anchor);
|
||||
void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options);
|
||||
|
||||
void facets_add_key_value(FACETS *facets, const char *key, const char *value);
|
||||
void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len);
|
||||
|
||||
void facets_report(FACETS *facets, BUFFER *wb);
|
||||
|
||||
#endif
|
||||
|
|
@ -1966,3 +1966,135 @@ int hash256_string(const unsigned char *string, size_t size, char *hash) {
|
|||
EVP_MD_CTX_destroy(ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Returns 1 if an absolute period was requested or 0 if it was a relative period
|
||||
bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) {
|
||||
time_t now = now_realtime_sec() - 1;
|
||||
|
||||
if(now_ptr)
|
||||
*now_ptr = now;
|
||||
|
||||
int absolute_period_requested = -1;
|
||||
long long after_requested, before_requested;
|
||||
|
||||
before_requested = *before;
|
||||
after_requested = *after;
|
||||
|
||||
// allow relative for before (smaller than API_RELATIVE_TIME_MAX)
|
||||
if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
|
||||
// if the user asked for a positive relative time,
|
||||
// flip it to a negative
|
||||
if(before_requested > 0)
|
||||
before_requested = -before_requested;
|
||||
|
||||
before_requested = now + before_requested;
|
||||
absolute_period_requested = 0;
|
||||
}
|
||||
|
||||
// allow relative for after (smaller than API_RELATIVE_TIME_MAX)
|
||||
if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
|
||||
if(after_requested > 0)
|
||||
after_requested = -after_requested;
|
||||
|
||||
// if the user didn't give an after, use the number of points
|
||||
// to give a sane default
|
||||
if(after_requested == 0)
|
||||
after_requested = -600;
|
||||
|
||||
// since the query engine now returns inclusive timestamps
|
||||
// it is awkward to return 6 points when after=-5 is given
|
||||
// so for relative queries we add 1 second, to give
|
||||
// more predictable results to users.
|
||||
after_requested = before_requested + after_requested + 1;
|
||||
absolute_period_requested = 0;
|
||||
}
|
||||
|
||||
if(absolute_period_requested == -1)
|
||||
absolute_period_requested = 1;
|
||||
|
||||
// check if the parameters are flipped
|
||||
if(after_requested > before_requested) {
|
||||
long long t = before_requested;
|
||||
before_requested = after_requested;
|
||||
after_requested = t;
|
||||
}
|
||||
|
||||
// if the query requests future data
|
||||
// shift the query back to be in the present time
|
||||
// (this may also happen because of the rules above)
|
||||
if(before_requested > now) {
|
||||
long long delta = before_requested - now;
|
||||
before_requested -= delta;
|
||||
after_requested -= delta;
|
||||
}
|
||||
|
||||
time_t absolute_minimum_time = now - (10 * 365 * 86400);
|
||||
time_t absolute_maximum_time = now + (1 * 365 * 86400);
|
||||
|
||||
if (after_requested < absolute_minimum_time && !unittest_running)
|
||||
after_requested = absolute_minimum_time;
|
||||
|
||||
if (after_requested > absolute_maximum_time && !unittest_running)
|
||||
after_requested = absolute_maximum_time;
|
||||
|
||||
if (before_requested < absolute_minimum_time && !unittest_running)
|
||||
before_requested = absolute_minimum_time;
|
||||
|
||||
if (before_requested > absolute_maximum_time && !unittest_running)
|
||||
before_requested = absolute_maximum_time;
|
||||
|
||||
*before = before_requested;
|
||||
*after = after_requested;
|
||||
|
||||
return (absolute_period_requested != 1);
|
||||
}
|
||||
|
||||
int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size) {
|
||||
static const unsigned char base64_table[256] = {
|
||||
['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
|
||||
['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
|
||||
['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
|
||||
['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
|
||||
['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
|
||||
['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
|
||||
['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
|
||||
['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
|
||||
[0 ... '+' - 1] = 255,
|
||||
['+' + 1 ... '/' - 1] = 255,
|
||||
['9' + 1 ... 'A' - 1] = 255,
|
||||
['Z' + 1 ... 'a' - 1] = 255,
|
||||
['z' + 1 ... 255] = 255
|
||||
};
|
||||
|
||||
size_t count = 0;
|
||||
unsigned int tmp = 0;
|
||||
int i, bit;
|
||||
|
||||
if (decoded_size < 1)
|
||||
return 0; // Buffer size must be at least 1 for null termination
|
||||
|
||||
for (i = 0, bit = 0; encoded[i]; i++) {
|
||||
unsigned char value = base64_table[(unsigned char)encoded[i]];
|
||||
if (value > 63)
|
||||
return -1; // Invalid character in input
|
||||
|
||||
tmp = tmp << 6 | value;
|
||||
if (++bit == 4) {
|
||||
if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full
|
||||
decoded[count++] = (tmp >> 16) & 0xFF;
|
||||
decoded[count++] = (tmp >> 8) & 0xFF;
|
||||
decoded[count++] = tmp & 0xFF;
|
||||
tmp = 0;
|
||||
bit = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (bit > 0 && count + 1 < decoded_size) {
|
||||
tmp <<= 6 * (4 - bit);
|
||||
if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF;
|
||||
if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF;
|
||||
}
|
||||
|
||||
decoded[count] = '\0'; // Null terminate the output string
|
||||
return count;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -836,6 +836,7 @@ extern char *netdata_configured_host_prefix;
|
|||
#include "yaml.h"
|
||||
#include "http/http_defs.h"
|
||||
#include "gorilla/gorilla.h"
|
||||
#include "facets/facets.h"
|
||||
#include "dyn_conf/dyn_conf.h"
|
||||
|
||||
// BEWARE: this exists in alarm-notify.sh
|
||||
|
|
@ -979,6 +980,14 @@ typedef enum {
|
|||
void timing_action(TIMING_ACTION action, TIMING_STEP step);
|
||||
|
||||
int hash256_string(const unsigned char *string, size_t size, char *hash);
|
||||
|
||||
extern bool unittest_running;
|
||||
#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
|
||||
|
||||
bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running);
|
||||
|
||||
int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size);
|
||||
|
||||
# ifdef __cplusplus
|
||||
}
|
||||
# endif
|
||||
|
|
|
|||
|
|
@ -11,27 +11,27 @@
|
|||
|
||||
typedef enum web_client_acl {
|
||||
WEB_CLIENT_ACL_NONE = (0),
|
||||
WEB_CLIENT_ACL_NOCHECK = (0), // Don't check anything - this should work on all channels
|
||||
WEB_CLIENT_ACL_DASHBOARD = (1 << 0),
|
||||
WEB_CLIENT_ACL_REGISTRY = (1 << 1),
|
||||
WEB_CLIENT_ACL_BADGE = (1 << 2),
|
||||
WEB_CLIENT_ACL_MGMT = (1 << 3),
|
||||
WEB_CLIENT_ACL_STREAMING = (1 << 4),
|
||||
WEB_CLIENT_ACL_NETDATACONF = (1 << 5),
|
||||
WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 6),
|
||||
WEB_CLIENT_ACL_SSL_FORCE = (1 << 7),
|
||||
WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8),
|
||||
WEB_CLIENT_ACL_ACLK = (1 << 9),
|
||||
WEB_CLIENT_ACL_WEBRTC = (1 << 10),
|
||||
WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 11), // allow unprotected access if bearer is not enabled in netdata
|
||||
WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 12), // allow access only if a valid bearer is used
|
||||
WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
|
||||
WEB_CLIENT_ACL_DASHBOARD = (1 << 1),
|
||||
WEB_CLIENT_ACL_REGISTRY = (1 << 2),
|
||||
WEB_CLIENT_ACL_BADGE = (1 << 3),
|
||||
WEB_CLIENT_ACL_MGMT = (1 << 4),
|
||||
WEB_CLIENT_ACL_STREAMING = (1 << 5),
|
||||
WEB_CLIENT_ACL_NETDATACONF = (1 << 6),
|
||||
WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7),
|
||||
WEB_CLIENT_ACL_SSL_FORCE = (1 << 8),
|
||||
WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9),
|
||||
WEB_CLIENT_ACL_ACLK = (1 << 10),
|
||||
WEB_CLIENT_ACL_WEBRTC = (1 << 11),
|
||||
WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
|
||||
WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
|
||||
} WEB_CLIENT_ACL;
|
||||
|
||||
#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)
|
||||
#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED)
|
||||
|
||||
#ifdef NETDATA_DEV_MODE
|
||||
#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD
|
||||
#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK
|
||||
#else
|
||||
#define ACL_DEV_OPEN_ACCESS 0
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
#define WORKER_UTILIZATION_MAX_JOB_TYPES 50
|
||||
|
||||
typedef enum {
|
||||
typedef enum __attribute__((packed)) {
|
||||
WORKER_METRIC_EMPTY = 0,
|
||||
WORKER_METRIC_IDLE_BUSY = 1,
|
||||
WORKER_METRIC_ABSOLUTE = 2,
|
||||
|
|
|
|||
|
|
@ -1290,6 +1290,11 @@ if [ "$(id -u)" -eq 0 ]; then
|
|||
run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
|
||||
run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/local-listeners"
|
||||
fi
|
||||
|
||||
if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin" ]; then
|
||||
run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
|
||||
run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
|
||||
fi
|
||||
else
|
||||
# non-privileged user installation
|
||||
run chown "${NETDATA_USER}:${NETDATA_GROUP}" "${NETDATA_LOG_DIR}"
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PER
|
|||
static inline void registry_json_header(RRDHOST *host, struct web_client *w, const char *action, const char *status) {
|
||||
buffer_flush(w->response.data);
|
||||
w->response.data->content_type = CT_APPLICATION_JSON;
|
||||
buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_string(w->response.data, "action", action);
|
||||
buffer_json_member_add_string(w->response.data, "status", status);
|
||||
buffer_json_member_add_string(w->response.data, "hostname", rrdhost_registry_hostname(host));
|
||||
|
|
|
|||
|
|
@ -874,7 +874,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
|
|||
sq[0] = '"';
|
||||
}
|
||||
|
||||
buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
buffer_json_member_add_uint64(wb, "api", 1);
|
||||
buffer_json_member_add_string(wb, "id", qt->id);
|
||||
|
|
@ -1289,7 +1289,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
|
|||
sq[0] = '\'';
|
||||
}
|
||||
|
||||
buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_uint64(wb, "api", 2);
|
||||
|
||||
if(options & RRDR_OPTION_DEBUG) {
|
||||
|
|
|
|||
|
|
@ -38,8 +38,6 @@ typedef enum {
|
|||
|
||||
#define HOSTNAME_MAX 1024
|
||||
|
||||
#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
|
||||
|
||||
#define DATASOURCE_FORMAT_JSON "json"
|
||||
#define DATASOURCE_FORMAT_JSON2 "json2"
|
||||
#define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
|
||||
|
|
|
|||
|
|
@ -2075,88 +2075,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
|
|||
}
|
||||
#endif // NETDATA_INTERNAL_CHECKS
|
||||
|
||||
// Returns 1 if an absolute period was requested or 0 if it was a relative period
|
||||
bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) {
|
||||
time_t now = now_realtime_sec() - 1;
|
||||
|
||||
if(now_ptr)
|
||||
*now_ptr = now;
|
||||
|
||||
int absolute_period_requested = -1;
|
||||
long long after_requested, before_requested;
|
||||
|
||||
before_requested = *before;
|
||||
after_requested = *after;
|
||||
|
||||
// allow relative for before (smaller than API_RELATIVE_TIME_MAX)
|
||||
if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
|
||||
// if the user asked for a positive relative time,
|
||||
// flip it to a negative
|
||||
if(before_requested > 0)
|
||||
before_requested = -before_requested;
|
||||
|
||||
before_requested = now + before_requested;
|
||||
absolute_period_requested = 0;
|
||||
}
|
||||
|
||||
// allow relative for after (smaller than API_RELATIVE_TIME_MAX)
|
||||
if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
|
||||
if(after_requested > 0)
|
||||
after_requested = -after_requested;
|
||||
|
||||
// if the user didn't give an after, use the number of points
|
||||
// to give a sane default
|
||||
if(after_requested == 0)
|
||||
after_requested = -600;
|
||||
|
||||
// since the query engine now returns inclusive timestamps
|
||||
// it is awkward to return 6 points when after=-5 is given
|
||||
// so for relative queries we add 1 second, to give
|
||||
// more predictable results to users.
|
||||
after_requested = before_requested + after_requested + 1;
|
||||
absolute_period_requested = 0;
|
||||
}
|
||||
|
||||
if(absolute_period_requested == -1)
|
||||
absolute_period_requested = 1;
|
||||
|
||||
// check if the parameters are flipped
|
||||
if(after_requested > before_requested) {
|
||||
long long t = before_requested;
|
||||
before_requested = after_requested;
|
||||
after_requested = t;
|
||||
}
|
||||
|
||||
// if the query requests future data
|
||||
// shift the query back to be in the present time
|
||||
// (this may also happen because of the rules above)
|
||||
if(before_requested > now) {
|
||||
long long delta = before_requested - now;
|
||||
before_requested -= delta;
|
||||
after_requested -= delta;
|
||||
}
|
||||
|
||||
time_t absolute_minimum_time = now - (10 * 365 * 86400);
|
||||
time_t absolute_maximum_time = now + (1 * 365 * 86400);
|
||||
|
||||
if (after_requested < absolute_minimum_time && !unittest_running)
|
||||
after_requested = absolute_minimum_time;
|
||||
|
||||
if (after_requested > absolute_maximum_time && !unittest_running)
|
||||
after_requested = absolute_maximum_time;
|
||||
|
||||
if (before_requested < absolute_minimum_time && !unittest_running)
|
||||
before_requested = absolute_minimum_time;
|
||||
|
||||
if (before_requested > absolute_maximum_time && !unittest_running)
|
||||
before_requested = absolute_maximum_time;
|
||||
|
||||
*before = before_requested;
|
||||
*after = after_requested;
|
||||
|
||||
return (absolute_period_requested != 1);
|
||||
}
|
||||
|
||||
// #define DEBUG_QUERY_LOGIC 1
|
||||
|
||||
#ifdef DEBUG_QUERY_LOGIC
|
||||
|
|
@ -2283,7 +2201,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
|
|||
}
|
||||
|
||||
// convert our before_wanted and after_wanted to absolute
|
||||
rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
|
||||
rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL, unittest_running);
|
||||
query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
|
||||
|
||||
if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
|
||||
|
|
|
|||
|
|
@ -206,8 +206,6 @@ RRDR *rrd2rrdr_legacy(
|
|||
RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
|
||||
bool query_target_calculate_window(struct query_target *qt);
|
||||
|
||||
bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
|
|||
size_t examined_dimensions, usec_t duration,
|
||||
WEIGHTS_STATS *stats) {
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
|
||||
points, method, group, options, shifts, examined_dimensions, duration, stats);
|
||||
|
|
@ -221,7 +221,7 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
|
|||
size_t examined_dimensions, usec_t duration,
|
||||
WEIGHTS_STATS *stats) {
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
|
||||
points, method, group, options, shifts, examined_dimensions, duration, stats);
|
||||
|
|
@ -739,7 +739,7 @@ static size_t registered_results_to_json_multinode_no_group_by(
|
|||
size_t examined_dimensions, struct query_weights_data *qwd,
|
||||
WEIGHTS_STATS *stats,
|
||||
struct query_versions *versions) {
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_uint64(wb, "api", 2);
|
||||
|
||||
results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
|
||||
|
|
@ -958,7 +958,7 @@ static size_t registered_results_to_json_multinode_group_by(
|
|||
size_t examined_dimensions, struct query_weights_data *qwd,
|
||||
WEIGHTS_STATS *stats,
|
||||
struct query_versions *versions) {
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_uint64(wb, "api", 2);
|
||||
|
||||
results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
|
||||
|
|
@ -1806,7 +1806,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
|
|||
}
|
||||
};
|
||||
|
||||
if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))
|
||||
if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL, false))
|
||||
buffer_no_cacheable(wb);
|
||||
else
|
||||
buffer_cacheable(wb);
|
||||
|
|
@ -1823,7 +1823,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
|
|||
if(qwr->baseline_before <= API_RELATIVE_TIME_MAX)
|
||||
qwr->baseline_before += qwr->after;
|
||||
|
||||
rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL);
|
||||
rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL, false);
|
||||
|
||||
if (qwr->baseline_before <= qwr->baseline_after) {
|
||||
resp = HTTP_RESP_BAD_REQUEST;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at
|
|||
DICTIONARY *netdata_authorized_bearers = NULL;
|
||||
|
||||
static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {
|
||||
if(endpoint_acl == WEB_CLIENT_ACL_NOCHECK)
|
||||
if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK))
|
||||
// the endpoint is totally public
|
||||
return true;
|
||||
|
||||
|
|
@ -23,11 +23,24 @@ static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL
|
|||
// endpoint does not require a bearer
|
||||
return true;
|
||||
|
||||
if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)) || api_check_bearer_token(w))
|
||||
if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)))
|
||||
// the request is coming from ACLK or WEBRTC (authorized already),
|
||||
// or we have a valid bearer on the request
|
||||
return true;
|
||||
|
||||
// at this point we need a bearer to serve the request
|
||||
// either because:
|
||||
//
|
||||
// 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or
|
||||
// 2. netdata_is_protected_by_bearer == true
|
||||
//
|
||||
|
||||
BEARER_STATUS t = api_check_bearer_token(w);
|
||||
if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED)
|
||||
// we have a valid bearer on the request
|
||||
return true;
|
||||
|
||||
netdata_log_info("BEARER: bearer is required for request: code %d", t);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -60,7 +73,7 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat
|
|||
return HTTP_RESP_BAD_REQUEST;
|
||||
}
|
||||
|
||||
if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
|
||||
if(unlikely(!web_client_check_acl_and_bearer(w, api_commands[i].acl)))
|
||||
return web_client_permission_denied(w);
|
||||
|
||||
char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
|
||||
|
|
|
|||
|
|
@ -11,9 +11,19 @@
|
|||
|
||||
extern bool netdata_is_protected_by_bearer;
|
||||
extern DICTIONARY *netdata_authorized_bearers;
|
||||
bool api_check_bearer_token(struct web_client *w);
|
||||
bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
|
||||
void bearer_tokens_init(void);
|
||||
typedef enum __attribute__((packed)) {
|
||||
BEARER_STATUS_NO_BEARER_IN_HEADERS,
|
||||
BEARER_STATUS_BEARER_DOES_NOT_FIT,
|
||||
BEARER_STATUS_NOT_PARSABLE,
|
||||
BEARER_STATUS_EXTRACTED_FROM_HEADER,
|
||||
BEARER_STATUS_NO_BEARERS_DICTIONARY,
|
||||
BEARER_STATUS_NOT_FOUND_IN_DICTIONARY,
|
||||
BEARER_STATUS_EXPIRED,
|
||||
BEARER_STATUS_AVAILABLE_AND_VALIDATED,
|
||||
} BEARER_STATUS;
|
||||
|
||||
BEARER_STATUS api_check_bearer_token(struct web_client *w);
|
||||
BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
|
||||
|
||||
struct web_api_command {
|
||||
const char *command;
|
||||
|
|
|
|||
|
|
@ -941,7 +941,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
|
|||
char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");
|
||||
if(cookie)
|
||||
strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);
|
||||
else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))
|
||||
else if(extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)) != BEARER_STATUS_EXTRACTED_FROM_HEADER)
|
||||
person_guid[0] = '\0';
|
||||
|
||||
char action = '\0';
|
||||
|
|
@ -1199,7 +1199,7 @@ static void host_collectors(RRDHOST *host, BUFFER *wb) {
|
|||
|
||||
extern int aclk_connected;
|
||||
inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) {
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
|
||||
buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
|
||||
buffer_json_member_add_string(wb, "uid", host->machine_guid);
|
||||
|
|
@ -1319,7 +1319,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
|
|||
buffer_flush(wb);
|
||||
wb->content_type = CT_APPLICATION_JSON;
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
ml_host_get_detection_info(host, wb);
|
||||
buffer_json_finalize(wb);
|
||||
|
||||
|
|
@ -1424,7 +1424,7 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
|
|||
wb->content_type = CT_APPLICATION_JSON;
|
||||
buffer_no_cacheable(wb);
|
||||
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
host_functions2json(host, wb);
|
||||
buffer_json_finalize(wb);
|
||||
|
||||
|
|
|
|||
|
|
@ -54,13 +54,13 @@ static time_t bearer_get_token(uuid_t *uuid) {
|
|||
|
||||
#define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "
|
||||
|
||||
bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
|
||||
BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
|
||||
const char *req = buffer_tostring(w->response.data);
|
||||
size_t req_len = buffer_strlen(w->response.data);
|
||||
const char *bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);
|
||||
|
||||
if(!bearer)
|
||||
return false;
|
||||
return BEARER_STATUS_NO_BEARER_IN_HEADERS;
|
||||
|
||||
const char *token_start = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;
|
||||
|
||||
|
|
@ -69,26 +69,33 @@ bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t d
|
|||
|
||||
const char *token_end = token_start + UUID_STR_LEN - 1 + 2;
|
||||
if (token_end > req + req_len)
|
||||
return false;
|
||||
return BEARER_STATUS_BEARER_DOES_NOT_FIT;
|
||||
|
||||
strncpyz(dst, token_start, dst_len - 1);
|
||||
uuid_t uuid;
|
||||
if (uuid_parse(dst, uuid) != 0)
|
||||
return false;
|
||||
return BEARER_STATUS_NOT_PARSABLE;
|
||||
|
||||
return true;
|
||||
return BEARER_STATUS_EXTRACTED_FROM_HEADER;
|
||||
}
|
||||
|
||||
bool api_check_bearer_token(struct web_client *w) {
|
||||
BEARER_STATUS api_check_bearer_token(struct web_client *w) {
|
||||
if(!netdata_authorized_bearers)
|
||||
return false;
|
||||
return BEARER_STATUS_NO_BEARERS_DICTIONARY;
|
||||
|
||||
char token[UUID_STR_LEN];
|
||||
if(!extract_bearer_token_from_request(w, token, sizeof(token)))
|
||||
return false;
|
||||
BEARER_STATUS t = extract_bearer_token_from_request(w, token, sizeof(token));
|
||||
if(t != BEARER_STATUS_EXTRACTED_FROM_HEADER)
|
||||
return t;
|
||||
|
||||
struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);
|
||||
return z && z->expires_s > now_monotonic_sec();
|
||||
if(!z)
|
||||
return BEARER_STATUS_NOT_FOUND_IN_DICTIONARY;
|
||||
|
||||
if(z->expires_s < now_monotonic_sec())
|
||||
return BEARER_STATUS_EXPIRED;
|
||||
|
||||
return BEARER_STATUS_AVAILABLE_AND_VALIDATED;
|
||||
}
|
||||
|
||||
static bool verify_agent_uuids(const char *machine_guid, const char *node_id, const char *claim_id) {
|
||||
|
|
@ -153,7 +160,7 @@ int api_v2_bearer_protection(RRDHOST *host __maybe_unused, struct web_client *w
|
|||
|
||||
BUFFER *wb = w->response.data;
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
|
||||
buffer_json_finalize(wb);
|
||||
|
||||
|
|
@ -192,7 +199,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
|
|||
|
||||
BUFFER *wb = w->response.data;
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
|
||||
buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
|
||||
buffer_json_member_add_uuid(wb, "token", &uuid);
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz
|
|||
web_client_timeout_checkpoint_set(w, 0);
|
||||
web_client_decode_path_and_query_string(w, path);
|
||||
path = (char *)buffer_tostring(w->url_path_decoded);
|
||||
w->response.code = (short)web_client_api_request_with_node_selection(rrdb.localhost, w, path);
|
||||
w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
|
||||
web_client_timeout_checkpoint_response_ready(w, NULL);
|
||||
|
||||
size_t sent_bytes = 0;
|
||||
|
|
@ -643,7 +643,7 @@ int webrtc_new_connection(const char *sdp, BUFFER *wb) {
|
|||
}
|
||||
|
||||
buffer_flush(wb);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, false);
|
||||
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
|
||||
wb->content_type = CT_APPLICATION_JSON;
|
||||
|
||||
WEBRTC_CONN *conn = webrtc_create_connection();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue