archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-17 19:22:40 +00:00
Code Issues Projects Releases Wiki Activity

Labels with dictionary ()

Browse source 
* squashed and rebased to master

* fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h

* added unittest for UTF-8 multibyte sanitization

* Fix unit test compilation

* Fix CMake build

* remove double sanitizer for opentsdb; cleanup sanitize_json_string()

* rename error_description to error_message to avoid conflict with json-c

* revert last and undef error_description from json-c

* more unittests; attempt to fix protobuf map issue

* get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer

* added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted

* better sorted dictionary checking

* proper unittesting for sorted dictionaries

* call dictionary deletion callback when destroying the dictionary

* remove obsolete variable

* Fix exporting unit tests

* Fix k8s label parsing test

* workaround for cmocka and strdupz()

* Bypass cmocka memory allocation check

* Revert "Bypass cmocka memory allocation check"

This reverts commit 4c49923839.

* Revert "workaround for cmocka and strdupz()"

This reverts commit 7bebee0480.

* Bypass cmocka memory allocation checks

* respect json formatting for chart labels

* cloud sends colons

* print the value only once

* allow parenthesis in values and spaces; make stream sender send quotes for values

Co-authored-by: Vladimir Kobal <vlad@prokk.net>
This commit is contained in:
Costa Tsaousis 2022-06-13 20:35:45 +03:00 • committed by GitHub
parent 4c64b8ea4f
commit 1b0f6c6b22
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
61 changed files with 2004 additions and 1437 deletions

6
CMakeLists.txt
View file

@ -1412,6 +1412,7 @@ if(BUILD_TESTING)
exporting/tests/exporting_doubles.c
exporting/tests/netdata_doubles.c
exporting/tests/system_doubles.c
database/rrdlabels.c
)
set(TEST_NAME exporting_engine)
set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
@ -1569,11 +1570,6 @@ endif()
database/rrd.h
)
add_executable(cgroups_testdriver ${CGROUPS_TEST_FILES} ${CGROUPS_PLUGIN_FILES})
target_link_options(
cgroups_testdriver
PRIVATE
-Wl,--wrap=add_label_to_list
)
target_link_libraries(cgroups_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
add_test(NAME test_cgroups COMMAND cgroups_testdriver)

3
Makefile.am
View file

@ -1166,6 +1166,7 @@ if ENABLE_UNITTESTS
$(EXPORTING_ENGINE_TEST_FILES) \
$(EXPORTING_ENGINE_FILES) \
$(LIBNETDATA_FILES) \
database/rrdlabels.c \
$(NULL)
exporting_tests_exporting_engine_testdriver_CFLAGS = \
$(AM_CFLAGS) \
@ -1259,7 +1260,7 @@ endif
$(NULL)
collectors_cgroups_plugin_tests_cgroups_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
collectors_cgroups_plugin_tests_cgroups_testdriver_LDFLAGS = \
-Wl,--wrap=add_label_to_list \
-Wl,--wrap=rrdlabels_add \
$(NULL)
endif

22
aclk/aclk_api.c
View file

@ -45,13 +45,15 @@ void aclk_single_update_enable()
}
#endif /* ENABLE_ACLK */
struct label *add_aclk_host_labels(struct label *label) {
void add_aclk_host_labels(void) {
DICTIONARY *labels = localhost->host_labels;
#ifdef ENABLE_ACLK
label = add_label_to_list(label, "_aclk_ng_available", "true", LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_aclk_ng_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
#else
label = add_label_to_list(label, "_aclk_ng_available", "false", LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_aclk_ng_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
#endif
label = add_label_to_list(label, "_aclk_legacy_available", "false", LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_aclk_legacy_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
#ifdef ENABLE_ACLK
ACLK_PROXY_TYPE aclk_proxy;
char *proxy_str;
@ -70,16 +72,16 @@ struct label *add_aclk_host_labels(struct label *label) {
}
int mqtt5 = config_get_boolean(CONFIG_SECTION_CLOUD, "mqtt5", CONFIG_BOOLEAN_NO);
label = add_label_to_list(label, "_mqtt_version", mqtt5 ? "5" : "3", LABEL_SOURCE_AUTO);
label = add_label_to_list(label, "_aclk_impl", "Next Generation", LABEL_SOURCE_AUTO);
label = add_label_to_list(label, "_aclk_proxy", proxy_str, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_mqtt_version", mqtt5 ? "5" : "3", RRDLABEL_SRC_AUTO);
rrdlabels_add(labels, "_aclk_impl", "Next Generation", RRDLABEL_SRC_AUTO);
rrdlabels_add(labels, "_aclk_proxy", proxy_str, RRDLABEL_SRC_AUTO);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "true", LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
#else
label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "false", LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
#endif
#endif
return label;
}
char *aclk_state(void) {

2
aclk/aclk_api.h
View file

@ -52,7 +52,7 @@ void aclk_host_state_update(RRDHOST *host, int connect);
#endif
struct label *add_aclk_host_labels(struct label *label);
void add_aclk_host_labels(void);
char *aclk_state(void);
char *aclk_state_json(void);

9
aclk/schema-wrappers/chart_stream.cc
View file

@ -76,7 +76,7 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance)
freez((char*)instance->id);
freez((char*)instance->claim_id);
free_label_list(instance->label_head);
rrdlabels_destroy(instance->chart_labels);
freez((char*)instance->config_hash);
}
@ -85,7 +85,6 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
{
google::protobuf::Map<std::string, std::string> *map;
aclk_lib::v1::ACLKMessagePosition *pos;
struct label *label;
chart->set_id(update->id);
chart->set_claim_id(update->claim_id);
@ -93,11 +92,7 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
chart->set_name(update->name);
map = chart->mutable_chart_labels();
label = update->label_head;
while (label) {
map->insert({label->key, label->value});
label = label->next;
}
rrdlabels_walkthrough_read(update->chart_labels, label_add_to_map_callback, map);
switch (update->memory_mode) {
case RRD_MEMORY_MODE_NONE:

2
aclk/schema-wrappers/chart_stream.h
View file

@ -57,7 +57,7 @@ struct chart_instance_updated {
const char *node_id;
const char *name;
struct label *label_head;
DICTIONARY *chart_labels;
RRD_MEMORY_MODE memory_mode;

8
aclk/schema-wrappers/node_info.cc
View file

@ -6,7 +6,6 @@
static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct aclk_node_info *data)
{
struct label *label;
google::protobuf::Map<std::string, std::string> *map;
if (data->name)
@ -67,12 +66,7 @@ static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct acl
ml_info->set_ml_enabled(data->ml_info.ml_enabled);
map = info->mutable_host_labels();
label = data->host_labels_head;
while (label) {
map->insert({label->key, label->value});
label = label->next;
}
rrdlabels_walkthrough_read(data->host_labels_ptr, label_add_to_map_callback, map);
return 0;
}

5
aclk/schema-wrappers/node_info.h
View file

@ -4,9 +4,10 @@
#define ACLK_SCHEMA_WRAPPER_NODE_INFO_H
#include <stdlib.h>
#include <stdint.h>
#include "database/rrd.h"
#include "capability.h"
#include "database/rrd.h"
#ifdef __cplusplus
extern "C" {
@ -54,7 +55,7 @@ struct aclk_node_info {
char *machine_guid;
struct label *host_labels_head;
DICTIONARY *host_labels_ptr;
struct machine_learning_info ml_info;
};

7
aclk/schema-wrappers/schema_wrapper_utils.cc
View file

@ -13,3 +13,10 @@ void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, st
tv->tv_sec = ts.seconds();
tv->tv_usec = ts.nanos()/1000;
}
int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
(void)ls;
auto map = (google::protobuf::Map<std::string, std::string> *)data;
map->insert({name, value});
return 1;
}

4
aclk/schema-wrappers/schema_wrapper_utils.h
View file

@ -3,8 +3,11 @@
#ifndef SCHEMA_WRAPPER_UTILS_H
#define SCHEMA_WRAPPER_UTILS_H
#include "database/rrd.h"
#include <sys/time.h>
#include <google/protobuf/timestamp.pb.h>
#include <google/protobuf/map.h>
#if GOOGLE_PROTOBUF_VERSION < 3001000
#define PROTO_COMPAT_MSG_SIZE(msg) (size_t)msg.ByteSize();
@ -16,5 +19,6 @@
void set_google_timestamp_from_timeval(struct timeval tv, google::protobuf::Timestamp *ts);
void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, struct timeval *tv);
int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
#endif /* SCHEMA_WRAPPER_UTILS_H */

142
collectors/cgroups.plugin/sys_fs_cgroup.c
View file

@ -776,7 +776,7 @@ struct cgroup {
char *chart_title;
struct label *chart_labels;
DICTIONARY *chart_labels;
struct cpuacct_stat cpuacct_stat;
struct cpuacct_usage cpuacct_usage;
@ -1735,34 +1735,24 @@ static inline void substitute_dots_in_id(char *s) {
}
}
char *k8s_parse_resolved_name(struct label **labels, char *data) {
// ----------------------------------------------------------------------------
// parse k8s labels
char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) {
// the first word, up to the first space is the name
char *name = mystrsep(&data, " ");
if (!data) {
return name;
}
while (data) {
char *key = mystrsep(&data, "=");
char *value;
if (data && *data == ',') {
value = "";
*data++ = '\0';
} else {
value = mystrsep(&data, ",");
}
value = strip_double_quotes(value, 1);
if (!key || *key == '\0' || !value || *value == '\0')
continue;
*labels = add_label_to_list(*labels, key, value, LABEL_SOURCE_KUBERNETES);
// the rest are key=value pairs separated by comma
while(data) {
char *pair = mystrsep(&data, ",");
rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S);
}
return name;
}
// ----------------------------------------------------------------------------
static inline void free_pressure(struct pressure *res) {
if (res->some.share_time.st) rrdset_is_obsolete(res->some.share_time.st);
if (res->some.total_time.st) rrdset_is_obsolete(res->some.total_time.st);
@ -1834,7 +1824,7 @@ static inline void cgroup_free(struct cgroup *cg) {
freez(cg->chart_id);
freez(cg->chart_title);
free_label_list(cg->chart_labels);
rrdlabels_destroy(cg->chart_labels);
freez(cg);
@ -1870,31 +1860,33 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {
case 0:
cg->pending_renames = 0;
break;
case 3:
cg->pending_renames = 0;
cg->processed = 1;
break;
}
if (cg->pending_renames || cg->processed) {
return;
}
if (!(new_name && *new_name && *new_name != '\n')) {
return;
}
new_name = trim(new_name);
if (!(new_name)) {
return;
}
if(cg->pending_renames || cg->processed) return;
if(!new_name || !*new_name || *new_name == '\n') return;
if(!(new_name = trim(new_name))) return;
char *name = new_name;
if (!strncmp(new_name, "k8s_", 4)) {
free_label_list(cg->chart_labels);
name = k8s_parse_resolved_name(&cg->chart_labels, new_name);
if(!cg->chart_labels) cg->chart_labels = rrdlabels_create();
// read the new labels and remove the obsolete ones
rrdlabels_unmark_all(cg->chart_labels);
name = k8s_parse_resolved_name_and_labels(cg->chart_labels, new_name);
rrdlabels_remove_all_unmarked(cg->chart_labels);
}
freez(cg->chart_title);
cg->chart_title = cgroup_title_strdupz(name);
freez(cg->chart_id);
cg->chart_id = cgroup_chart_id_strdupz(name);
substitute_dots_in_id(cg->chart_id);
cg->hash_chart = simple_hash(cg->chart_id);
}
@ -3782,7 +3774,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_STACKED
);
rrdset_update_labels(cg->st_cpu, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu, cg->chart_labels);
if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
rrddim_add(cg->st_cpu, "user", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
@ -3855,7 +3847,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_cpu_limit, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu_limit, cg->chart_labels);
if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED))
rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
@ -3908,7 +3900,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu_nr_throttled, cg->chart_labels);
rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
} else {
rrdset_next(cg->st_cpu_nr_throttled);
@ -3934,7 +3926,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu_throttled_time, cg->chart_labels);
rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(cg->st_cpu_throttled_time);
@ -3962,7 +3954,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_cpu_shares, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu_shares, cg->chart_labels);
rrddim_add(cg->st_cpu_shares, "shares", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
} else {
rrdset_next(cg->st_cpu_shares);
@ -3993,7 +3985,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_STACKED
);
rrdset_update_labels(cg->st_cpu_per_core, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_cpu_per_core, cg->chart_labels);
for(i = 0; i < cg->cpuacct_usage.cpus; i++) {
snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
@ -4028,8 +4020,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_STACKED
);
rrdset_update_labels(cg->st_mem, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem, cg->chart_labels);
if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@ -4089,7 +4081,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_AREA
);
rrdset_update_labels(cg->st_writeback, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_writeback, cg->chart_labels);
if(cg->memory.detailed_has_dirty)
rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@ -4124,7 +4116,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_mem_activity, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem_activity, cg->chart_labels);
rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@ -4155,7 +4147,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_pgfaults, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_pgfaults, cg->chart_labels);
rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@ -4187,7 +4179,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_STACKED
);
rrdset_update_labels(cg->st_mem_usage, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem_usage, cg->chart_labels);
rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@ -4254,7 +4246,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_STACKED
);
rrdset_update_labels(cg->st_mem_usage_limit, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem_usage_limit, cg->chart_labels);
rrddim_add(cg->st_mem_usage_limit, "available", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@ -4286,7 +4278,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_AREA
);
rrdset_update_labels(cg->st_mem_utilization, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem_utilization, cg->chart_labels);
rrddim_add(cg->st_mem_utilization, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
} else
@ -4334,8 +4326,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_mem_failcnt, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_mem_failcnt, cg->chart_labels);
rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
@ -4365,7 +4357,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_AREA
);
rrdset_update_labels(cg->st_io, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_io, cg->chart_labels);
rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@ -4397,7 +4389,7 @@ void update_cgroup_charts(int update_every) {
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_serviced_ops, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_serviced_ops, cg->chart_labels);
rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -4428,8 +4420,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_AREA
);
rrdset_update_labels(cg->st_throttle_io, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_throttle_io, cg->chart_labels);
rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@ -4460,8 +4452,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_throttle_serviced_ops, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_throttle_serviced_ops, cg->chart_labels);
rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -4492,8 +4484,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_queued_ops, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_queued_ops, cg->chart_labels);
rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
@ -4524,8 +4516,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(cg->st_merged_ops, cg->chart_labels);
rrdset_update_rrdlabels(cg->st_merged_ops, cg->chart_labels);
rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@ -4562,7 +4554,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4586,7 +4578,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);
@ -4614,7 +4606,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4638,7 +4630,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);
@ -4668,8 +4660,8 @@ void update_cgroup_charts(int update_every) {
, cgroup_containers_chart_priority + 2300
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4693,7 +4685,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);
@ -4723,8 +4715,8 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4748,7 +4740,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);
@ -4779,7 +4771,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4803,7 +4795,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);
@ -4832,7 +4824,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@ -4856,7 +4848,7 @@ void update_cgroup_charts(int update_every) {
, update_every
, RRDSET_TYPE_LINE
);
rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
} else {
rrdset_next(pcs->total_time.st);

2
collectors/cgroups.plugin/sys_fs_cgroup.h
View file

@ -39,6 +39,6 @@ typedef struct netdata_ebpf_cgroup_shm {
#include "../proc.plugin/plugin_proc.h"
extern char *k8s_parse_resolved_name(struct label **labels, char *data);
char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data);
#endif //NETDATA_SYS_FS_CGROUP_H

87
collectors/cgroups.plugin/tests/test_cgroups_plugin.c
View file

@ -8,18 +8,36 @@ int netdata_zero_metrics_enabled = 1;
struct config netdata_config;
char *netdata_configured_primary_plugins_dir = NULL;
struct k8s_test_data {
char *data;
char *name;
char *key[3];
char *value[3];
const char *result_key[3];
const char *result_value[3];
int result_ls[3];
int i;
};
static int read_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data)
{
struct k8s_test_data *test_data = (struct k8s_test_data *)data;
test_data->result_key[test_data->i] = name;
test_data->result_value[test_data->i] = value;
test_data->result_ls[test_data->i] = ls;
test_data->i++;
return 1;
}
static void test_k8s_parse_resolved_name(void **state)
{
UNUSED(state);
struct label *labels = (struct label *)0xff;
struct k8s_test_data {
char *data;
char *name;
char *key[3];
char *value[3];
};
DICTIONARY *labels = rrdlabels_create();
struct k8s_test_data test_data[] = {
// One label
@ -40,29 +58,29 @@ static void test_k8s_parse_resolved_name(void **state)
.key[0] = "label1", .value[0] = "value1" },
// Equals sign in the value
{ .data = "name label1=\"value=1\"",
.name = "name",
.key[0] = "label1", .value[0] = "value=1" },
// { .data = "name label1=\"value=1\"",
// .name = "name",
// .key[0] = "label1", .value[0] = "value=1" },
// Double quotation mark in the value
{ .data = "name label1=\"value\"1\"",
.name = "name",
.key[0] = "label1", .value[0] = "value" },
// { .data = "name label1=\"value\"1\"",
// .name = "name",
// .key[0] = "label1", .value[0] = "value" },
// Escaped double quotation mark in the value
{ .data = "name label1=\"value\\\"1\"",
.name = "name",
.key[0] = "label1", .value[0] = "value\\\"1" },
// { .data = "name label1=\"value\\\"1\"",
// .name = "name",
// .key[0] = "label1", .value[0] = "value\\\"1" },
// Equals sign in the key
{ .data = "name label=1=\"value1\"",
.name = "name",
.key[0] = "label", .value[0] = "1=\"value1\"" },
// { .data = "name label=1=\"value1\"",
// .name = "name",
// .key[0] = "label", .value[0] = "1=\"value1\"" },
// Skipped value
{ .data = "name label1=,label2=\"value2\"",
.name = "name",
.key[0] = "label2", .value[0] = "value2" },
// { .data = "name label1=,label2=\"value2\"",
// .name = "name",
// .key[0] = "label2", .value[0] = "value2" },
// A pair of equals signs
{ .data = "name= =",
@ -78,22 +96,25 @@ static void test_k8s_parse_resolved_name(void **state)
for (int i = 0; test_data[i].data != NULL; i++) {
char *data = strdup(test_data[i].data);
char *name = k8s_parse_resolved_name_and_labels(labels, data);
assert_string_equal(name, test_data[i].name);
rrdlabels_walkthrough_read(labels, read_label_callback, &test_data[i]);
for (int l = 0; l < 3 && test_data[i].key[l] != NULL; l++) {
char *key = test_data[i].key[l];
char *value = test_data[i].value[l];
expect_function_call(__wrap_add_label_to_list);
expect_value(__wrap_add_label_to_list, l, 0xff);
expect_string(__wrap_add_label_to_list, key, key);
expect_string(__wrap_add_label_to_list, value, value);
expect_value(__wrap_add_label_to_list, label_source, LABEL_SOURCE_KUBERNETES);
const char *result_key = test_data[i].result_key[l];
const char *result_value = test_data[i].result_value[l];
int ls = test_data[i].result_ls[l];
assert_string_equal(key, result_key);
assert_string_equal(value, result_value);
assert_int_equal(RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S, ls);
}
char *name = k8s_parse_resolved_name(&labels, data);
assert_string_equal(name, test_data[i].name);
assert_ptr_equal(labels, 0xff);
free(data);
}
}

26
collectors/cgroups.plugin/tests/test_doubles.c
View file

@ -44,22 +44,6 @@ void mountinfo_free_all(struct mountinfo *mi)
UNUSED(mi);
}
struct label *__wrap_add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source)
{
function_called();
check_expected_ptr(l);
check_expected_ptr(key);
check_expected_ptr(value);
check_expected(label_source);
return l;
}
void rrdset_update_labels(RRDSET *st, struct label *labels)
{
UNUSED(st);
UNUSED(labels);
}
RRDSET *rrdset_create_custom(
RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,
const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,
@ -148,7 +132,7 @@ void update_pressure_charts(struct pressure_charts *charts)
}
void netdev_rename_device_add(
const char *host_device, const char *container_device, const char *container_name, struct label *labels)
const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels)
{
UNUSED(host_device);
UNUSED(container_device);
@ -160,3 +144,11 @@ void netdev_rename_device_del(const char *host_device)
{
UNUSED(host_device);
}
void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
{
UNUSED(chart_uuid);
UNUSED(source_type);
UNUSED(label);
UNUSED(value);
}

61
collectors/plugins.d/pluginsd_parser.c
View file

@ -159,22 +159,28 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name
return PARSER_RC_OK;
}
PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source)
PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source)
{
((PARSER_USER_OBJECT *) user)->new_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->new_labels, key, value, source);
if(unlikely(!((PARSER_USER_OBJECT *) user)->new_host_labels))
((PARSER_USER_OBJECT *) user)->new_host_labels = rrdlabels_create();
rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_host_labels, key, value, source);
return PARSER_RC_OK;
}
PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source)
PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source)
{
((PARSER_USER_OBJECT *) user)->chart_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->chart_labels, key, value, source);
if(unlikely(!((PARSER_USER_OBJECT *) user)->new_chart_labels))
((PARSER_USER_OBJECT *) user)->new_chart_labels = rrdlabels_create();
rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_chart_labels, key, value, source);
return PARSER_RC_OK;
}
PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels)
PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels)
{
RRDSET *st = ((PARSER_USER_OBJECT *)user)->st;
if (unlikely(!st)) {
@ -182,21 +188,20 @@ PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label
return PARSER_RC_OK;
}
rrdset_update_labels(st, new_labels);
rrdset_update_rrdlabels(st, new_chart_labels);
return PARSER_RC_OK;
}
PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels)
PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels)
{
UNUSED(user);
if (!host->labels.head) {
host->labels.head = new_labels;
} else {
rrdhost_rdlock(host);
replace_label_list(&host->labels, new_labels);
rrdhost_unlock(host);
}
if(!host->host_labels)
host->host_labels = rrdlabels_create();
rrdlabels_migrate_to_these(host->host_labels, new_host_labels);
return PARSER_RC_OK;
}
@ -615,14 +620,15 @@ PARSER_RC pluginsd_clabel_commit(char **words, void *user, PLUGINSD_ACTION *plu
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
debug(D_PLUGINSD, "requested to commit chart labels");
struct label *chart_labels = ((PARSER_USER_OBJECT *)user)->chart_labels;
((PARSER_USER_OBJECT *)user)->chart_labels = NULL;
PARSER_RC rc = PARSER_RC_OK;
if (plugins_action->clabel_commit_action) {
return plugins_action->clabel_commit_action(user, host, chart_labels);
}
if (plugins_action->clabel_commit_action)
rc = plugins_action->clabel_commit_action(user, host, ((PARSER_USER_OBJECT *)user)->new_chart_labels);
return PARSER_RC_OK;
rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_chart_labels);
((PARSER_USER_OBJECT *)user)->new_chart_labels = NULL;
return rc;
}
PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins_action)
@ -630,16 +636,17 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins
UNUSED(words);
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
debug(D_PLUGINSD, "requested a OVERWRITE a variable");
debug(D_PLUGINSD, "requested to OVERWRITE host labels");
struct label *new_labels = ((PARSER_USER_OBJECT *)user)->new_labels;
((PARSER_USER_OBJECT *)user)->new_labels = NULL;
PARSER_RC rc = PARSER_RC_OK;
if (plugins_action->overwrite_action) {
return plugins_action->overwrite_action(user, host, new_labels);
}
if (plugins_action->overwrite_action)
rc = plugins_action->overwrite_action(user, host, ((PARSER_USER_OBJECT *)user)->new_host_labels);
return PARSER_RC_OK;
rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels);
((PARSER_USER_OBJECT *)user)->new_host_labels = NULL;
return rc;
}
PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)

12
collectors/plugins.d/pluginsd_parser.h
View file

@ -13,8 +13,8 @@ typedef struct parser_user_object {
void *opaque;
struct plugind *cd;
int trust_durations;
struct label *new_labels;
struct label *chart_labels;
DICTIONARY *new_host_labels;
DICTIONARY *new_chart_labels;
size_t count;
int enabled;
uint8_t st_exists;
@ -34,10 +34,10 @@ extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st,
calculated_number value);
extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels);
extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source);
extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source);
extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels);
extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels);
extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source);
#endif //NETDATA_PLUGINSD_PARSER_H

2
collectors/proc.plugin/plugin_proc.h
View file

@ -54,7 +54,7 @@ extern unsigned long long zfs_arcstats_shrinkable_cache_size_bytes;
// netdev renames
extern void netdev_rename_device_add(
const char *host_device, const char *container_device, const char *container_name, struct label *labels);
const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels);
extern void netdev_rename_device_del(const char *host_device);
#include "proc_self_mountinfo.h"

44
collectors/proc.plugin/proc_net_dev.c
View file

@ -90,7 +90,7 @@ static struct netdev {
const char *chart_family;
struct label *chart_labels;
DICTIONARY *chart_labels;
int flipped;
unsigned long priority;
@ -273,7 +273,7 @@ static void netdev_free_chart_strings(struct netdev *d) {
static void netdev_free(struct netdev *d) {
netdev_charts_release(d);
netdev_free_chart_strings(d);
free_label_list(d->chart_labels);
rrdlabels_destroy(d->chart_labels);
freez((void *)d->name);
freez((void *)d->filename_speed);
@ -295,7 +295,7 @@ static struct netdev_rename {
const char *container_device;
const char *container_name;
struct label *chart_labels;
DICTIONARY *chart_labels;
int processed;
@ -316,9 +316,7 @@ static struct netdev_rename *netdev_rename_find(const char *host_device, uint32_
}
// other threads can call this function to register a rename to a netdev
void netdev_rename_device_add(
const char *host_device, const char *container_device, const char *container_name, struct label *labels)
{
void netdev_rename_device_add(const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels) {
netdata_mutex_lock(&netdev_rename_mutex);
uint32_t hash = simple_hash(host_device);
@ -328,7 +326,8 @@ void netdev_rename_device_add(
r->host_device = strdupz(host_device);
r->container_device = strdupz(container_device);
r->container_name = strdupz(container_name);
update_label_list(&r->chart_labels, labels);
r->chart_labels = rrdlabels_create();
rrdlabels_migrate_to_these(r->chart_labels, labels);
r->hash = hash;
r->next = netdev_rename_root;
r->processed = 0;
@ -344,7 +343,7 @@ void netdev_rename_device_add(
r->container_device = strdupz(container_device);
r->container_name = strdupz(container_name);
update_label_list(&r->chart_labels, labels);
rrdlabels_migrate_to_these(r->chart_labels, labels);
r->processed = 0;
netdev_pending_renames++;
@ -377,7 +376,7 @@ void netdev_rename_device_del(const char *host_device) {
freez((void *) r->host_device);
freez((void *) r->container_name);
freez((void *) r->container_device);
free_label_list(r->chart_labels);
rrdlabels_destroy(r->chart_labels);
freez((void *) r);
break;
}
@ -449,7 +448,7 @@ static inline void netdev_rename_cgroup(struct netdev *d, struct netdev_rename *
snprintfz(buffer, RRD_ID_LENGTH_MAX, "net %s", r->container_device);
d->chart_family = strdupz(buffer);
update_label_list(&d->chart_labels, r->chart_labels);
rrdlabels_migrate_to_these(d->chart_labels, r->chart_labels);
d->priority = NETDATA_CHART_PRIO_CGROUP_NET_IFACE;
d->flipped = 1;
@ -542,6 +541,7 @@ static struct netdev *get_netdev(const char *name) {
d->name = strdupz(name);
d->hash = simple_hash(d->name);
d->len = strlen(d->name);
d->chart_labels = rrdlabels_create();
d->chart_type_net_bytes = strdupz("net");
d->chart_type_net_compressed = strdupz("net_compressed");
@ -881,7 +881,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
, RRDSET_TYPE_AREA
);
rrdset_update_labels(d->st_bandwidth, d->chart_labels);
rrdset_update_rrdlabels(d->st_bandwidth, d->chart_labels);
d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
@ -947,7 +947,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_speed, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_speed, d->chart_labels);
rrdset_update_rrdlabels(d->st_speed, d->chart_labels);
d->rd_speed = rrddim_add(d->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
@ -982,7 +982,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_duplex, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_duplex, d->chart_labels);
rrdset_update_rrdlabels(d->st_duplex, d->chart_labels);
d->rd_duplex = rrddim_add(d->st_duplex, "duplex", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
@ -1013,7 +1013,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_operstate, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_operstate, d->chart_labels);
rrdset_update_rrdlabels(d->st_operstate, d->chart_labels);
d->rd_operstate = rrddim_add(d->st_operstate, "state", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
@ -1044,7 +1044,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_carrier, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_carrier, d->chart_labels);
rrdset_update_rrdlabels(d->st_carrier, d->chart_labels);
d->rd_carrier = rrddim_add(d->st_carrier, "carrier", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
@ -1075,7 +1075,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_mtu, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_mtu, d->chart_labels);
rrdset_update_rrdlabels(d->st_mtu, d->chart_labels);
d->rd_mtu = rrddim_add(d->st_mtu, "mtu", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
}
@ -1111,7 +1111,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_packets, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_packets, d->chart_labels);
rrdset_update_rrdlabels(d->st_packets, d->chart_labels);
d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_tpackets = rrddim_add(d->st_packets, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -1159,7 +1159,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_errors, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_errors, d->chart_labels);
rrdset_update_rrdlabels(d->st_errors, d->chart_labels);
d->rd_rerrors = rrddim_add(d->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_terrors = rrddim_add(d->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -1205,7 +1205,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_drops, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_drops, d->chart_labels);
rrdset_update_rrdlabels(d->st_drops, d->chart_labels);
d->rd_rdrops = rrddim_add(d->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_tdrops = rrddim_add(d->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -1251,7 +1251,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_fifo, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_fifo, d->chart_labels);
rrdset_update_rrdlabels(d->st_fifo, d->chart_labels);
d->rd_rfifo = rrddim_add(d->st_fifo, "receive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_tfifo = rrddim_add(d->st_fifo, "transmit", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -1297,7 +1297,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_compressed, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_compressed, d->chart_labels);
rrdset_update_rrdlabels(d->st_compressed, d->chart_labels);
d->rd_rcompressed = rrddim_add(d->st_compressed, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_tcompressed = rrddim_add(d->st_compressed, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@ -1343,7 +1343,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
rrdset_flag_set(d->st_events, RRDSET_FLAG_DETAIL);
rrdset_update_labels(d->st_events, d->chart_labels);
rrdset_update_rrdlabels(d->st_events, d->chart_labels);
d->rd_rframe = rrddim_add(d->st_events, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
d->rd_tcollisions = rrddim_add(d->st_events, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);

12
daemon/commands.c
View file

@ -217,17 +217,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
reload_host_labels();
BUFFER *wb = buffer_create(10);
rrdhost_rdlock(localhost);
netdata_rwlock_rdlock(&localhost->labels.labels_rwlock);
struct label *l = localhost->labels.head;
while (l != NULL) {
buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value);
l = l->next;
}
netdata_rwlock_unlock(&localhost->labels.labels_rwlock);
rrdhost_unlock(localhost);
rrdlabels_log_to_buffer(localhost->host_labels, wb);
(*message)=strdupz(buffer_tostring(wb));
buffer_free(wb);

17
daemon/get-kubernetes-labels.sh.in
View file

@ -1,4 +1,5 @@
#!/usr/bin/env bash
me="$(basename "${0}")"
# Checks if netdata is running in a kubernetes pod and fetches:
# - pod's labels
@ -8,8 +9,8 @@ if [ -z "${KUBERNETES_SERVICE_HOST}" ] || [ -z "${KUBERNETES_PORT_443_TCP_PORT}"
exit 0
fi
if ! command -v jq > /dev/null 2>&1; then
echo "jq command not available. Please install jq to get host labels for kubernetes pods."
if ! command -v jq >/dev/null 2>&1; then
echo >&2 "${me}: jq command not available. Please install jq to get host labels for kubernetes pods."
exit 1
fi
@ -18,24 +19,24 @@ HEADER="Authorization: Bearer $TOKEN"
HOST="$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT"
URL="https://$HOST/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME"
if ! POD_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then
echo "error on curl '${URL}': ${POD_DATA}."
if ! POD_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then
echo >&2 "${me}: error on curl '${URL}': ${POD_DATA}."
exit 1
fi
URL="https://$HOST/api/v1/namespaces/kube-system"
if ! KUBE_SYSTEM_NS_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then
echo "error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}."
if ! KUBE_SYSTEM_NS_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then
echo >&2 "${me}: error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}."
exit 1
fi
if ! POD_LABELS=$(jq -r '.metadata.labels' <<< "$POD_DATA" | grep ':' | tr -d '," ' 2>&1); then
echo "error on 'jq' parse pod data: ${POD_LABELS}."
echo >&2 "${me}: error on 'jq' parse pod data: ${POD_LABELS}."
exit 1
fi
if ! KUBE_SYSTEM_NS_UID=$(jq -r '.metadata.uid' <<< "$KUBE_SYSTEM_NS_DATA" 2>&1); then
echo "error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}."
echo >&2 "${me}: error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}."
exit 1
fi

3
daemon/main.c
View file

@ -898,6 +898,9 @@ int main(int argc, char **argv) {
else if(strcmp(optarg, "dicttest") == 0) {
return dictionary_unittest(10000);
}
else if(strcmp(optarg, "rrdlabelstest") == 0) {
return rrdlabels_unittest();
}
else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
optarg += strlen(createdataset_string);
unsigned history_seconds = strtoul(optarg, NULL, 0);

109
database/rrd.h
View file

@ -3,6 +3,10 @@
#ifndef NETDATA_RRD_H
#define NETDATA_RRD_H 1
#ifdef __cplusplus
extern "C" {
#endif
// forward typedefs
typedef struct rrdhost RRDHOST;
typedef struct rrddim RRDDIM;
@ -22,7 +26,6 @@ typedef void *ml_dimension_t;
struct rrddim_volatile;
struct rrdset_volatile;
struct context_param;
struct label;
#ifdef ENABLE_DBENGINE
struct rrdeng_page_descr;
struct rrdengine_instance;
@ -177,66 +180,45 @@ typedef enum rrddim_flags {
#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
typedef enum label_source {
LABEL_SOURCE_AUTO = 0,
LABEL_SOURCE_NETDATA_CONF = 1,
LABEL_SOURCE_DOCKER = 2,
LABEL_SOURCE_ENVIRONMENT = 3,
LABEL_SOURCE_KUBERNETES = 4
} LABEL_SOURCE;
typedef enum rrdlabel_source {
RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
#define LABEL_FLAG_UPDATE_STREAM 1
#define LABEL_FLAG_STOP_STREAM 2
// more sources can be added here
struct label {
char *key, *value;
uint32_t key_hash;
LABEL_SOURCE label_source;
struct label *next;
};
RRDLABEL_FLAG_OLD = (1 << 30), // marks set for rrdlabels internal use - they are not exposed outside rrdlabels
RRDLABEL_FLAG_NEW = (1 << 31) //
} RRDLABEL_SRC;
struct label_index {
struct label *head; // Label list
netdata_rwlock_t labels_rwlock; // lock for the label list
uint32_t labels_flag; // Flags for labels
};
extern DICTIONARY *rrdlabels_create(void);
extern void rrdlabels_destroy(DICTIONARY *labels_dict);
extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
extern void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
extern void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
typedef enum strip_quotes {
DO_NOT_STRIP_QUOTES,
STRIP_QUOTES
} STRIP_QUOTES_OPTION;
extern void rrdlabels_unmark_all(DICTIONARY *labels);
extern void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
typedef enum skip_escaped_characters {
DO_NOT_SKIP_ESCAPED_CHARACTERS,
SKIP_ESCAPED_CHARACTERS
} SKIP_ESCAPED_CHARACTERS_OPTION;
extern int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
extern int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
extern void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
extern bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
extern bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal);
extern void rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
extern void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
char *translate_label_source(LABEL_SOURCE l);
struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
extern struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
extern void update_label_list(struct label **labels, struct label *new_labels);
extern void replace_label_list(struct label_index *labels, struct label *new_labels);
extern int is_valid_label_value(char *value);
extern int is_valid_label_key(char *key);
extern void free_label_list(struct label *labels);
extern struct label *label_list_lookup_key(struct label *head, char *key, uint32_t key_hash);
extern struct label *label_list_lookup_keylist(struct label *head, char *keylist);
extern int label_list_contains_keylist(struct label *head, char *keylist);
extern int label_list_contains_key(struct label *head, char *key, uint32_t key_hash);
extern int label_list_contains(struct label *head, struct label *check);
extern struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri);
extern void strip_last_symbol(
char *str,
char symbol,
SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
extern char *strip_double_quotes(char *str, SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
void reload_host_labels(void);
extern void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source);
extern void rrdset_finalize_labels(RRDSET *st);
extern void rrdset_update_labels(RRDSET *st, struct label *labels);
extern int rrdset_contains_label_keylist(RRDSET *st, char *key);
extern int rrdset_matches_label_keys(RRDSET *st, char *key, char *words[], uint32_t *hash_key_list, int *word_count, int size);
extern struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash);
extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
extern int rrdlabels_unittest(void);
// unfortunately this break when defined in exporting_engine.h
extern bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
// ----------------------------------------------------------------------------
// RRD DIMENSION - this is a metric
@ -402,8 +384,7 @@ struct rrdset_volatile {
char *old_units;
char *old_context;
uuid_t hash_id;
struct label *new_labels;
struct label_index labels;
DICTIONARY *chart_labels;
bool is_ar_chart;
};
@ -593,11 +574,14 @@ typedef enum rrdhost_flags {
RRDHOST_FLAG_ORPHAN = 1 << 0, // this host is orphan (not receiving data)
RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts
RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan
RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases
RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases
RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
RRDHOST_FLAG_STREAM_LABELS_UPDATE = 1 << 8,
RRDHOST_FLAG_STREAM_LABELS_STOP = 1 << 9,
} RRDHOST_FLAGS;
#define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@ -859,7 +843,7 @@ struct rrdhost {
// ------------------------------------------------------------------------
// Support for host-level labels
struct label_index labels;
DICTIONARY *host_labels;
// ------------------------------------------------------------------------
// indexes
@ -1349,4 +1333,9 @@ extern void set_host_properties(
#include "sqlite/sqlite_aclk_alert.h"
#include "sqlite/sqlite_aclk_node.h"
#include "sqlite/sqlite_health.h"
#ifdef __cplusplus
}
#endif
#endif /* NETDATA_RRD_H */

91
database/rrdcalc.c
View file

@ -109,51 +109,23 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
health_alarm_log(host, ae);
}
static inline int rrdcalc_test_additional_restriction(RRDCALC *rc, RRDSET *st){
if (rc->module_match && !simple_pattern_matches(rc->module_pattern, st->module_name))
static int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) {
if((rc->hash_chart != st->hash || strcmp(rc->chart, st->id) != 0) &&
(rc->hash_chart != st->hash_name || strcmp(rc->chart, st->name) != 0))
return 0;
if (rc->plugin_match && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name))
if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, st->module_name))
return 0;
if (rc->labels) {
int labels_count=1;
int labels_match=0;
char *s = rc->labels;
while (*s) {
if (*s==' ')
labels_count++;
s++;
}
RRDHOST *host = st->rrdhost;
char cmp[CONFIG_FILE_LINE_MAX+1];
struct label *move = host->labels.head;
while(move) {
snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
if (simple_pattern_matches(rc->splabels, move->key) ||
simple_pattern_matches(rc->splabels, cmp)) {
labels_match++;
}
move = move->next;
}
if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name))
return 0;
if (labels_match != labels_count)
return 0;
}
if (st->rrdhost->host_labels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->host_labels, rc->host_labels_pattern, '='))
return 0;
return 1;
}
static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
if(((rc->hash_chart == st->hash && !strcmp(rc->chart, st->id)) ||
(rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name))) &&
rrdcalc_test_additional_restriction(rc, st)) {
return 1;
}
return 0;
}
// this has to be called while the RRDHOST is locked
inline void rrdsetcalc_link_matching(RRDSET *st) {
RRDHOST *host = st->rrdhost;
@ -164,7 +136,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) {
if(unlikely(rc->rrdset))
continue;
if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
if(unlikely(rrdcalc_is_matching_rrdset(rc, st)))
rrdsetcalc_link(st, rc);
}
}
@ -382,7 +354,7 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) {
// link it to its chart
RRDSET *st;
rrdset_foreach_read(st, host) {
if(rrdcalc_is_matching_this_rrdset(rc, st)) {
if(rrdcalc_is_matching_rrdset(rc, st)) {
rrdsetcalc_link(st, rc);
break;
}
@ -612,8 +584,8 @@ void rrdcalc_free(RRDCALC *rc) {
freez(rc->component);
freez(rc->type);
simple_pattern_free(rc->spdim);
freez(rc->labels);
simple_pattern_free(rc->splabels);
freez(rc->host_labels);
simple_pattern_free(rc->host_labels_pattern);
freez(rc->module_match);
simple_pattern_free(rc->module_pattern);
freez(rc->plugin_match);
@ -681,51 +653,26 @@ void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
}
static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) {
RRDCALC *rc = alarms;
while (rc) {
if (!rc->labels) {
rc = rc->next;
continue;
}
for(RRDCALC *rc = alarms ; rc ; rc = rc->next ) {
if (!rc->host_labels) continue;
char cmp[CONFIG_FILE_LINE_MAX+1];
struct label *move = host->labels.head;
while(move) {
snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
if (simple_pattern_matches(rc->splabels, move->key) ||
simple_pattern_matches(rc->splabels, cmp)) {
break;
}
move = move->next;
}
RRDCALC *next = rc->next;
if(!move) {
if(!rrdlabels_match_simple_pattern_parsed(host->host_labels, rc->host_labels_pattern, '=')) {
info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
rc->name,
host->hostname,
rc->labels);
rc->host_labels);
if(host->alarms == alarms) {
if(host->alarms == alarms)
rrdcalc_unlink_and_free(host, rc);
} else
else
rrdcalc_foreach_unlink_and_free(host, rc);
}
rc = next;
}
}
void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) {
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
rrdcalc_labels_unlink_alarm_loop(host, host->alarms);
rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach);
netdata_rwlock_unlock(&host->labels.labels_rwlock);
}
void rrdcalc_labels_unlink() {
@ -736,7 +683,7 @@ void rrdcalc_labels_unlink() {
if (unlikely(!host->health_enabled))
continue;
if (host->labels.head) {
if (host->host_labels) {
rrdhost_wrlock(host);
rrdcalc_labels_unlink_alarm_from_host(host);

4
database/rrdcalc.h
View file

@ -103,8 +103,8 @@ struct rrdcalc {
// ------------------------------------------------------------------------
// Labels settings
char *labels; // the label read from an alarm file
SIMPLE_PATTERN *splabels; // the simple pattern of labels
char *host_labels; // the label read from an alarm file
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
// ------------------------------------------------------------------------
// runtime information

108
database/rrdcalctemplate.c
View file

@ -4,62 +4,6 @@
#include "rrd.h"
// ----------------------------------------------------------------------------
static int rrdcalctemplate_is_there_label_restriction(RRDCALCTEMPLATE *rt, RRDHOST *host) {
if(!rt->labels)
return 0;
errno = 0;
struct label *move = host->labels.head;
char cmp[CONFIG_FILE_LINE_MAX+1];
int ret;
if(move) {
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
while(move) {
snprintfz(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
if (simple_pattern_matches(rt->splabels, move->key) ||
simple_pattern_matches(rt->splabels, cmp)) {
break;
}
move = move->next;
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
if(!move) {
error("Health template '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
rt->name,
host->hostname,
rt->labels
);
ret = 1;
} else {
ret = 0;
}
} else {
ret =0;
}
return ret;
}
static inline int rrdcalctemplate_test_additional_restriction(RRDCALCTEMPLATE *rt, RRDSET *st) {
if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, st->name))
return 0;
if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, st->family))
return 0;
if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, st->module_name))
return 0;
if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, st->plugin_name))
return 0;
return 1;
}
// RRDCALCTEMPLATE management
/**
* RRDCALC TEMPLATE LINK MATCHING
@ -67,35 +11,43 @@ static inline int rrdcalctemplate_test_additional_restriction(RRDCALCTEMPLATE *r
* @param rt is the template used to create the chart.
* @param st is the chart where the alarm will be attached.
*/
void rrdcalctemplate_link_matching_test(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) {
if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context) &&
rrdcalctemplate_test_additional_restriction(rt, st) ) {
if (!rrdcalctemplate_is_there_label_restriction(rt, host)) {
RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
if (unlikely(!rc))
info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed",
rt->name, st->id, host->hostname);
void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) {
if(rt->hash_context != st->hash_context || strcmp(rt->context, st->context) != 0)
return;
if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, st->name))
return;
if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, st->family))
return;
if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, st->module_name))
return;
if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, st->plugin_name))
return;
if(host->host_labels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->host_labels, rt->host_labels_pattern, '='))
return;
RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
if (unlikely(!rc))
info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname);
#ifdef NETDATA_INTERNAL_CHECKS
else if (rc->rrdset != st &&
!rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
error("Health alarm '%s.%s' should be linked to chart '%s', but it is not",
rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
#endif
}
}
}
void rrdcalctemplate_link_matching(RRDSET *st) {
RRDHOST *host = st->rrdhost;
RRDCALCTEMPLATE *rt;
for(rt = host->templates; rt ; rt = rt->next) {
rrdcalctemplate_link_matching_test(rt, st, host);
}
for(rt = host->templates; rt ; rt = rt->next)
rrdcalctemplate_check_conditions_and_link(rt, st, host);
for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next) {
rrdcalctemplate_link_matching_test(rt, st, host);
}
for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next)
rrdcalctemplate_check_conditions_and_link(rt, st, host);
}
inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
@ -129,9 +81,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
freez(rt->info);
freez(rt->dimensions);
freez(rt->foreachdim);
freez(rt->labels);
freez(rt->host_labels);
simple_pattern_free(rt->spdim);
simple_pattern_free(rt->splabels);
simple_pattern_free(rt->host_labels_pattern);
freez(rt);
}

4
database/rrdcalctemplate.h
View file

@ -74,8 +74,8 @@ struct rrdcalctemplate {
// ------------------------------------------------------------------------
// Labels settings
char *labels; // the label read from an alarm file
SIMPLE_PATTERN *splabels; // the simple pattern of labels
char *host_labels; // the label read from an alarm file
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
// ------------------------------------------------------------------------
// expressions related to the alarm

282
database/rrdhost.c
View file

@ -199,7 +199,7 @@ RRDHOST *rrdhost_create(const char *hostname,
#endif
netdata_rwlock_init(&host->rrdhost_rwlock);
netdata_rwlock_init(&host->labels.labels_rwlock);
host->host_labels = rrdlabels_create();
netdata_mutex_init(&host->aclk_state_lock);
@ -976,7 +976,7 @@ void rrdhost_free(RRDHOST *host) {
freez(host->aclk_state.claimed_id);
freez(host->aclk_state.prev_claimed_id);
freez((void *)host->tags);
free_label_list(host->labels.head);
rrdlabels_destroy(host->host_labels);
freez((void *)host->os);
freez((void *)host->timezone);
freez((void *)host->abbrev_timezone);
@ -994,7 +994,6 @@ void rrdhost_free(RRDHOST *host) {
freez(host->registry_hostname);
simple_pattern_free(host->rrdpush_send_charts_matching);
rrdhost_unlock(host);
netdata_rwlock_destroy(&host->labels.labels_rwlock);
netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
netdata_rwlock_destroy(&host->rrdhost_rwlock);
freez(host->node_id);
@ -1038,297 +1037,138 @@ void rrdhost_save_charts(RRDHOST *host) {
rrdhost_unlock(host);
}
static struct label *rrdhost_load_auto_labels(void)
{
struct label *label_list = NULL;
static void rrdhost_load_auto_labels(void) {
DICTIONARY *labels = localhost->host_labels;
if (localhost->system_info->cloud_provider_type)
label_list =
add_label_to_list(label_list, "_cloud_provider_type", localhost->system_info->cloud_provider_type, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
if (localhost->system_info->cloud_instance_type)
label_list =
add_label_to_list(label_list, "_cloud_instance_type", localhost->system_info->cloud_instance_type, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
if (localhost->system_info->cloud_instance_region)
label_list =
add_label_to_list(label_list, "_cloud_instance_region", localhost->system_info->cloud_instance_region, LABEL_SOURCE_AUTO);
rrdlabels_add(
labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_os_name)
label_list =
add_label_to_list(label_list, "_os_name", localhost->system_info->host_os_name, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_os_version)
label_list =
add_label_to_list(label_list, "_os_version", localhost->system_info->host_os_version, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_os_version", localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
if (localhost->system_info->kernel_version)
label_list =
add_label_to_list(label_list, "_kernel_version", localhost->system_info->kernel_version, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_kernel_version", localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_cores)
label_list =
add_label_to_list(label_list, "_system_cores", localhost->system_info->host_cores, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_system_cores", localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_cpu_freq)
label_list =
add_label_to_list(label_list, "_system_cpu_freq", localhost->system_info->host_cpu_freq, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_system_cpu_freq", localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_ram_total)
label_list =
add_label_to_list(label_list, "_system_ram_total", localhost->system_info->host_ram_total, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_system_ram_total", localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_disk_space)
label_list =
add_label_to_list(label_list, "_system_disk_space", localhost->system_info->host_disk_space, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_system_disk_space", localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
if (localhost->system_info->architecture)
label_list =
add_label_to_list(label_list, "_architecture", localhost->system_info->architecture, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_architecture", localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
if (localhost->system_info->virtualization)
label_list =
add_label_to_list(label_list, "_virtualization", localhost->system_info->virtualization, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_virtualization", localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
if (localhost->system_info->container)
label_list =
add_label_to_list(label_list, "_container", localhost->system_info->container, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_container", localhost->system_info->container, RRDLABEL_SRC_AUTO);
if (localhost->system_info->container_detection)
label_list =
add_label_to_list(label_list, "_container_detection", localhost->system_info->container_detection, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_container_detection", localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
if (localhost->system_info->virt_detection)
label_list =
add_label_to_list(label_list, "_virt_detection", localhost->system_info->virt_detection, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_virt_detection", localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
if (localhost->system_info->is_k8s_node)
label_list =
add_label_to_list(label_list, "_is_k8s_node", localhost->system_info->is_k8s_node, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_is_k8s_node", localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
if (localhost->system_info->install_type)
label_list =
add_label_to_list(label_list, "_install_type", localhost->system_info->install_type, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_install_type", localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
if (localhost->system_info->prebuilt_arch)
label_list =
add_label_to_list(label_list, "_prebuilt_arch", localhost->system_info->prebuilt_arch, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_prebuilt_arch", localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
if (localhost->system_info->prebuilt_dist)
label_list =
add_label_to_list(label_list, "_prebuilt_dist", localhost->system_info->prebuilt_dist, LABEL_SOURCE_AUTO);
rrdlabels_add(labels, "_prebuilt_dist", localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
label_list = add_aclk_host_labels(label_list);
add_aclk_host_labels();
label_list = add_label_to_list(
label_list, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", LABEL_SOURCE_AUTO);
rrdlabels_add(
labels, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO);
if (localhost->rrdpush_send_destination)
label_list =
add_label_to_list(label_list, "_streams_to", localhost->rrdpush_send_destination, LABEL_SOURCE_AUTO);
return label_list;
rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
}
static inline int rrdhost_is_valid_label_config_option(char *name, char *value)
{
return (is_valid_label_key(name) && is_valid_label_value(value) && strcmp(name, "from environment") &&
strcmp(name, "from kubernetes pods"));
}
static struct label *rrdhost_load_config_labels()
{
static void rrdhost_load_config_labels(void) {
int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
if(!status) {
char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
error("LABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
}
struct label *l = NULL;
struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
if(co) {
config_section_wrlock(co);
struct config_option *cv;
for(cv = co->values; cv ; cv = cv->next) {
if(rrdhost_is_valid_label_config_option(cv->name, cv->value)) {
l = add_label_to_list(l, cv->name, cv->value, LABEL_SOURCE_NETDATA_CONF);
cv->flags |= CONFIG_VALUE_USED;
} else {
error("LABELS: It was not possible to create the label '%s' because it contains invalid character(s) or values."
, cv->name);
}
rrdlabels_add(localhost->host_labels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
cv->flags |= CONFIG_VALUE_USED;
}
config_section_unlock(co);
}
return l;
}
struct label *parse_simple_tags(
struct label *label_list,
const char *tags,
char key_value_separator,
char label_separator,
STRIP_QUOTES_OPTION strip_quotes_from_key,
STRIP_QUOTES_OPTION strip_quotes_from_value,
SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters)
{
const char *end = tags;
while (*end) {
const char *start = end;
char key[CONFIG_MAX_VALUE + 1];
char value[CONFIG_MAX_VALUE + 1];
while (*end && *end != key_value_separator)
end++;
strncpyz(key, start, end - start);
if (*end)
start = ++end;
while (*end && *end != label_separator)
end++;
strncpyz(value, start, end - start);
label_list = add_label_to_list(
label_list,
strip_quotes_from_key ? strip_double_quotes(trim(key), skip_escaped_characters) : trim(key),
strip_quotes_from_value ? strip_double_quotes(trim(value), skip_escaped_characters) : trim(value),
LABEL_SOURCE_NETDATA_CONF);
if (*end)
end++;
}
return label_list;
}
struct label *parse_json_tags(struct label *label_list, const char *tags)
{
char tags_buf[CONFIG_MAX_VALUE + 1];
strncpy(tags_buf, tags, CONFIG_MAX_VALUE);
char *str = tags_buf;
switch (*str) {
case '{':
str++;
strip_last_symbol(str, '}', SKIP_ESCAPED_CHARACTERS);
label_list = parse_simple_tags(label_list, str, ':', ',', STRIP_QUOTES, STRIP_QUOTES, SKIP_ESCAPED_CHARACTERS);
break;
case '[':
str++;
strip_last_symbol(str, ']', SKIP_ESCAPED_CHARACTERS);
char *end = str + strlen(str);
size_t i = 0;
while (str < end) {
char key[CONFIG_MAX_VALUE + 1];
snprintfz(key, CONFIG_MAX_VALUE, "host_tag%zu", i);
str = strip_double_quotes(trim(str), SKIP_ESCAPED_CHARACTERS);
label_list = add_label_to_list(label_list, key, str, LABEL_SOURCE_NETDATA_CONF);
// skip to the next element in the array
str += strlen(str) + 1;
while (*str && *str != ',')
str++;
str++;
i++;
}
break;
case '"':
label_list = add_label_to_list(
label_list, "host_tag", strip_double_quotes(str, SKIP_ESCAPED_CHARACTERS), LABEL_SOURCE_NETDATA_CONF);
break;
default:
label_list = add_label_to_list(label_list, "host_tag", str, LABEL_SOURCE_NETDATA_CONF);
break;
}
return label_list;
}
static struct label *rrdhost_load_kubernetes_labels(void)
{
struct label *l=NULL;
char *label_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2));
static void rrdhost_load_kubernetes_labels(void) {
char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
if (unlikely(access(label_script, R_OK) != 0)) {
error("Kubernetes pod label fetching script %s not found.",label_script);
freez(label_script);
} else {
pid_t command_pid;
debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
FILE *fp = mypopen(label_script, &command_pid);
if(fp) {
int MAX_LINE_SIZE=300;
char buffer[MAX_LINE_SIZE + 1];
while (fgets(buffer, MAX_LINE_SIZE, fp) != NULL) {
char *name=buffer;
char *value=buffer;
while (*value && *value != ':') value++;
if (*value == ':') {
*value = '\0';
value++;
}
char *eos=value;
while (*eos && *eos != '\n') eos++;
if (*eos == '\n') *eos = '\0';
if (strlen(value)>0) {
if (is_valid_label_key(name)){
l = add_label_to_list(l, name, value, LABEL_SOURCE_KUBERNETES);
} else {
info("Ignoring invalid label name '%s'", name);
}
} else {
error("%s outputted unexpected result: '%s'", label_script, name);
}
};
// Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
// Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
int retcode=mypclose(fp, command_pid);
if (retcode) {
error("%s exited abnormally. No kubernetes labels will be added to the host.", label_script);
struct label *ll=l;
while (ll != NULL) {
info("Ignoring Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(ll->label_source), ll->key, ll->value);
ll = ll->next;
freez(l);
l=ll;
}
}
}
freez(label_script);
return;
}
return l;
debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
pid_t pid;
FILE *fp = mypopen(label_script, &pid);
if(!fp) return;
char buffer[1000 + 1];
while (fgets(buffer, 1000, fp) != NULL)
rrdlabels_add_pair(localhost->host_labels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
// Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
// Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
int rc = mypclose(fp, pid);
if(rc) error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
}
void reload_host_labels(void)
{
struct label *from_auto = rrdhost_load_auto_labels();
struct label *from_k8s = rrdhost_load_kubernetes_labels();
struct label *from_config = rrdhost_load_config_labels();
void reload_host_labels(void) {
if(!localhost->host_labels)
localhost->host_labels = rrdlabels_create();
struct label *new_labels = merge_label_lists(from_auto, from_k8s);
new_labels = merge_label_lists(new_labels, from_config);
rrdlabels_unmark_all(localhost->host_labels);
rrdhost_rdlock(localhost);
replace_label_list(&localhost->labels, new_labels);
// priority is important here
rrdhost_load_config_labels();
rrdhost_load_kubernetes_labels();
rrdhost_load_auto_labels();
rrdlabels_remove_all_unmarked(localhost->host_labels);
health_label_log_save(localhost);
rrdhost_unlock(localhost);
/* TODO-GAPS - fix this so that it looks properly at the state and version of the sender
if(localhost->rrdpush_send_enabled && localhost->rrdpush_sender_buffer){
localhost->labels.labels_flag |= LABEL_FLAG_UPDATE_STREAM;
localhost->labels.labels_flag |= RRDHOST_FLAG_STREAM_LABELS_UPDATE;
rrdpush_send_labels(localhost);
}
*/

1308
database/rrdlabels.c
View file

File diff suppressed because it is too large Load diff

105
database/rrdset.c
View file

@ -387,7 +387,6 @@ void rrdset_free(RRDSET *st) {
// free it
netdata_rwlock_destroy(&st->rrdset_rwlock);
netdata_rwlock_destroy(&st->state->labels.labels_rwlock);
// free directly allocated members
freez((void *)st->name);
@ -402,7 +401,7 @@ void rrdset_free(RRDSET *st) {
freez(st->state->old_title);
freez(st->state->old_units);
freez(st->state->old_context);
free_label_list(st->state->labels.head);
rrdlabels_destroy(st->state->chart_labels);
freez(st->state);
freez(st->chart_uuid);
@ -888,7 +887,7 @@ RRDSET *rrdset_create_custom(
avl_init_lock(&st->rrdvar_root_index, rrdvar_compare);
netdata_rwlock_init(&st->rrdset_rwlock);
netdata_rwlock_init(&st->state->labels.labels_rwlock);
st->state->chart_labels = rrdlabels_create();
if(name && *name && rrdset_set_name(st, name))
// we did set the name
@ -1957,103 +1956,3 @@ after_second_database_work:
netdata_thread_enable_cancelability();
}
void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source)
{
st->state->new_labels = add_label_to_list(st->state->new_labels, key, value, source);
}
void rrdset_finalize_labels(RRDSET *st)
{
struct label *new_labels = st->state->new_labels;
struct label_index *labels = &st->state->labels;
if (!labels->head) {
labels->head = new_labels;
} else {
replace_label_list(labels, new_labels);
}
netdata_rwlock_rdlock(&labels->labels_rwlock);
struct label *lbl = labels->head;
while (lbl) {
sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
lbl = lbl->next;
}
netdata_rwlock_unlock(&labels->labels_rwlock);
st->state->new_labels = NULL;
}
void rrdset_update_labels(RRDSET *st, struct label *labels)
{
if (!labels)
return;
update_label_list(&st->state->new_labels, labels);
rrdset_finalize_labels(st);
}
int rrdset_contains_label_keylist(RRDSET *st, char *keylist)
{
struct label_index *labels = &st->state->labels;
int ret;
if (!labels->head)
return 0;
netdata_rwlock_rdlock(&labels->labels_rwlock);
ret = label_list_contains_keylist(labels->head, keylist);
netdata_rwlock_unlock(&labels->labels_rwlock);
return ret;
}
struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash)
{
struct label_index *labels = &st->state->labels;
struct label *ret = NULL;
if (labels->head) {
netdata_rwlock_rdlock(&labels->labels_rwlock);
ret = label_list_lookup_key(labels->head, key, key_hash);
netdata_rwlock_unlock(&labels->labels_rwlock);
}
return ret;
}
static inline int k8s_space(char c) {
switch(c) {
case ':':
case ',':
return 1;
default:
return 0;
}
}
int rrdset_matches_label_keys(RRDSET *st, char *keylist, char *words[], uint32_t *hash_key_list, int *word_count, int size)
{
struct label_index *labels = &st->state->labels;
if (!labels->head)
return 0;
struct label *one_label;
if (!*word_count) {
*word_count = quoted_strings_splitter(keylist, words, size, k8s_space, NULL, NULL, 0);
for (int i = 0; i < *word_count - 1; i += 2) {
hash_key_list[i] = simple_hash(words[i]);
}
}
int ret = 1;
netdata_rwlock_rdlock(&labels->labels_rwlock);
for (int i = 0; ret && i < *word_count - 1; i += 2) {
one_label = label_list_lookup_key(labels->head, words[i], hash_key_list[i]);
ret = (one_label && !strcmp(one_label->value, words[i + 1]));
}
netdata_rwlock_unlock(&labels->labels_rwlock);
return ret;
}

12
database/sqlite/sqlite_aclk_chart.c
View file

@ -158,16 +158,8 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat
chart_payload.claim_id = claim_id;
chart_payload.id = strdupz(st->id);
struct label_index *labels = &st->state->labels;
netdata_rwlock_rdlock(&labels->labels_rwlock);
struct label *label_list = labels->head;
struct label *chart_label = NULL;
while (label_list) {
chart_label = add_label_to_list(chart_label, label_list->key, label_list->value, label_list->label_source);
label_list = label_list->next;
}
netdata_rwlock_unlock(&labels->labels_rwlock);
chart_payload.label_head = chart_label;
chart_payload.chart_labels = rrdlabels_create();
rrdlabels_copy(chart_payload.chart_labels, st->state->chart_labels);
size_t size;
char *payload = generate_chart_instance_updated(&size, &chart_payload);

5
database/sqlite/sqlite_aclk_node.c
View file

@ -75,14 +75,11 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
node_info.data.ml_info.ml_capable = host->system_info->ml_capable;
node_info.data.ml_info.ml_enabled = host->system_info->ml_enabled;
struct label_index *labels = &host->labels;
netdata_rwlock_rdlock(&labels->labels_rwlock);
node_info.data.host_labels_head = labels->head;
node_info.data.host_labels_ptr = host->host_labels;
aclk_update_node_info(&node_info);
log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, wc->host->hostname, wc->host_guid, wc->host == localhost ? "parent" : "child");
netdata_rwlock_unlock(&labels->labels_rwlock);
rrd_unlock();
freez(node_info.claim_id);
freez(host_version);

8
exporting/check_filters.c
View file

@ -2,6 +2,14 @@
#include "exporting_engine.h"
bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
(void)name;
(void)value;
struct instance *instance = (struct instance *)data;
return should_send_label(instance, ls);
}
/**
* Check if the connector instance should export the host metrics
*

2
exporting/clean_connectors.c
View file

@ -33,7 +33,7 @@ static void clean_instance_config(struct instance_config *config)
void clean_instance(struct instance *instance)
{
clean_instance_config(&instance->config);
buffer_free(instance->labels);
buffer_free(instance->labels_buffer);
uv_cond_destroy(&instance->cond_var);
// uv_mutex_destroy(&instance->mutex);

9
exporting/exporting_engine.h
View file

@ -4,7 +4,6 @@
#define NETDATA_EXPORTING_ENGINE_H 1
#include "daemon/common.h"
#include <uv.h>
#define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value)
@ -40,11 +39,11 @@ extern const char *global_exporting_prefix;
#define sending_labels_configured(instance) \
(instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))
#define should_send_label(instance, label) \
#define should_send_label(instance, label_source) \
((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS && \
label->label_source == LABEL_SOURCE_NETDATA_CONF) || \
label_source & RRDLABEL_SRC_CONFIG) || \
(instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \
label->label_source != LABEL_SOURCE_NETDATA_CONF))
label_source & RRDLABEL_SRC_AUTO))
typedef enum exporting_connector_types {
EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type
@ -205,7 +204,7 @@ struct instance {
int skip_host;
int skip_chart;
BUFFER *labels;
BUFFER *labels_buffer;
time_t after;
time_t before;

30
exporting/graphite/graphite.c
View file

@ -71,7 +71,7 @@ int init_graphite_instance(struct instance *instance)
* @param len the maximum number of characters copied.
*/
void sanitize_graphite_label_value(char *dst, char *src, size_t len)
void sanitize_graphite_label_value(char *dst, const char *src, size_t len)
{
while (*src != '\0' && len) {
if (isspace(*src) || *src == ';' || *src == '~')
@ -91,29 +91,19 @@ void sanitize_graphite_label_value(char *dst, char *src, size_t len)
* @param host a data collecting host.
* @return Always returns 0.
*/
int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
{
if (!instance->labels)
instance->labels = buffer_create(1024);
if (!instance->labels_buffer)
instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char value[CONFIG_MAX_VALUE + 1];
sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE);
if (*value) {
buffer_strcat(instance->labels, ";");
buffer_sprintf(instance->labels, "%s=%s", label->key, value);
}
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
buffer_strcat(instance->labels_buffer, ";");
rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", ";",
exporting_labels_filter_callback, instance,
NULL, sanitize_graphite_label_value);
return 0;
}
@ -151,7 +141,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
(instance->labels) ? buffer_tostring(instance->labels) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
rd->last_collected_value,
(unsigned long long)rd->last_collected_time.tv_sec);
@ -197,7 +187,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
(instance->labels) ? buffer_tostring(instance->labels) : "",
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
value,
(unsigned long long)last_t);

2
exporting/graphite/graphite.h
View file

@ -7,7 +7,7 @@
int init_graphite_instance(struct instance *instance);
void sanitize_graphite_label_value(char *dst, char *src, size_t len);
void sanitize_graphite_label_value(char *dst, const char *src, size_t len);
int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host);
int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);

34
exporting/json/json.c
View file

@ -113,34 +113,20 @@ int init_json_http_instance(struct instance *instance)
* @param host a data collecting host.
* @return Always returns 0.
*/
int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
{
if (!instance->labels)
instance->labels = buffer_create(1024);
if (!instance->labels_buffer)
instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
buffer_strcat(instance->labels, "\"labels\":{");
int count = 0;
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char value[CONFIG_MAX_VALUE * 2 + 1];
sanitize_json_string(value, label->value, CONFIG_MAX_VALUE);
if (count > 0)
buffer_strcat(instance->labels, ",");
buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
count++;
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
buffer_strcat(instance->labels, "},");
buffer_strcat(instance->labels_buffer, "\"labels\":{");
rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
exporting_labels_filter_callback, instance,
NULL, sanitize_json_string);
buffer_strcat(instance->labels_buffer, "},");
return 0;
}
@ -203,7 +189,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
tags_pre,
tags,
tags_post,
instance->labels ? buffer_tostring(instance->labels) : "",
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,
@ -288,7 +274,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
tags_pre,
tags,
tags_post,
instance->labels ? buffer_tostring(instance->labels) : "",
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,

70
exporting/opentsdb/opentsdb.c
View file

@ -124,13 +124,14 @@ int init_opentsdb_http_instance(struct instance *instance)
* @param len the maximum number of characters copied.
*/
void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len)
{
while (*src != '\0' && len) {
if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
*dst++ = *src;
else
*dst++ = '_';
src++;
len--;
}
@ -144,28 +145,18 @@ void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
* @param host a data collecting host.
* @return Always returns 0.
*/
int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host)
{
if (!instance->labels)
instance->labels = buffer_create(1024);
int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) {
if(!instance->labels_buffer)
instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char value[CONFIG_MAX_VALUE + 1];
sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE);
if (*value)
buffer_sprintf(instance->labels, " %s=%s", label->key, value);
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
buffer_strcat(instance->labels_buffer, " ");
rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ",
exporting_labels_filter_callback, instance,
NULL, sanitize_opentsdb_label_value);
return 0;
}
@ -204,7 +195,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(instance->labels) ? buffer_tostring(instance->labels) : "");
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@ -250,7 +241,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
(instance->labels) ? buffer_tostring(instance->labels) : "");
(instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@ -287,33 +278,18 @@ void opentsdb_http_prepare_header(struct instance *instance)
* @param host a data collecting host.
* @return Always returns 0.
*/
int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host)
{
if (!instance->labels)
instance->labels = buffer_create(1024);
int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) {
if (!instance->labels_buffer)
instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char escaped_value[CONFIG_MAX_VALUE * 2 + 1];
sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE);
char value[CONFIG_MAX_VALUE + 1];
sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE);
if (*value) {
buffer_strcat(instance->labels, ",");
buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
}
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
buffer_strcat(instance->labels_buffer, ",");
rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
exporting_labels_filter_callback, instance,
NULL, sanitize_opentsdb_label_value);
return 0;
}
@ -362,7 +338,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
instance->labels ? buffer_tostring(instance->labels) : "");
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@ -418,7 +394,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
instance->labels ? buffer_tostring(instance->labels) : "");
instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}

2
exporting/opentsdb/opentsdb.h
View file

@ -8,7 +8,7 @@
int init_opentsdb_telnet_instance(struct instance *instance);
int init_opentsdb_http_instance(struct instance *instance);
void sanitize_opentsdb_label_value(char *dst, char *src, size_t len);
void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len);
int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host);
int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host);

4
exporting/process_data.c
View file

@ -358,8 +358,8 @@ int flush_host_labels(struct instance *instance, RRDHOST *host)
{
(void)host;
if (instance->labels)
buffer_flush(instance->labels);
if (instance->labels_buffer)
buffer_flush(instance->labels_buffer);
return 0;
}

79
exporting/prometheus/prometheus.c
View file

@ -290,35 +290,44 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
* @param instance an instance data structure.
* @param host a data collecting host.
*/
struct format_prometheus_label_callback {
struct instance *instance;
size_t count;
};
static int format_prometheus_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
struct format_prometheus_label_callback *d = (struct format_prometheus_label_callback *)data;
if (!should_send_label(d->instance, ls)) return 0;
char k[PROMETHEUS_ELEMENT_MAX + 1];
char v[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
if (*k && *v) {
if (d->count > 0) buffer_strcat(d->instance->labels_buffer, ",");
buffer_sprintf(d->instance->labels_buffer, "%s=\"%s\"", k, v);
d->count++;
}
return 1;
}
void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
{
if (unlikely(!sending_labels_configured(instance)))
return;
if (!instance->labels)
instance->labels = buffer_create(1024);
if (!instance->labels_buffer)
instance->labels_buffer = buffer_create(1024);
int count = 0;
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char key[PROMETHEUS_ELEMENT_MAX + 1];
char value[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
if (*key && *value) {
if (count > 0)
buffer_strcat(instance->labels, ",");
buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
count++;
}
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
struct format_prometheus_label_callback tmp = {
.instance = instance,
.count = 0
};
rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp);
}
struct host_variables_callback_options {
@ -534,13 +543,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
char labels[PROMETHEUS_LABELS_MAX + 1] = "";
if (allhosts) {
if (instance->labels && buffer_tostring(instance->labels)) {
if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
buffer_sprintf(
wb,
"netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
hostname,
buffer_tostring(instance->labels),
buffer_tostring(instance->labels_buffer),
now_realtime_usec() / USEC_PER_MS);
// deprecated, exists only for compatibility with older queries
@ -548,45 +557,45 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
wb,
"netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
hostname,
buffer_tostring(instance->labels),
buffer_tostring(instance->labels_buffer),
now_realtime_usec() / USEC_PER_MS);
} else {
buffer_sprintf(
wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
// deprecated, exists only for compatibility with older queries
buffer_sprintf(
wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
}
}
snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
} else {
if (instance->labels && buffer_tostring(instance->labels)) {
if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
buffer_sprintf(
wb,
"netdata_host_tags_info{%s} 1 %llu\n",
buffer_tostring(instance->labels),
buffer_tostring(instance->labels_buffer),
now_realtime_usec() / USEC_PER_MS);
// deprecated, exists only for compatibility with older queries
buffer_sprintf(
wb,
"netdata_host_tags{%s} 1 %llu\n",
buffer_tostring(instance->labels),
buffer_tostring(instance->labels_buffer),
now_realtime_usec() / USEC_PER_MS);
} else {
buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels_buffer));
// deprecated, exists only for compatibility with older queries
buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels_buffer));
}
}
}
if (instance->labels)
buffer_flush(instance->labels);
if (instance->labels_buffer)
buffer_flush(instance->labels_buffer);
// send custom variables set for the host
if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {

40
exporting/prometheus/remote_write/remote_write.c
View file

@ -141,6 +141,26 @@ int init_prometheus_remote_write_instance(struct instance *instance)
* @param host a data collecting host.
* @return Always returns 0.
*/
struct format_remote_write_label_callback {
struct instance *instance;
void *write_request;
};
static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data;
if (!should_send_label(d->instance, ls)) return 0;
char k[PROMETHEUS_ELEMENT_MAX + 1];
char v[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
add_label(d->write_request, k, v);
return 1;
}
int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
{
struct simple_connector_data *simple_connector_data =
@ -159,21 +179,11 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
"netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
if (unlikely(sending_labels_configured(instance))) {
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if (!should_send_label(instance, label))
continue;
char key[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
char value[PROMETHEUS_ELEMENT_MAX + 1];
prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
add_label(connector_specific_data->write_request, key, value);
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
struct format_remote_write_label_callback tmp = {
.write_request = connector_specific_data->write_request,
.instance = instance
};
rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp);
}
return 0;

23
exporting/tests/exporting_fixtures.c
View file

@ -41,17 +41,9 @@ int setup_rrdhost()
localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
struct label *label = calloc(1, sizeof(struct label));
label->key = strdupz("key1");
label->value = strdupz("value1");
label->label_source = LABEL_SOURCE_NETDATA_CONF;
localhost->labels.head = label;
label = calloc(1, sizeof(struct label));
label->key = strdupz("key2");
label->value = strdupz("value2");
label->label_source = LABEL_SOURCE_AUTO;
localhost->labels.head->next = label;
localhost->host_labels = rrdlabels_create();
rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG);
rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG);
localhost->rrdset_root = calloc(1, sizeof(RRDSET));
RRDSET *st = localhost->rrdset_root;
@ -94,12 +86,7 @@ int teardown_rrdhost()
free((void *)st->name);
free(st);
free(localhost->labels.head->next->key);
free(localhost->labels.head->next->value);
free(localhost->labels.head->next);
free(localhost->labels.head->key);
free(localhost->labels.head->value);
free(localhost->labels.head);
rrdlabels_destroy(localhost->host_labels);
free((void *)localhost->tags);
free(localhost);
@ -124,7 +111,7 @@ int teardown_initialized_engine(void **state)
struct engine *engine = *state;
teardown_rrdhost();
buffer_free(engine->instance_root->labels);
buffer_free(engine->instance_root->labels_buffer);
buffer_free(engine->instance_root->buffer);
teardown_configured_engine(state);

8
exporting/tests/netdata_doubles.c
View file

@ -245,3 +245,11 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
function_called();
}
void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
{
(void)chart_uuid;
(void)source_type;
(void)label;
(void)value;
}

22
exporting/tests/test_exporting_engine.c
View file

@ -381,7 +381,7 @@ static void test_prepare_buffers(void **state)
expect_value(__mock_end_batch_formatting, instance, instance);
will_return(__mock_end_batch_formatting, 0);
assert_int_equal(__real_prepare_buffers(engine), 0);
__real_prepare_buffers(engine);
assert_int_equal(instance->stats.buffered_metrics, 1);
@ -393,7 +393,7 @@ static void test_prepare_buffers(void **state)
instance->end_chart_formatting = NULL;
instance->end_host_formatting = NULL;
instance->end_batch_formatting = NULL;
assert_int_equal(__real_prepare_buffers(engine), 0);
__real_prepare_buffers(engine);
assert_int_equal(instance->scheduled, 0);
assert_int_equal(instance->after, 2);
@ -705,7 +705,7 @@ static void test_format_host_labels_json_plaintext(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0);
assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
assert_string_equal(buffer_tostring(instance->labels_buffer), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
}
static void test_format_host_labels_graphite_plaintext(void **state)
@ -717,7 +717,7 @@ static void test_format_host_labels_graphite_plaintext(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0);
assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2");
assert_string_equal(buffer_tostring(instance->labels_buffer), ";key1=value1;key2=value2");
}
static void test_format_host_labels_opentsdb_telnet(void **state)
@ -729,7 +729,7 @@ static void test_format_host_labels_opentsdb_telnet(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0);
assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2");
assert_string_equal(buffer_tostring(instance->labels_buffer), " key1=value1 key2=value2");
}
static void test_format_host_labels_opentsdb_http(void **state)
@ -741,7 +741,7 @@ static void test_format_host_labels_opentsdb_http(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0);
assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\"");
assert_string_equal(buffer_tostring(instance->labels_buffer), ",\"key1\":\"value1\",\"key2\":\"value2\"");
}
static void test_flush_host_labels(void **state)
@ -749,12 +749,12 @@ static void test_flush_host_labels(void **state)
struct engine *engine = *state;
struct instance *instance = engine->instance_root;
instance->labels = buffer_create(12);
buffer_strcat(instance->labels, "check string");
assert_int_equal(buffer_strlen(instance->labels), 12);
instance->labels_buffer = buffer_create(12);
buffer_strcat(instance->labels_buffer, "check string");
assert_int_equal(buffer_strlen(instance->labels_buffer), 12);
assert_int_equal(flush_host_labels(instance, localhost), 0);
assert_int_equal(buffer_strlen(instance->labels), 0);
assert_int_equal(buffer_strlen(instance->labels_buffer), 0);
}
static void test_create_main_rusage_chart(void **state)
@ -1048,7 +1048,7 @@ static void test_format_host_labels_prometheus(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
format_host_labels_prometheus(instance, localhost);
assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\"");
assert_string_equal(buffer_tostring(instance->labels_buffer), "key1=\"value1\",key2=\"value2\"");
}
static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)

7
exporting/tests/test_exporting_engine.h
View file

@ -30,7 +30,14 @@
#include <stddef.h>
#include <setjmp.h>
#include <stdint.h>
#ifndef UNIT_TESTING
#include <cmocka.h>
#else
#undef UNIT_TESTING
#include <cmocka.h>
#define UNIT_TESTING
#endif
#define MAX_LOG_LINE 1024
extern char log_line[];

26
health/health_config.c
View file

@ -955,17 +955,17 @@ static int health_readfile(const char *filename, void *data) {
}
else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
alert_cfg->host_labels = strdupz(value);
if(rc->labels) {
if(strcmp(rc->labels, value) != 0)
if(rc->host_labels) {
if(strcmp(rc->host_labels, value) != 0)
error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
line, filename, rc->name, key, value, value);
freez(rc->labels);
simple_pattern_free(rc->splabels);
freez(rc->host_labels);
simple_pattern_free(rc->host_labels_pattern);
}
rc->labels = simple_pattern_trim_around_equal(value);
rc->splabels = simple_pattern_create(rc->labels, NULL, SIMPLE_PATTERN_EXACT);
rc->host_labels = simple_pattern_trim_around_equal(value);
rc->host_labels_pattern = simple_pattern_create(rc->host_labels, NULL, SIMPLE_PATTERN_EXACT);
}
else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
alert_cfg->plugin = strdupz(value);
@ -1204,17 +1204,17 @@ static int health_readfile(const char *filename, void *data) {
}
else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
alert_cfg->host_labels = strdupz(value);
if(rt->labels) {
if(strcmp(rt->labels, value) != 0)
if(rt->host_labels) {
if(strcmp(rt->host_labels, value) != 0)
error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
line, filename, rt->name, key, rt->labels, value, value);
line, filename, rt->name, key, rt->host_labels, value, value);
freez(rt->labels);
simple_pattern_free(rt->splabels);
freez(rt->host_labels);
simple_pattern_free(rt->host_labels_pattern);
}
rt->labels = simple_pattern_trim_around_equal(value);
rt->splabels = simple_pattern_create(rt->labels, NULL, SIMPLE_PATTERN_EXACT);
rt->host_labels = simple_pattern_trim_around_equal(value);
rt->host_labels_pattern = simple_pattern_create(rt->host_labels, NULL, SIMPLE_PATTERN_EXACT);
}
else {
error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",

21
health/health_log.c
View file

@ -74,28 +74,15 @@ inline void health_label_log_save(RRDHOST *host) {
if(unlikely(host->health_log_fp)) {
BUFFER *wb = buffer_create(1024);
rrdhost_check_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
struct label *l=localhost->labels.head;
while (l != NULL) {
buffer_sprintf(wb,"%s=%s\t ", l->key, l->value);
l = l->next;
}
netdata_rwlock_unlock(&host->labels.labels_rwlock);
char *write = (char *) buffer_tostring(wb) ;
rrdlabels_to_buffer(localhost->host_labels, wb, "", "=", "", "\t ", NULL, NULL, NULL, NULL);
char *write = (char *) buffer_tostring(wb);
write[wb->len-2] = '\n';
write[wb->len-1] = '\0';
if (unlikely(fprintf(host->health_log_fp, "L\t%s"
, write
) < 0))
if (unlikely(fprintf(host->health_log_fp, "L\t%s", write) < 0))
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.",
host->hostname, host->health_log_filename);
else {
else
host->health_log_entries_written++;
}
buffer_free(wb);
}

12
libnetdata/config/appconfig.c
View file

@ -805,6 +805,18 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
struct section *co;
struct config_option *cv;
{
int found_host_labels = 0;
for (co = root->first_section; co; co = co->next)
if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL))
found_host_labels = 1;
if(!found_host_labels) {
appconfig_section_create(root, CONFIG_SECTION_HOST_LABEL);
appconfig_get(root, CONFIG_SECTION_HOST_LABEL, "name", "value");
}
}
buffer_strcat(wb,
"# netdata configuration\n"
"#\n"

321
libnetdata/dictionary/dictionary.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// NOT TO BE USED BY USERS YET
#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 6) // maintain reference counter in walkthrough and foreach
#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 5) // maintain reference counter in walkthrough and foreach
typedef struct dictionary DICTIONARY;
#define DICTIONARY_INTERNALS
@ -79,10 +79,13 @@ typedef struct name_value {
char *name; // the name of the dictionary item
void *value; // the value of the dictionary item
size_t name_len; // the size of the name, including the terminating zero
size_t value_len; // the size of the value (assumed binary)
} NAME_VALUE;
/*
* When DICTIONARY_FLAG_WITH_STATISTICS is set, we need to keep track of all the memory
* When DICTIONARY_FLAG_REFERENCE_COUNTERS is set, we need to keep track of all the memory
* we allocate and free. So, we need to keep track of the sizes of all names and values.
* We do this by overloading NAME_VALUE with the following additional fields.
*/
@ -92,24 +95,12 @@ typedef enum name_value_flags {
NAME_VALUE_FLAG_DELETED = (1 << 0), // this item is deleted
} NAME_VALUE_FLAGS;
typedef struct name_value_with_stats {
typedef struct name_value_with_reference_counters {
NAME_VALUE name_value_data_here; // never used - just to put the lengths at the right position
size_t name_len; // the size of the name, including the terminating zero
size_t value_len; // the size of the value (assumed binary)
size_t refcount; // the reference counter
NAME_VALUE_FLAGS flags; // the flags for this item
} NAME_VALUE_WITH_STATS;
struct dictionary_stats {
size_t inserts;
size_t deletes;
size_t searches;
size_t resets;
size_t entries;
size_t memory;
};
} NAME_VALUE_WITH_REFERENCE_COUNTERS;
struct dictionary {
DICTIONARY_FLAGS flags; // the flags of the dictionary
@ -137,7 +128,13 @@ struct dictionary {
void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data);
void *conflict_callback_data;
struct dictionary_stats *stats; // the statistics when DICTIONARY_FLAG_WITH_STATISTICS is set
size_t inserts;
size_t deletes;
size_t searches;
size_t resets;
size_t entries;
size_t walkthroughs;
size_t memory;
};
void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data) {
@ -159,60 +156,49 @@ void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_cal
// dictionary statistics maintenance
size_t dictionary_stats_allocated_memory(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->memory;
return 0;
return dict->memory;
}
size_t dictionary_stats_entries(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->entries;
return 0;
return dict->entries;
}
size_t dictionary_stats_searches(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->searches;
return 0;
return dict->searches;
}
size_t dictionary_stats_inserts(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->inserts;
return 0;
return dict->inserts;
}
size_t dictionary_stats_deletes(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->deletes;
return 0;
return dict->deletes;
}
size_t dictionary_stats_resets(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
return dict->stats->resets;
return 0;
return dict->resets;
}
size_t dictionary_stats_walkthroughs(DICTIONARY *dict) {
return dict->walkthroughs;
}
static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
dict->stats->searches++;
__atomic_fetch_add(&dict->searches, 1, __ATOMIC_SEQ_CST);
}
static inline void DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict, size_t size) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
dict->stats->inserts++;
dict->stats->entries++;
dict->stats->memory += size;
}
__atomic_fetch_add(&dict->inserts, 1, __ATOMIC_SEQ_CST);
__atomic_fetch_add(&dict->entries, 1, __ATOMIC_SEQ_CST);
__atomic_fetch_add(&dict->memory, size, __ATOMIC_SEQ_CST);
}
static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict, size_t size) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
dict->stats->deletes++;
dict->stats->entries--;
dict->stats->memory -= size;
}
__atomic_fetch_add(&dict->deletes, 1, __ATOMIC_SEQ_CST);
__atomic_fetch_sub(&dict->entries, 1, __ATOMIC_SEQ_CST);
__atomic_fetch_sub(&dict->memory, size, __ATOMIC_SEQ_CST);
}
static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t oldsize, size_t newsize) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
dict->stats->resets++;
dict->stats->memory += newsize;
dict->stats->memory -= oldsize;
}
__atomic_fetch_add(&dict->resets, 1, __ATOMIC_SEQ_CST);
__atomic_fetch_add(&dict->memory, newsize, __ATOMIC_SEQ_CST);
__atomic_fetch_sub(&dict->memory, oldsize, __ATOMIC_SEQ_CST);
}
static inline void DICTIONARY_STATS_WALKTHROUGHS_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->walkthroughs, 1, __ATOMIC_SEQ_CST);
}
// ----------------------------------------------------------------------------
@ -279,21 +265,21 @@ static inline size_t reference_counter_free(DICTIONARY *dict) {
static void reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) {
if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
__atomic_fetch_add(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
}
}
static void reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv) {
if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
__atomic_fetch_sub(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
}
}
static int reference_counter_mark_deleted(DICTIONARY *dict, NAME_VALUE *nv) {
if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
nvs->flags |= NAME_VALUE_FLAG_DELETED;
return 1;
}
@ -491,35 +477,7 @@ static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VAL
// NAME_VALUE methods
static inline size_t namevalue_alloc_size(DICTIONARY *dict) {
return (dict->flags & DICTIONARY_FLAG_WITH_STATISTICS) ? sizeof(NAME_VALUE_WITH_STATS) : sizeof(NAME_VALUE);
}
static inline size_t namevalue_get_namelen(DICTIONARY *dict, NAME_VALUE *nv) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
return nvs->name_len;
}
return 0;
}
static inline size_t namevalue_get_valuelen(DICTIONARY *dict, NAME_VALUE *nv) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
return nvs->value_len;
}
return 0;
}
static inline void namevalue_set_valuelen(DICTIONARY *dict, NAME_VALUE *nv, size_t value_len) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
nvs->value_len = value_len;
}
}
static inline void namevalue_set_namevaluelen(DICTIONARY *dict, NAME_VALUE *nv, size_t name_len, size_t value_len) {
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
nvs->name_len = name_len;
nvs->value_len = value_len;
}
return (dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) ? sizeof(NAME_VALUE_WITH_REFERENCE_COUNTERS) : sizeof(NAME_VALUE);
}
static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len) {
@ -529,7 +487,8 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s
NAME_VALUE *nv = mallocz(size);
size_t allocated = size;
namevalue_set_namevaluelen(dict, nv, name_len, value_len);
nv->name_len = name_len;
nv->value_len = value_len;
if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))
nv->name = (char *)name;
@ -565,47 +524,63 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s
DICTIONARY_STATS_ENTRIES_PLUS1(dict, allocated);
if(dict->ins_callback)
dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
return nv;
}
static void namevalue_reset_unsafe(DICTIONARY *dict, NAME_VALUE *nv, void *value, size_t value_len) {
debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", nv->name);
if(dict->del_callback)
dict->del_callback(nv->name, nv->value, dict->del_callback_data);
if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
debug(D_DICTIONARY, "Dictionary: linking value to '%s'", nv->name);
nv->value = value;
namevalue_set_valuelen(dict, nv, value_len);
nv->value_len = value_len;
}
else {
debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", nv->name);
DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, namevalue_get_valuelen(dict, nv), value_len);
DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, nv->value_len, value_len);
void *old = nv->value;
void *new = mallocz(value_len);
memcpy(new, value, value_len);
nv->value = new;
namevalue_set_valuelen(dict, nv, value_len);
void *oldvalue = nv->value;
void *newvalue = NULL;
if(value_len) {
newvalue = mallocz(value_len);
if(value) memcpy(newvalue, value, value_len);
else memset(newvalue, 0, value_len);
}
nv->value = newvalue;
nv->value_len = value_len;
debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", nv->name);
freez(old);
freez(oldvalue);
}
if(dict->ins_callback)
dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
}
static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv) {
debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
if(dict->del_callback)
dict->del_callback(nv->name, nv->value, dict->del_callback_data);
size_t freed = 0;
if(unlikely(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))) {
debug(D_DICTIONARY, "Dictionary freeing value of '%s'", nv->name);
freez(nv->value);
freed += namevalue_get_valuelen(dict, nv);
freed += nv->value_len;
}
if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) {
debug(D_DICTIONARY, "Dictionary freeing name '%s'", nv->name);
freez(nv->name);
freed += namevalue_get_namelen(dict, nv);
freed += nv->name_len;
}
freez(nv);
@ -627,11 +602,6 @@ DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) {
flags &= ~DICTIONARY_FLAG_REFERENCE_COUNTERS;
}
if(flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) {
// we need statistics to allocate the extra NAME_VALUE attributes
flags |= DICTIONARY_FLAG_WITH_STATISTICS;
}
DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
size_t allocated = sizeof(DICTIONARY);
@ -640,20 +610,15 @@ DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) {
allocated += dictionary_lock_init(dict);
allocated += reference_counter_init(dict);
if(flags & DICTIONARY_FLAG_WITH_STATISTICS) {
dict->stats = callocz(1, sizeof(struct dictionary_stats));
allocated += sizeof(struct dictionary_stats);
dict->stats->memory = allocated;
}
else
dict->stats = NULL;
dict->memory = allocated;
hashtable_init_unsafe(dict);
return (DICTIONARY *)dict;
}
size_t dictionary_destroy(DICTIONARY *dict) {
if(!dict) return 0;
debug(D_DICTIONARY, "Destroying dictionary.");
dictionary_lock_wrlock(dict);
@ -680,12 +645,6 @@ size_t dictionary_destroy(DICTIONARY *dict) {
freed += dictionary_lock_free(dict);
freed += reference_counter_free(dict);
if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
freez(dict->stats);
dict->stats = NULL;
freed += sizeof(struct dictionary_stats);
}
freez(dict);
freed += sizeof(DICTIONARY);
@ -722,9 +681,6 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz
nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len);
hashtable_inserted_name_value_unsafe(dict, name, name_len, nv);
linkedlist_namevalue_link_unsafe(dict, nv);
if(dict->ins_callback)
dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
}
else {
// the item is already in the index
@ -732,16 +688,9 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz
// or overwrite the value, depending on dictionary flags
nv = *pnv;
if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) {
if(dict->del_callback)
dict->del_callback(nv->name, nv->value, dict->del_callback_data);
if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE))
namevalue_reset_unsafe(dict, nv, value, value_len);
if(dict->ins_callback)
dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
}
else if(dict->conflict_callback)
dict->conflict_callback(nv->name, nv->value, value, dict->conflict_callback_data);
}
@ -811,10 +760,6 @@ int dictionary_del_unsafe(DICTIONARY *dict, const char *name) {
if(!reference_counter_mark_deleted(dict, nv)) {
linkedlist_namevalue_unlink_unsafe(dict, nv);
if(dict->del_callback)
dict->del_callback(nv->name, nv->value, dict->del_callback_data);
namevalue_destroy_unsafe(dict, nv);
}
ret = 0;
@ -835,6 +780,8 @@ int dictionary_del(DICTIONARY *dict, const char *name) {
void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
if(unlikely(!dfe || !dict)) return NULL;
DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
dfe->dict = dict;
dfe->started_ut = now_realtime_usec();
@ -912,6 +859,10 @@ usec_t dictionary_foreach_done(DICTFE *dfe) {
// do not use other dictionary calls while walking the dictionary - deadlock!
int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) {
if(unlikely(!dict)) return 0;
DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
if(rw == 'r' || rw == 'R')
dictionary_lock_rlock(dict);
else
@ -942,6 +893,54 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c
return ret;
}
// ----------------------------------------------------------------------------
// sort
static int dictionary_sort_compar(const void *nv1, const void *nv2) {
return strcmp((*(NAME_VALUE **)nv1)->name, (*(NAME_VALUE **)nv2)->name);
}
int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) {
if(unlikely(!dict || !dict->entries)) return 0;
DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
if(rw == 'r' || rw == 'R')
dictionary_lock_rlock(dict);
else
dictionary_lock_wrlock(dict);
size_t count = dict->entries;
NAME_VALUE **array = mallocz(sizeof(NAME_VALUE *) * count);
size_t i;
NAME_VALUE *nv;
for(nv = dict->first_item, i = 0; nv && i < count ;nv = nv->next, i++)
array[i] = nv;
if(unlikely(nv))
error("DICTIONARY: during sorting expected to have %zu items in dictionary, but there are more. Sorted results may be incomplete. This is internal error - dictionaries fail to maintain an accurate number of the number of entries they have.", count);
if(unlikely(i != count)) {
error("DICTIONARY: during sorting expected to have %zu items in dictionary, but there are %zu. Sorted results may be incomplete. This is internal error - dictionaries fail to maintain an accurate number of the number of entries they have.", count, i);
count = i;
}
qsort(array, count, sizeof(NAME_VALUE *), dictionary_sort_compar);
int ret = 0;
for(i = 0; i < count ;i++) {
int r = callback((array[i])->name, (array[i])->value, data);
if(r < 0) { ret = r; break; }
ret += r;
}
dictionary_unlock(dict);
freez(array);
return ret;
}
// ----------------------------------------------------------------------------
// unit test
@ -1196,7 +1195,7 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m
return dt;
}
void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
static void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_clone);
dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
@ -1211,7 +1210,7 @@ void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, si
dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
}
void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
static void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_nonclone);
dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_nonclone);
dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
@ -1226,6 +1225,48 @@ void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values,
dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
}
struct dictionary_unittest_sorting {
const char *oldname;
const char *oldvalue;
size_t count;
};
static int dictionary_unittest_sorting_callback(const char *name, void *value, void *data) {
struct dictionary_unittest_sorting *t = (struct dictionary_unittest_sorting *)data;
const char *v = (const char *)value;
int ret = 0;
if(t->oldname && strcmp(t->oldname, name) > 0) {
fprintf(stderr, "name '%s' should be after '%s'\n", t->oldname, name);
ret = 1;
}
t->count++;
t->oldname = name;
t->oldvalue = v;
return ret;
}
static size_t dictionary_unittest_sorted_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) {
(void)names;
(void)values;
struct dictionary_unittest_sorting tmp = { .oldname = NULL, .oldvalue = NULL, .count = 0 };
size_t errors;
errors = dictionary_sorted_walkthrough_read(dict, dictionary_unittest_sorting_callback, &tmp);
if(tmp.count != entries) {
fprintf(stderr, "Expected %zu entries, counted %zu\n", entries, tmp.count);
errors++;
}
return errors;
}
static void dictionary_unittest_sorting(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
dictionary_unittest_run_and_measure_time(dict, "sorted walkthrough", names, values, entries, errors, dictionary_unittest_sorted_walkthrough);
dictionary_unittest_run_and_measure_time(dict, "destroying dictionary", names, values, entries, errors, dictionary_unittest_destroy);
}
int dictionary_unittest(size_t entries) {
if(entries < 10) entries = 10;
@ -1237,23 +1278,23 @@ int dictionary_unittest(size_t entries) {
char **values = dictionary_unittest_generate_values(entries);
fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
dictionary_unittest_clone(dict, names, values, entries, &errors);
fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS);
dict = dictionary_create(DICTIONARY_FLAG_NONE);
dictionary_unittest_clone(dict, names, values, entries, &errors);
fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
dictionary_unittest_nonclone(dict, names, values, entries, &errors);
fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
dictionary_unittest_nonclone(dict, names, values, entries, &errors);
fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone);
dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach);
@ -1262,19 +1303,23 @@ int dictionary_unittest(size_t entries) {
dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this);
dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this);
dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop empty", names, values, 0, &errors, dictionary_unittest_foreach);
dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback empty", names, values, 0, &errors, dictionary_unittest_walkthrough);
dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
dictionary_unittest_sorting(dict, names, values, entries, &errors);
dictionary_unittest_free_char_pp(names, entries);
dictionary_unittest_free_char_pp(values, entries);

12
libnetdata/dictionary/dictionary.h
View file

@ -44,10 +44,9 @@ typedef enum dictionary_flags {
DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks)
DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy)
DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy)
DICTIONARY_FLAG_WITH_STATISTICS = (1 << 3), // maintain statistics about dictionary operations (default: disabled)
DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 4), // don't overwrite values of dictionary items (default: overwrite)
DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 5), // add dictionary items at the front of the linked list (default: at the end)
DICTIONARY_FLAG_RESERVED1 = (1 << 6), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS
DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite)
DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end)
DICTIONARY_FLAG_RESERVED1 = (1 << 5), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS
} DICTIONARY_FLAGS;
// Create a dictionary
@ -126,6 +125,10 @@ extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name);
#define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data)
extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *value, void *data), void *data);
#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data)
#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data)
int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data);
// Traverse with foreach
//
// Use like this:
@ -184,6 +187,7 @@ extern size_t dictionary_stats_inserts(DICTIONARY *dict);
extern size_t dictionary_stats_searches(DICTIONARY *dict);
extern size_t dictionary_stats_deletes(DICTIONARY *dict);
extern size_t dictionary_stats_resets(DICTIONARY *dict);
extern size_t dictionary_stats_walkthroughs(DICTIONARY *dict);
extern int dictionary_unittest(size_t entries);

28
libnetdata/inlined.h
View file

@ -233,21 +233,21 @@ static inline char *strncpyz(char *dst, const char *src, size_t n) {
return p;
}
static inline void sanitize_json_string(char *dst, char *src, size_t len) {
while (*src != '\0' && len > 1) {
if (*src == '\\' || *src == '\"' || *src < 0x1F) {
if (*src < 0x1F) {
*dst++ = '_';
src++;
len--;
} else {
*dst++ = '\\';
*dst++ = *src++;
len -= 2;
}
} else {
static inline void sanitize_json_string(char *dst, const char *src, size_t dst_size) {
while (*src != '\0' && dst_size > 1) {
if (*src < 0x1F) {
*dst++ = '_';
src++;
dst_size--;
}
else if (*src == '\\' || *src == '\"') {
*dst++ = '\\';
*dst++ = *src++;
len--;
dst_size -= 2;
}
else {
*dst++ = *src++;
dst_size--;
}
}
*dst = '\0';

9
libnetdata/json/json.h
View file

@ -3,8 +3,13 @@
#if ENABLE_JSONC
# include <json-c/json.h>
#endif
#include <json-c/json.h>
// fix an older json-c bug
// https://github.com/json-c/json-c/issues/135
#ifdef error_description
#undef error_description
#endif // error_description
#endif // ENABLE_JSONC
#include "jsmn.h"

8
parser/parser.h
View file

@ -29,10 +29,10 @@ typedef struct pluginsd_action {
PARSER_RC (*flush_action)(void *user, RRDSET *st);
PARSER_RC (*disable_action)(void *user);
PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
PARSER_RC (*label_action)(void *user, char *key, char *value, LABEL_SOURCE source);
PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, struct label *new_labels);
PARSER_RC (*clabel_action)(void *user, char *key, char *value, LABEL_SOURCE source);
PARSER_RC (*clabel_commit_action)(void *user, RRDHOST *host, struct label *new_labels);
PARSER_RC (*label_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
PARSER_RC (*clabel_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
PARSER_RC (*clabel_commit_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
PARSER_RC (*context_action)(void *user, uuid_t *uuid);

2
registry/registry_internals.h
View file

@ -8,7 +8,7 @@
#define REGISTRY_URL_FLAGS_DEFAULT 0x00
#define REGISTRY_URL_FLAGS_EXPIRED 0x01
#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED | DICTIONARY_FLAG_WITH_STATISTICS)
#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED)
// ----------------------------------------------------------------------------
// COMMON structures

9
streaming/receiver.c
View file

@ -660,7 +660,14 @@ static int rrdpush_receive(struct receiver_state *rpt)
*/
// rpt->host->connected_senders++;
rpt->host->labels.labels_flag = (rpt->stream_version > 0)?LABEL_FLAG_UPDATE_STREAM:LABEL_FLAG_STOP_STREAM;
if(rpt->stream_version > 0) {
rrdhost_flag_set(rpt->host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_STREAM_LABELS_STOP);
}
else {
rrdhost_flag_set(rpt->host, RRDHOST_FLAG_STREAM_LABELS_STOP);
rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
}
if(health_enabled != CONFIG_BOOLEAN_NO) {
if(alarms_delay > 0) {

48
streaming/rrdpush.c
View file

@ -193,20 +193,15 @@ static inline int need_to_send_chart_definition(RRDSET *st) {
}
// chart labels
static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
BUFFER *wb = (BUFFER *)data;
buffer_sprintf(wb, "CLABEL \"%s\" \"%s\" %d\n", name, value, ls);
return 1;
}
void rrdpush_send_clabels(RRDHOST *host, RRDSET *st) {
struct label_index *labels_c = &st->state->labels;
if (labels_c) {
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
struct label *lbl = labels_c->head;
while(lbl) {
buffer_sprintf(host->sender->build,
"CLABEL \"%s\" \"%s\" %d\n", lbl->key, lbl->value, (int)lbl->label_source);
lbl = lbl->next;
}
if (labels_c->head)
if (st->state && st->state->chart_labels) {
if(rrdlabels_walkthrough_read(st->state->chart_labels, send_clabels_callback, host->sender->build) > 0)
buffer_sprintf(host->sender->build,"CLABEL_COMMIT\n");
netdata_rwlock_unlock(&host->labels.labels_rwlock);
}
}
@ -364,36 +359,25 @@ void rrdset_done_push(RRDSET *st) {
}
// labels
static int send_labels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
BUFFER *wb = (BUFFER *)data;
buffer_sprintf(wb, "LABEL \"%s\" = %d \"%s\"\n", name, ls, value);
return 1;
}
void rrdpush_send_labels(RRDHOST *host) {
if (!host->labels.head || !(host->labels.labels_flag & LABEL_FLAG_UPDATE_STREAM) || (host->labels.labels_flag & LABEL_FLAG_STOP_STREAM))
if (!host->host_labels || !rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE) || (rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_LABELS_STOP)))
return;
sender_start(host->sender);
rrdhost_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
struct label *label_i = host->labels.head;
while(label_i) {
buffer_sprintf(host->sender->build
, "LABEL \"%s\" = %d %s\n"
, label_i->key
, (int)label_i->label_source
, label_i->value);
label_i = label_i->next;
}
buffer_sprintf(host->sender->build
, "OVERWRITE %s\n", "labels");
netdata_rwlock_unlock(&host->labels.labels_rwlock);
rrdhost_unlock(host);
rrdlabels_walkthrough_read(host->host_labels, send_labels_callback, host->sender->build);
buffer_sprintf(host->sender->build, "OVERWRITE %s\n", "labels");
sender_commit(host->sender);
if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
host->labels.labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
}
void rrdpush_claimed_id(RRDHOST *host)

8
streaming/sender.c
View file

@ -172,8 +172,8 @@ static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
}
static inline void rrdpush_set_flags_to_newest_stream(RRDHOST *host) {
host->labels.labels_flag |= LABEL_FLAG_UPDATE_STREAM;
host->labels.labels_flag &= ~LABEL_FLAG_STOP_STREAM;
rrdhost_flag_set(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_STOP);
}
void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host)
@ -236,8 +236,8 @@ static inline long int parse_stream_version(RRDHOST *host, char *http)
answer = memcmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT));
if (!answer) {
stream_version = 0;
host->labels.labels_flag |= LABEL_FLAG_STOP_STREAM;
host->labels.labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
rrdhost_flag_set(host, RRDHOST_FLAG_STREAM_LABELS_STOP);
rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
}
else {
stream_version = parse_stream_version_for_errors(http);

41
web/api/formatters/json_wrapper.c
View file

@ -19,6 +19,21 @@ static int value_list_output(const char *name, void *entry, void *data) {
return 0;
}
static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
(void)ls;
DICTIONARY *dict = (DICTIONARY *)data;
char n[RRD_ID_LENGTH_MAX * 2 + 2];
char output[RRD_ID_LENGTH_MAX * 2 + 8];
char v[RRD_ID_LENGTH_MAX * 2 + 1];
sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v);
snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v);
dictionary_set(dict, n, output, len + 1);
return 1;
}
void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
QUERY_PARAMS *rrdset_query_data)
{
@ -122,7 +137,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
char name[RRD_ID_LENGTH_MAX * 2 + 2];
char output[RRD_ID_LENGTH_MAX * 2 + 8];
char value[RRD_ID_LENGTH_MAX * 2 + 1];
struct value_output co = {.c = 0, .wb = wb};
@ -153,19 +167,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
st = rd->rrdset;
if (likely(st->state)) {
struct label_index *labels = &st->state->labels;
if (labels->head) {
netdata_rwlock_rdlock(&labels->labels_rwlock);
for (struct label *label = labels->head; label; label = label->next) {
sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
dictionary_set(dict, name, output, len + 1);
}
netdata_rwlock_unlock(&labels->labels_rwlock);
}
}
if (st->state && st->state->chart_labels)
rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict);
}
dictionary_walkthrough_read(dict, value_list_output, &co);
dictionary_destroy(dict);
@ -207,8 +210,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
char *label_key = NULL;
int keys = 0;
while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
uint32_t key_hash = simple_hash(label_key);
struct label *current_label;
if (keys)
buffer_strcat(wb, ", ");
@ -223,13 +224,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if (i)
buffer_strcat(wb, ", ");
current_label = rrdset_lookup_label_key(rd->rrdset, label_key, key_hash);
if (current_label) {
buffer_strcat(wb, sq);
buffer_strcat(wb, current_label->value);
buffer_strcat(wb, sq);
} else
buffer_strcat(wb, "null");
rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null");
i++;
}
if (!i) {

18
web/api/formatters/rrdset2json.c
View file

@ -4,8 +4,10 @@
void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
{
if(unlikely(!st->state || !st->state->chart_labels))
return;
char tabs[11];
struct label_index *labels = &st->state->labels;
if (indentation > 10)
indentation = 10;
@ -16,20 +18,8 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
indentation--;
}
int count = 0;
netdata_rwlock_rdlock(&labels->labels_rwlock);
for (struct label *label = labels->head; label; label = label->next) {
if(count > 0) buffer_strcat(wb, ",\n");
buffer_strcat(wb, tabs);
char value[CONFIG_MAX_VALUE * 2 + 1];
sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
count++;
}
rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
buffer_strcat(wb, "\n");
netdata_rwlock_unlock(&labels->labels_rwlock);
}
// generate JSON for the /api/v1/chart API call

34
web/api/web_api_v1.c
View file

@ -392,6 +392,7 @@ void fix_google_param(char *s) {
}
}
// returns the HTTP code
inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url) {
debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
@ -528,18 +529,23 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
uint32_t context_hash = simple_hash(context);
SIMPLE_PATTERN *chart_label_key_pattern = NULL;
if(chart_label_key)
chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
SIMPLE_PATTERN *chart_labels_filter_pattern = NULL;
if(chart_labels_filter)
chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
rrdhost_rdlock(host);
char *words[MAX_CHART_LABELS_FILTER];
uint32_t hash_key_list[MAX_CHART_LABELS_FILTER];
int word_count = 0;
rrdset_foreach_read(st1, host) {
if (st1->hash_context == context_hash && !strcmp(st1->context, context) &&
(!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) &&
(!chart_labels_filter ||
rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER)))
(!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) &&
(!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':')))
build_context_param_list(owa, &context_param_list, st1);
}
rrdhost_unlock(host);
if (likely(context_param_list && context_param_list->rd)) // Just set the first one
st = context_param_list->rd->rrdset;
else {
@ -964,22 +970,8 @@ inline void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation) {
indentation--;
}
int count = 0;
rrdhost_rdlock(host);
netdata_rwlock_rdlock(&host->labels.labels_rwlock);
for (struct label *label = host->labels.head; label; label = label->next) {
if(count > 0) buffer_strcat(wb, ",\n");
buffer_strcat(wb, tabs);
char value[CONFIG_MAX_VALUE * 2 + 1];
sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
count++;
}
rrdlabels_to_buffer(host->host_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
buffer_strcat(wb, "\n");
netdata_rwlock_unlock(&host->labels.labels_rwlock);
rrdhost_unlock(host);
}
extern int aclk_connected;