diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd803848ce..fe0dcb331e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES
database/engine/pagecache.h
database/engine/rrdenglocking.c
database/engine/rrdenglocking.h
+ database/engine/metadata_log/metadatalog.c
+ database/engine/metadata_log/metadatalog.h
+ database/engine/metadata_log/metadatalogapi.c
+ database/engine/metadata_log/metadatalogapi.h
+ database/engine/metadata_log/logfile.h
+ database/engine/metadata_log/logfile.c
+ database/engine/metadata_log/metadatalogprotocol.h
+ database/engine/metadata_log/metalogpluginsd.c
+ database/engine/metadata_log/metalogpluginsd.h
+ database/engine/metadata_log/compaction.c
+ database/engine/metadata_log/compaction.h
+ database/engine/global_uuid_map/global_uuid_map.c
+ database/engine/global_uuid_map/global_uuid_map.h
)
set(WEB_PLUGIN_FILES
diff --git a/Makefile.am b/Makefile.am
index 889a3d8569..9f72ede368 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -375,6 +375,19 @@ if ENABLE_DBENGINE
database/engine/pagecache.h \
database/engine/rrdenglocking.c \
database/engine/rrdenglocking.h \
+ database/engine/metadata_log/metadatalog.c \
+ database/engine/metadata_log/metadatalog.h \
+ database/engine/metadata_log/metadatalogapi.c \
+ database/engine/metadata_log/metadatalogapi.h \
+ database/engine/metadata_log/logfile.h \
+ database/engine/metadata_log/logfile.c \
+ database/engine/metadata_log/metadatalogprotocol.h \
+ database/engine/metadata_log/metalogpluginsd.c \
+ database/engine/metadata_log/metalogpluginsd.h \
+ database/engine/metadata_log/compaction.c \
+ database/engine/metadata_log/compaction.h \
+ database/engine/global_uuid_map/global_uuid_map.c \
+ database/engine/global_uuid_map/global_uuid_map.h \
$(NULL)
endif
diff --git a/aclk/agent_cloud_link.c b/aclk/agent_cloud_link.c
index cc366a8332..7b5aad5c4a 100644
--- a/aclk/agent_cloud_link.c
+++ b/aclk/agent_cloud_link.c
@@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi
void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
{
struct _collector *tmp_collector;
+ if (unlikely(!netdata_ready)) {
+ return;
+ }
COLLECTOR_LOCK;
@@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha
void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
{
struct _collector *tmp_collector;
+ if (unlikely(!netdata_ready)) {
+ return;
+ }
COLLECTOR_LOCK;
@@ -1752,7 +1758,7 @@ int aclk_send_info_metadata()
debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len);
buffer_sprintf(local_buffer, ", \n\t \"charts\" : ");
- charts2json(localhost, local_buffer, 1);
+ charts2json(localhost, local_buffer, 1, 0);
buffer_sprintf(local_buffer, "\n}\n}");
debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len);
@@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd)
UNUSED(chart_name);
return 0;
#else
+ if (unlikely(!netdata_ready))
+ return 0;
+
if (!netdata_cloud_setting)
return 0;
@@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
{
BUFFER *local_buffer = NULL;
+ if (unlikely(!netdata_ready))
+ return 0;
+
if (host != localhost)
return 0;
diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c
index eab607d84f..374f7cdf1c 100644
--- a/collectors/diskspace.plugin/plugin_diskspace.c
+++ b/collectors/diskspace.plugin/plugin_diskspace.c
@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
if(unlikely(!m->st_space)) {
m->do_space = CONFIG_BOOLEAN_YES;
- m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
+ m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
if(unlikely(!m->st_space)) {
char title[4096 + 1];
snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
if(unlikely(!m->st_inodes)) {
m->do_inodes = CONFIG_BOOLEAN_YES;
- m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
+ m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
if(unlikely(!m->st_inodes)) {
char title[4096 + 1];
snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);
diff --git a/collectors/freebsd.plugin/plugin_freebsd.c b/collectors/freebsd.plugin/plugin_freebsd.c
index 5cde371131..bee8395f53 100644
--- a/collectors/freebsd.plugin/plugin_freebsd.c
+++ b/collectors/freebsd.plugin/plugin_freebsd.c
@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
static RRDSET *st = NULL;
if(unlikely(!st)) {
- st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
+ st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
if(!st) {
st = rrdset_create_localhost(
diff --git a/collectors/macos.plugin/macos_fw.c b/collectors/macos.plugin/macos_fw.c
index f253489a5b..d0b3e0fd2c 100644
--- a/collectors/macos.plugin/macos_fw.c
+++ b/collectors/macos.plugin/macos_fw.c
@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
total_disk_writes += diskstat.bytes_write;
}
- st = rrdset_find_bytype_localhost("disk", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
}
- st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
}
- st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
}
- st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
+ st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
}
if (likely(do_io)) {
- st = rrdset_find_bytype_localhost("system", "io");
+ st = rrdset_find_active_bytype_localhost("system", "io");
if (unlikely(!st)) {
st = rrdset_create_localhost(
"system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------------
if (likely(do_space)) {
- st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+ st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
if (unlikely(!st)) {
snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------------
if (likely(do_inodes)) {
- st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+ st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
if (unlikely(!st)) {
snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
+ st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
+ st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
+ st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
+ st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
+ st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
if (unlikely(!st)) {
st = rrdset_create_localhost(
"net_events"
diff --git a/collectors/macos.plugin/macos_mach_smi.c b/collectors/macos.plugin/macos_mach_smi.c
index 800b2ce56b..250186cef7 100644
--- a/collectors/macos.plugin/macos_mach_smi.c
+++ b/collectors/macos.plugin/macos_mach_smi.c
@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
error("DISABLED: system.cpu");
} else {
- st = rrdset_find_bytype_localhost("system", "cpu");
+ st = rrdset_find_active_bytype_localhost("system", "cpu");
if (unlikely(!st)) {
st = rrdset_create_localhost(
"system"
diff --git a/collectors/macos.plugin/macos_sysctl.c b/collectors/macos.plugin/macos_sysctl.c
index dddafc9f51..80b66963b5 100644
--- a/collectors/macos.plugin/macos_sysctl.c
+++ b/collectors/macos.plugin/macos_sysctl.c
@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
error("DISABLED: system.load");
} else {
- st = rrdset_find_bytype_localhost("system", "load");
+ st = rrdset_find_active_bytype_localhost("system", "load");
if (unlikely(!st)) {
st = rrdset_create_localhost(
"system"
diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h
index d8bb1b955f..fd99b35843 100644
--- a/collectors/plugins.d/plugins_d.h
+++ b/collectors/plugins.d/plugins_d.h
@@ -31,10 +31,10 @@
#define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
#define PLUGINSD_KEYWORD_LABEL "LABEL"
#define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
#define PLUGINSD_KEYWORD_GUID "GUID"
-#define PLUGINSD_KEYWORD_HOST "HOST"
+#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
#define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
+#define PLUGINSD_KEYWORD_HOST "HOST"
#define PLUGINSD_LINE_MAX 1024
diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c
index 90558f7e58..ce5bf5fcf0 100644
--- a/collectors/plugins.d/pluginsd_parser.c
+++ b/collectors/plugins.d/pluginsd_parser.c
@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins
return PARSER_RC_OK;
}
+PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+ char *uuid_str = words[1];
+ uuid_t uuid;
+
+ if (unlikely(!uuid_str)) {
+ error("requested a GUID, without a uuid.");
+ return PARSER_RC_ERROR;
+ }
+ if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+ error("requested a GUID, without a valid uuid string.");
+ return PARSER_RC_ERROR;
+ }
+
+ debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+ if (plugins_action->guid_action) {
+ return plugins_action->guid_action(user, &uuid);
+ }
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+ char *uuid_str = words[1];
+ uuid_t uuid;
+
+ if (unlikely(!uuid_str)) {
+ error("requested a CONTEXT, without a uuid.");
+ return PARSER_RC_ERROR;
+ }
+ if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+ error("requested a CONTEXT, without a valid uuid string.");
+ return PARSER_RC_ERROR;
+ }
+
+ debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+ if (plugins_action->context_action) {
+ return plugins_action->context_action(user, &uuid);
+ }
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+ char *uuid_str = words[1];
+ uuid_t uuid;
+
+ if (unlikely(!uuid_str)) {
+ error("requested a TOMBSTONE, without a uuid.");
+ return PARSER_RC_ERROR;
+ }
+ if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+ error("requested a TOMBSTONE, without a valid uuid string.");
+ return PARSER_RC_ERROR;
+ }
+
+ debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+ if (plugins_action->tombstone_action) {
+ return plugins_action->tombstone_action(user, &uuid);
+ }
+
+ return PARSER_RC_OK;
+}
// New plugins.d parser
diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h
index ea9ef40b58..ba79373cd6 100644
--- a/collectors/plugins.d/pluginsd_parser.h
+++ b/collectors/plugins.d/pluginsd_parser.h
@@ -16,19 +16,23 @@ typedef struct parser_user_object {
struct label *new_labels;
size_t count;
int enabled;
+ void *private; // the user can set this for private use
} PARSER_USER_OBJECT;
-PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
-PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
-PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
- char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
-PARSER_RC pluginsd_disable_action(void *user);
-PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
-PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
- RRD_ALGORITHM algorithm_type);
-PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
-PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
+extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
+extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+ char *title, char *units, char *plugin, char *module, int priority,
+ int update_every, RRDSET_TYPE chart_type, char *options);
+extern PARSER_RC pluginsd_disable_action(void *user);
+extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global,
+ calculated_number value);
+extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+ long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
+extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
+extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+
#endif //NETDATA_PLUGINSD_PARSER_H
diff --git a/collectors/proc.plugin/plugin_proc.c b/collectors/proc.plugin/plugin_proc.c
index c9eef2c257..1334f5d210 100644
--- a/collectors/proc.plugin/plugin_proc.c
+++ b/collectors/proc.plugin/plugin_proc.c
@@ -148,7 +148,7 @@ void *proc_main(void *ptr) {
static RRDSET *st = NULL;
if(unlikely(!st)) {
- st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules");
+ st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
if(!st) {
st = rrdset_create_localhost(
diff --git a/collectors/proc.plugin/proc_net_softnet_stat.c b/collectors/proc.plugin/proc_net_softnet_stat.c
index 7ec783e77d..a29ccccd1a 100644
--- a/collectors/proc.plugin/proc_net_softnet_stat.c
+++ b/collectors/proc.plugin/proc_net_softnet_stat.c
@@ -81,7 +81,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
// --------------------------------------------------------------------
- st = rrdset_find_bytype_localhost("system", "softnet_stat");
+ st = rrdset_find_active_bytype_localhost("system", "softnet_stat");
if(unlikely(!st)) {
st = rrdset_create_localhost(
"system"
@@ -114,7 +114,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
char id[50+1];
snprintfz(id, 50, "cpu%zu_softnet_stat", l);
- st = rrdset_find_bytype_localhost("cpu", id);
+ st = rrdset_find_active_bytype_localhost("cpu", id);
if(unlikely(!st)) {
char title[100+1];
snprintfz(title, 100, "CPU%zu softnet_stat", l);
diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c
index df69031dea..1e69d53460 100644
--- a/collectors/statsd.plugin/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -1461,6 +1461,8 @@ static inline RRDSET *statsd_private_rrdset_create(
, chart_type // chart type
, memory_mode // memory mode
, history // history
+ , 0 // not archived
+ , NULL // no known UUID
);
rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
@@ -1999,6 +2001,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
, chart->chart_type // chart type
, app->rrd_memory_mode // memory mode
, app->rrd_history_entries // history
+ , 0 // not archived
+ , NULL // no known UUID
);
rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
diff --git a/configure.ac b/configure.ac
index 8fa593c001..4bebda621d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1437,6 +1437,8 @@ AC_CONFIG_FILES([
daemon/Makefile
database/Makefile
database/engine/Makefile
+ database/engine/metadata_log/Makefile
+ database/engine/global_uuid_map/Makefile
diagrams/Makefile
exporting/Makefile
exporting/graphite/Makefile
diff --git a/daemon/common.h b/daemon/common.h
index 742ca4a766..f35c6a8828 100644
--- a/daemon/common.h
+++ b/daemon/common.h
@@ -68,9 +68,15 @@
// netdata agent cloud link
#include "aclk/agent_cloud_link.h"
+// global GUID map functions
+
// netdata agent spawn server
#include "spawn/spawn.h"
+#ifdef ENABLE_DBENGINE
+#include "database/engine/global_uuid_map/global_uuid_map.h"
+#endif
+
// the netdata deamon
#include "daemon.h"
#include "main.h"
diff --git a/daemon/main.c b/daemon/main.c
index dd556af640..2087d651ff 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -1400,6 +1400,9 @@ int main(int argc, char **argv) {
struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
get_system_info(system_info);
+#ifdef ENABLE_DBENGINE
+ init_global_guid_map();
+#endif
if(rrd_init(netdata_configured_hostname, system_info))
fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);
@@ -1417,6 +1420,9 @@ int main(int argc, char **argv) {
// Load host labels
reload_host_labels();
+#ifdef ENABLE_DBENGINE
+ metalog_commit_update_host(localhost);
+#endif
// ------------------------------------------------------------------------
// spawn the threads
diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index 323ae285a3..116fe7061d 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -1833,9 +1833,10 @@ int test_dbengine(void)
}
}
error_out:
- rrdeng_exit(host->rrdeng_ctx);
rrd_wrlock();
+ rrdeng_prepare_exit(host->rrdeng_ctx);
rrdhost_delete_charts(host);
+ rrdeng_exit(host->rrdeng_ctx);
rrd_unlock();
return errors;
@@ -2222,9 +2223,10 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
freez(query_threads[i]);
}
freez(query_threads);
- rrdeng_exit(host->rrdeng_ctx);
rrd_wrlock();
+ rrdeng_prepare_exit(host->rrdeng_ctx);
rrdhost_delete_charts(host);
+ rrdeng_exit(host->rrdeng_ctx);
rrd_unlock();
}
diff --git a/database/engine/Makefile.am b/database/engine/Makefile.am
index 161784b8f6..90fdc6bac7 100644
--- a/database/engine/Makefile.am
+++ b/database/engine/Makefile.am
@@ -3,6 +3,11 @@
AUTOMAKE_OPTIONS = subdir-objects
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+SUBDIRS = \
+ metadata_log \
+ global_uuid_map \
+ $(NULL)
+
dist_noinst_DATA = \
README.md \
$(NULL)
diff --git a/database/engine/global_uuid_map/Makefile.am b/database/engine/global_uuid_map/Makefile.am
new file mode 100644
index 0000000000..161784b8f6
--- /dev/null
+++ b/database/engine/global_uuid_map/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/database/engine/global_uuid_map/README.md b/database/engine/global_uuid_map/README.md
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/database/engine/global_uuid_map/global_uuid_map.c b/database/engine/global_uuid_map/global_uuid_map.c
new file mode 100644
index 0000000000..340a353528
--- /dev/null
+++ b/database/engine/global_uuid_map/global_uuid_map.c
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "global_uuid_map.h"
+
+static Pvoid_t JGUID_map = (Pvoid_t) NULL;
+static Pvoid_t JGUID_object_map = (Pvoid_t) NULL;
+static uv_rwlock_t guid_lock;
+static uv_rwlock_t object_lock;
+static uv_rwlock_t global_lock;
+
+
+void dump_object(uuid_t *index, void *object)
+{
+ char uuid_s[36 + 1];
+ uuid_unparse_lower(*index, uuid_s);
+ char local_object[3 * 36 + 2 + 1];
+
+ switch (*(char *) object) {
+ case GUID_TYPE_CHAR:
+ debug(D_GUIDLOG, "OBJECT GUID %s on [%s]", uuid_s, (char *)object + 1);
+ break;
+ case GUID_TYPE_CHART:
+ uuid_unparse_lower((const unsigned char *)object + 1, local_object);
+ uuid_unparse_lower((const unsigned char *)object + 17, local_object+37);
+ local_object[36] = ':';
+ local_object[74] = '\0';
+ debug(D_GUIDLOG, "CHART GUID %s on [%s]", uuid_s, local_object);
+ break;
+ case GUID_TYPE_DIMENSION:
+ uuid_unparse_lower((const unsigned char *)object + 1, local_object);
+ uuid_unparse_lower((const unsigned char *)object + 17, local_object + 37);
+ uuid_unparse_lower((const unsigned char *)object + 33, local_object + 74);
+ local_object[36] = ':';
+ local_object[73] = ':';
+ local_object[110] = '\0';
+ debug(D_GUIDLOG, "DIM GUID %s on [%s]", uuid_s, local_object);
+ break;
+ default:
+ debug(D_GUIDLOG, "Unknown object");
+ }
+}
+
+/* Returns 0 if it successfully stores the uuid-object mapping or if an identical mapping already exists */
+static inline int guid_store_nolock(uuid_t *uuid, void *object, GUID_TYPE object_type)
+{
+ char *existing_object;
+ GUID_TYPE existing_object_type;
+
+ if (unlikely(!object) || uuid == NULL)
+ return 0;
+
+ Pvoid_t *PValue;
+
+ PValue = JudyHSIns(&JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t), PJE0);
+ if (PPJERR == PValue)
+ fatal("JudyHSIns() fatal error.");
+ if (*PValue) {
+ existing_object = *PValue;
+ existing_object_type = existing_object[0];
+ if (existing_object_type != object_type)
+ return 1;
+ switch (existing_object_type) {
+ case GUID_TYPE_DIMENSION:
+ if (memcmp(existing_object, object, 1 + 16 + 16 + 16))
+ return 1;
+ break;
+ case GUID_TYPE_CHART:
+ if (memcmp(existing_object, object, 1 + 16 + 16))
+ return 1;
+ break;
+ case GUID_TYPE_CHAR:
+ if (strcmp(existing_object + 1, (char *)object))
+ return 1;
+ break;
+ default:
+ return 1;
+ }
+ freez(existing_object);
+ }
+
+ *PValue = (Pvoid_t *) object;
+
+ PValue = JudyHSIns(&JGUID_object_map, (void *)object, (Word_t) object_type?(object_type * 16)+1:strlen((char *) object+1)+2, PJE0);
+ if (PPJERR == PValue)
+ fatal("JudyHSIns() fatal error.");
+ if (*PValue == NULL) {
+ uuid_t *value = (uuid_t *) mallocz(sizeof(uuid_t));
+ uuid_copy(*value, *uuid);
+ *PValue = value;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ static uint32_t count = 0;
+ count++;
+ char uuid_s[36 + 1];
+ uuid_unparse_lower(*uuid, uuid_s);
+ debug(D_GUIDLOG,"GUID added item %" PRIu32" [%s] as:", count, uuid_s);
+ dump_object(uuid, object);
+#endif
+ return 0;
+}
+
+
+inline int guid_store(uuid_t *uuid, char *object, GUID_TYPE object_type)
+{
+ uv_rwlock_wrlock(&global_lock);
+ int rc = guid_store_nolock(uuid, object, object_type);
+ uv_rwlock_wrunlock(&global_lock);
+ return rc;
+}
+
+/*
+ * This can be used to bulk load entries into the global map
+ *
+ * A lock must be aquired since it will call guid_store_nolock
+ * with a "no lock" parameter.
+ *
+ * Note: object memory must be allocated by caller and not released
+ */
+int guid_bulk_load(char *uuid, char *object)
+{
+ uuid_t target_uuid;
+ if (likely(!uuid_parse(uuid, target_uuid))) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ debug(D_GUIDLOG,"Mapping GUID [%s] on [%s]", uuid, object);
+#endif
+ return guid_store_nolock(&target_uuid, object, GUID_TYPE_CHAR);
+ }
+ return 1;
+}
+
+/*
+ * Given a GUID, find if an object is stored
+ * - Optionally return the object
+ */
+
+GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
+{
+ Pvoid_t *PValue;
+ GUID_TYPE value_type;
+
+ uv_rwlock_rdlock(&global_lock);
+ PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t));
+ if (unlikely(!PValue)) {
+ uv_rwlock_rdunlock(&global_lock);
+ return GUID_TYPE_NOTFOUND;
+ }
+
+ value_type = *(char *) *PValue;
+
+ if (likely(object && max_bytes)) {
+ switch (value_type) {
+ case GUID_TYPE_CHAR:
+ if (unlikely(max_bytes - 1 < strlen((char *) *PValue+1)))
+ return GUID_TYPE_NOSPACE;
+ strncpyz(object, (char *) *PValue+1, max_bytes - 1);
+ break;
+ case GUID_TYPE_CHART:
+ case GUID_TYPE_DIMENSION:
+ if (unlikely(max_bytes < (size_t) value_type * 16))
+ return GUID_TYPE_NOSPACE;
+ memcpy(object, *PValue+1, value_type * 16);
+ break;
+ default:
+ uv_rwlock_rdunlock(&global_lock);
+ return GUID_TYPE_NOTFOUND;
+ }
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ dump_object(uuid, *PValue);
+#endif
+ uv_rwlock_rdunlock(&global_lock);
+ return value_type;
+}
+
+/*
+ * Find a GUID of an object
+ * - Optionally return the GUID
+ *
+ */
+
+int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE object_type)
+{
+ Pvoid_t *PValue;
+
+ uv_rwlock_rdlock(&global_lock);
+ PValue = JudyHSGet(JGUID_object_map, (void *)object, (Word_t)object_type?object_type*16+1:strlen(object+1)+2);
+ if (unlikely(!PValue)) {
+ uv_rwlock_rdunlock(&global_lock);
+ return 1;
+ }
+
+ if (likely(uuid))
+ uuid_copy(*uuid, *PValue);
+ uv_rwlock_rdunlock(&global_lock);
+ return 0;
+}
+
+int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate)
+{
+ char *target_object;
+ uuid_t temp_uuid;
+ int rc;
+
+ switch (object_type) {
+ case GUID_TYPE_DIMENSION:
+ if (unlikely(find_or_generate_guid((void *) ((RRDDIM *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
+ return 1;
+ target_object = mallocz(49);
+ target_object[0] = object_type;
+ memcpy(target_object + 1, ((RRDDIM *)object)->rrdset->rrdhost->host_uuid, 16);
+ memcpy(target_object + 17, ((RRDDIM *)object)->rrdset->chart_uuid, 16);
+ memcpy(target_object + 33, temp_uuid, 16);
+ break;
+ case GUID_TYPE_CHART:
+ if (unlikely(find_or_generate_guid((void *) ((RRDSET *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
+ return 1;
+ target_object = mallocz(33);
+ target_object[0] = object_type;
+ memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
+ memcpy(target_object + 17, temp_uuid, 16);
+ break;
+ case GUID_TYPE_CHAR:
+ target_object = mallocz(strlen((char *) object)+2);
+ target_object[0] = object_type;
+ strcpy(target_object+1, (char *) object);
+ break;
+ default:
+ return 1;
+ }
+ rc = find_guid_by_object(target_object, uuid, object_type);
+ if (rc) {
+ if (!replace_instead_of_generate) /* else take *uuid as user input */
+ uuid_generate(*uuid);
+ uv_rwlock_wrlock(&global_lock);
+ int rc = guid_store_nolock(uuid, target_object, object_type);
+ uv_rwlock_wrunlock(&global_lock);
+ return rc;
+ }
+ //uv_rwlock_wrunlock(&global_lock);
+#ifdef NETDATA_INTERNAL_CHECKS
+ dump_object(uuid, target_object);
+#endif
+ return 0;
+}
+
+void init_global_guid_map()
+{
+ static int init = 0;
+
+ if (init)
+ return;
+
+ init = 1;
+ info("Configuring locking mechanism for global GUID map");
+ assert(0 == uv_rwlock_init(&guid_lock));
+ assert(0 == uv_rwlock_init(&object_lock));
+ assert(0 == uv_rwlock_init(&global_lock));
+
+// int rc = guid_bulk_load("6fc56a64-05d7-47a7-bc82-7f3235d8cbda","d6b37186-74db-11ea-88b2-0bf5095b1f9e/cgroup_qemu_ubuntu18.04.cpu_per_core/cpu3");
+// rc = guid_bulk_load("75c6fa02-97cc-40c1-aacd-a0132190472e","d6b37186-74db-11ea-88b2-0bf5095b1f9e/services.throttle_io_ops_write/system.slice_setvtrgb.service");
+// if (rc == 0)
+// info("BULK GUID load successful");
+
+ return;
+}
+
+
diff --git a/database/engine/global_uuid_map/global_uuid_map.h b/database/engine/global_uuid_map/global_uuid_map.h
new file mode 100644
index 0000000000..3def854baf
--- /dev/null
+++ b/database/engine/global_uuid_map/global_uuid_map.h
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_GLOBAL_UUID_MAP_H
+#define NETDATA_GLOBAL_UUID_MAP_H
+
+#include "libnetdata/libnetdata.h"
+#include <Judy.h>
+#include "../../rrd.h"
+
+typedef enum guid_type {
+ GUID_TYPE_CHAR,
+ GUID_TYPE_HOST,
+ GUID_TYPE_CHART,
+ GUID_TYPE_DIMENSION,
+ GUID_TYPE_NOTFOUND,
+ GUID_TYPE_NOSPACE
+} GUID_TYPE;
+
+extern int guid_store(uuid_t *uuid, char *object, GUID_TYPE);
+extern GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes);
+extern int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE);
+extern void init_global_guid_map();
+extern int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate);
+
+
+#endif //NETDATA_GLOBAL_UUID_MAP_H
diff --git a/database/engine/metadata_log/Makefile.am b/database/engine/metadata_log/Makefile.am
new file mode 100644
index 0000000000..161784b8f6
--- /dev/null
+++ b/database/engine/metadata_log/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/database/engine/metadata_log/README.md b/database/engine/metadata_log/README.md
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/database/engine/metadata_log/compaction.c b/database/engine/metadata_log/compaction.c
new file mode 100644
index 0000000000..9113fdd940
--- /dev/null
+++ b/database/engine/metadata_log/compaction.c
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+void after_compact_old_records(struct metalog_worker_config* wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ int error;
+
+ mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log, &ctx->compaction_state.new_metadata_logfiles);
+ uv_run(wc->loop, UV_RUN_DEFAULT);
+
+ error = uv_thread_join(wc->now_compacting_files);
+ if (error) {
+ error("uv_thread_join(): %s", uv_strerror(error));
+ }
+ freez(wc->now_compacting_files);
+ /* unfreeze command processing */
+ wc->now_compacting_files = NULL;
+
+ wc->cleanup_thread_compacting_files = 0;
+
+ /* interrupt event loop */
+ uv_stop(wc->loop);
+
+ info("Finished metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
+}
+
+static void metalog_flush_compaction_records(struct metalog_instance *ctx)
+{
+ struct metalog_cmd cmd;
+ struct completion compaction_completion;
+
+ init_completion(&compaction_completion);
+
+ cmd.opcode = METALOG_COMPACTION_FLUSH;
+ cmd.record_io_descr.completion = &compaction_completion;
+ metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+ wait_for_completion(&compaction_completion);
+ destroy_completion(&compaction_completion);
+}
+
+/* The caller must have called metalog_flush_compaction_records() before to synchronize and quiesce the event loop. */
+static void compaction_test_quota(struct metalog_worker_config *wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ struct logfile_compaction_state *compaction_state;
+ struct metadata_logfile *oldmetalogfile, *newmetalogfile;
+ unsigned current_size;
+ int ret;
+
+ compaction_state = &ctx->compaction_state;
+ newmetalogfile = compaction_state->new_metadata_logfiles.last;
+
+ oldmetalogfile = ctx->metadata_logfiles.first;
+
+ current_size = newmetalogfile->pos;
+ if (unlikely(current_size >= MAX_METALOGFILE_SIZE && newmetalogfile->starting_fileno < oldmetalogfile->fileno)) {
+ /* It's safe to finalize the compacted metadata log file and create a new one since it has already replaced
+ * an older one. */
+
+ /* Finalize as the immediately previous file than the currently compacted one. */
+ ret = rename_metadata_logfile(newmetalogfile, 0, newmetalogfile->fileno - 1);
+ if (ret < 0)
+ return;
+
+ ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles,
+ ctx->metadata_logfiles.first->fileno, ctx->metadata_logfiles.first->fileno);
+
+ if (likely(!ret)) {
+ compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
+ }
+ }
+}
+
+
+static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
+{
+ GUID_TYPE ret;
+ RRDHOST *host = ctx->rrdeng_ctx->host;
+ RRDSET *st;
+ RRDDIM *rd;
+ BUFFER *buffer;
+
+ ret = find_object_by_guid(uuid, NULL, 0);
+ switch (ret) {
+ case GUID_TYPE_CHAR:
+ assert(0);
+ break;
+ case GUID_TYPE_CHART:
+ st = metalog_get_chart_from_uuid(ctx, uuid);
+ if (st) {
+ if (ctx->current_compaction_id > st->compaction_id) {
+ st->compaction_id = ctx->current_compaction_id;
+ buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
+ metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+ } else {
+ debug(D_METADATALOG, "Chart has already been compacted, ignoring record.");
+ }
+ } else {
+ debug(D_METADATALOG, "Ignoring nonexistent chart metadata record.");
+ }
+ break;
+ case GUID_TYPE_DIMENSION:
+ rd = metalog_get_dimension_from_uuid(ctx, uuid);
+ if (rd) {
+ if (ctx->current_compaction_id > rd->state->compaction_id) {
+ rd->state->compaction_id = ctx->current_compaction_id;
+ buffer = metalog_update_dimension_buffer(rd);
+ metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+ } else {
+ debug(D_METADATALOG, "Dimension has already been compacted, ignoring record.");
+ }
+ } else {
+ debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record.");
+ }
+ break;
+ case GUID_TYPE_HOST:
+ if (ctx->current_compaction_id > host->compaction_id) {
+ host->compaction_id = ctx->current_compaction_id;
+ buffer = metalog_update_host_buffer(host);
+ metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+ } else {
+ debug(D_METADATALOG, "Host has already been compacted, ignoring record.");
+ }
+ break;
+ case GUID_TYPE_NOTFOUND:
+ debug(D_METADATALOG, "Ignoring nonexistent metadata record.");
+ break;
+ default:
+ assert(0);
+ break;
+ }
+}
+
+/* Returns 0 on success. */
+static int compact_metadata_logfile_records(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
+{
+ struct metalog_worker_config* wc = &ctx->worker_config;
+ struct logfile_compaction_state *compaction_state;
+ struct metalog_record *record;
+ struct metalog_record_block *record_block, *prev_record_block;
+ int ret;
+ unsigned iterated_records;
+#define METADATA_LOG_RECORD_BATCH 128 /* Flush I/O and check sizes whenever this many records have been iterated */
+
+ info("Compacting metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+
+ compaction_state = &ctx->compaction_state;
+ record_block = prev_record_block = NULL;
+ iterated_records = 0;
+ for (record = mlf_record_get_first(metalogfile) ; record != NULL ; record = mlf_record_get_next(metalogfile)) {
+ if ((record_block = metalogfile->records.iterator.current) != prev_record_block) {
+ if (prev_record_block) { /* Deallocate iterated record blocks */
+ rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
+ freez(prev_record_block);
+ }
+ prev_record_block = record_block;
+ }
+ compact_record_by_uuid(ctx, &record->uuid);
+ if (0 == ++iterated_records % METADATA_LOG_RECORD_BATCH) {
+ metalog_flush_compaction_records(ctx);
+ if (compaction_state->throttle) {
+ (void)sleep_usec(10000); /* 10 msec throttle compaction */
+ }
+ compaction_test_quota(wc);
+ }
+ }
+ if (prev_record_block) { /* Deallocate iterated record blocks */
+ rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
+ freez(prev_record_block);
+ }
+
+ info("Compacted metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+
+ metadata_logfile_list_delete(&ctx->metadata_logfiles, metalogfile);
+ ret = destroy_metadata_logfile(metalogfile);
+ if (!ret) {
+ info("Deleted file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+ rrd_atomic_fetch_add(&ctx->disk_space, -metalogfile->pos);
+ } else {
+ error("Failed to delete file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+ }
+ freez(metalogfile);
+
+ return ret;
+}
+
+static void compact_old_records(void *arg)
+{
+ struct metalog_instance *ctx = arg;
+ struct metalog_worker_config* wc = &ctx->worker_config;
+ struct logfile_compaction_state *compaction_state;
+ struct metadata_logfile *metalogfile, *nextmetalogfile, *newmetalogfile;
+ int ret;
+
+ compaction_state = &ctx->compaction_state;
+
+ nextmetalogfile = NULL;
+ for (metalogfile = ctx->metadata_logfiles.first ;
+ metalogfile != compaction_state->last_original_logfile ;
+ metalogfile = nextmetalogfile) {
+ nextmetalogfile = metalogfile->next;
+
+ newmetalogfile = compaction_state->new_metadata_logfiles.last;
+ ret = rename_metadata_logfile(newmetalogfile, newmetalogfile->starting_fileno, metalogfile->fileno);
+ if (ret < 0) {
+ error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
+ }
+
+ ret = compact_metadata_logfile_records(ctx, metalogfile);
+ if (ret) {
+ error("Metadata log compaction failed, cancelling.");
+ break;
+ }
+ }
+ assert(nextmetalogfile); /* There are always more than 1 metadata log files during compaction */
+
+ newmetalogfile = compaction_state->new_metadata_logfiles.last;
+ if (newmetalogfile->starting_fileno != 0) { /* Must rename the last compacted file */
+ ret = rename_metadata_logfile(newmetalogfile, 0, nextmetalogfile->fileno - 1);
+ if (ret < 0) {
+ error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+ ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
+ }
+ }
+ /* Connect the compacted files to the metadata log */
+ newmetalogfile->next = nextmetalogfile;
+ ctx->metadata_logfiles.first = compaction_state->new_metadata_logfiles.first;
+
+ wc->cleanup_thread_compacting_files = 1;
+ /* wake up event loop */
+ assert(0 == uv_async_send(&wc->async));
+}
+
+/* Returns 0 on success. */
+static int init_compaction_state(struct metalog_instance *ctx)
+{
+ struct metadata_logfile *newmetalogfile;
+ struct logfile_compaction_state *compaction_state;
+ int ret;
+
+ compaction_state = &ctx->compaction_state;
+ compaction_state->new_metadata_logfiles.first = NULL;
+ compaction_state->new_metadata_logfiles.last = NULL;
+ compaction_state->starting_fileno = ctx->metadata_logfiles.first->fileno;
+ compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
+ compaction_state->last_original_logfile = ctx->metadata_logfiles.last;
+ compaction_state->throttle = 0;
+
+ ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, compaction_state->starting_fileno,
+ compaction_state->fileno);
+ if (unlikely(ret)) {
+ error("Cannot create new metadata log files, compaction aborted.");
+ return ret;
+ }
+ newmetalogfile = compaction_state->new_metadata_logfiles.first;
+ assert(newmetalogfile == compaction_state->new_metadata_logfiles.last);
+ init_metadata_record_log(&compaction_state->records_log);
+
+ return 0;
+}
+
+void metalog_do_compaction(struct metalog_worker_config *wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ int error;
+
+ if (wc->now_compacting_files) {
+ /* already compacting metadata log files */
+ return;
+ }
+ wc->now_compacting_files = mallocz(sizeof(*wc->now_compacting_files));
+ wc->cleanup_thread_compacting_files = 0;
+ metalog_try_link_new_metadata_logfile(wc);
+
+ error = init_compaction_state(ctx);
+ if (unlikely(error)) {
+ error("Cannot create new metadata log files, compaction aborted.");
+ return;
+ }
+ ++ctx->current_compaction_id; /* Signify a new compaction */
+
+ info("Starting metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
+ error = uv_thread_create(wc->now_compacting_files, compact_old_records, ctx);
+ if (error) {
+ error("uv_thread_create(): %s", uv_strerror(error));
+ freez(wc->now_compacting_files);
+ wc->now_compacting_files = NULL;
+ }
+
+}
+
+/* Return 0 on success. */
+int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
+ unsigned *matched_files)
+{
+ int ret;
+ unsigned starting_fileno, fileno, i, j, recovered_files;
+ struct metadata_logfile *metalogfile, *compactionfile, **tmp_metalogfiles;
+ char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+ for (i = 0 ; i < *matched_files ; ++i) {
+ metalogfile = metalogfiles[i];
+ if (0 == metalogfile->starting_fileno)
+ continue; /* skip standard metadata log files */
+ break; /* this is a compaction temporary file */
+ }
+ if (i == *matched_files) /* no recovery needed */
+ return 0;
+ info("Starting metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+
+ if (*matched_files - i > 1) { /* Can't have more than 1 temporary compaction files */
+ error("Metadata log files are in an invalid state. Cannot proceed.");
+ return 1;
+ }
+ compactionfile = metalogfile;
+ starting_fileno = compactionfile->starting_fileno;
+ fileno = compactionfile->fileno;
+ /* scratchpad space to move file pointers around */
+ tmp_metalogfiles = callocz(*matched_files, sizeof(*tmp_metalogfiles));
+
+ for (j = 0, recovered_files = 0 ; j < i ; ++j) {
+ metalogfile = metalogfiles[j];
+ assert(0 == metalogfile->starting_fileno);
+ if (metalogfile->fileno < starting_fileno) {
+ tmp_metalogfiles[recovered_files++] = metalogfile;
+ continue;
+ }
+ break; /* reached compaction file serial number */
+ }
+
+ if ((j == i) /* Shouldn't be possible, invalid compaction temporary file */ ||
+ (metalogfile->fileno == starting_fileno && metalogfile->fileno == fileno)) {
+ error("Deleting invalid compaction temporary file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
+ METALOG_EXTENSION"\"", dbfiles_path, starting_fileno, fileno);
+ unlink_metadata_logfile(compactionfile);
+ freez(compactionfile);
+ freez(tmp_metalogfiles);
+ --*matched_files; /* delete the last one */
+
+ info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+ return 0;
+ }
+
+ for ( ; j < i ; ++j) { /* continue iterating through normal metadata log files */
+ metalogfile = metalogfiles[j];
+ assert(0 == metalogfile->starting_fileno);
+ if (metalogfile->fileno < fileno) { /* It has already been compacted */
+ error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
+ METALOG_EXTENSION"\"", dbfiles_path, 0U, metalogfile->fileno);
+ unlink_metadata_logfile(metalogfile);
+ freez(metalogfile);
+ continue;
+ }
+ tmp_metalogfiles[recovered_files++] = metalogfile;
+ }
+
+ /* compaction temporary file is valid */
+ tmp_metalogfiles[recovered_files++] = compactionfile;
+ ret = rename_metadata_logfile(compactionfile, 0, starting_fileno);
+ if (ret < 0) {
+ error("Cannot rename temporary compaction files. Cannot proceed.");
+ freez(tmp_metalogfiles);
+ return 1;
+ }
+
+ memcpy(metalogfiles, tmp_metalogfiles, recovered_files * sizeof(*metalogfiles));
+ *matched_files = recovered_files;
+ freez(tmp_metalogfiles);
+
+ info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+ return 0;
+}
diff --git a/database/engine/metadata_log/compaction.h b/database/engine/metadata_log/compaction.h
new file mode 100644
index 0000000000..da4765eeb5
--- /dev/null
+++ b/database/engine/metadata_log/compaction.h
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_COMPACTION_H
+#define NETDATA_COMPACTION_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include "../rrdengine.h"
+
+struct logfile_compaction_state {
+ unsigned fileno; /* Starts at 1 */
+ unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
+
+ struct metadata_record_commit_log records_log;
+ struct metadata_logfile_list new_metadata_logfiles;
+ struct metadata_logfile *last_original_logfile; /* Marks the end of compaction */
+ uint8_t throttle; /* set non-zero to throttle compaction */
+};
+
+extern int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
+ unsigned *matched_files);
+extern void metalog_do_compaction(struct metalog_worker_config *wc);
+extern void after_compact_old_records(struct metalog_worker_config* wc);
+
+#endif /* NETDATA_COMPACTION_H */
diff --git a/database/engine/metadata_log/logfile.c b/database/engine/metadata_log/logfile.c
new file mode 100644
index 0000000000..9ae2866298
--- /dev/null
+++ b/database/engine/metadata_log/logfile.c
@@ -0,0 +1,790 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "metadatalog.h"
+#include "metalogpluginsd.h"
+
+static void mlf_record_block_insert(struct metadata_logfile *metalogfile, struct metalog_record_block *record_block)
+{
+
+ if (likely(NULL != metalogfile->records.last)) {
+ metalogfile->records.last->next = record_block;
+ }
+ if (unlikely(NULL == metalogfile->records.first)) {
+ metalogfile->records.first = record_block;
+ }
+ metalogfile->records.last = record_block;
+}
+
+void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record)
+{
+ struct metalog_record_block *record_block;
+ struct metalog_instance *ctx = metalogfile->ctx;
+
+ record_block = metalogfile->records.last;
+ if (likely(NULL != record_block && record_block->records_nr < MAX_METALOG_RECORDS_PER_BLOCK)) {
+ record_block->record_array[record_block->records_nr++] = *record;
+ } else { /* Create new record block, the last one filled up */
+ record_block = mallocz(sizeof(*record_block));
+ record_block->records_nr = 1;
+ record_block->record_array[0] = *record;
+ record_block->next = NULL;
+
+ mlf_record_block_insert(metalogfile, record_block);
+ }
+ rrd_atomic_fetch_add(&ctx->records_nr, 1);
+}
+
+struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile)
+{
+ struct metalog_records *records = &metalogfile->records;
+ struct metalog_record_block *record_block = metalogfile->records.first;
+
+ records->iterator.current = record_block;
+ records->iterator.record_i = 0;
+
+ if (unlikely(NULL == record_block || !record_block->records_nr)) {
+ error("Cannot iterate empty metadata log file %u-%u.", metalogfile->starting_fileno, metalogfile->fileno);
+ return NULL;
+ }
+
+ return &record_block->record_array[0];
+}
+
+/* Must have called mlf_record_get_first before calling this function. */
+struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile)
+{
+ struct metalog_records *records = &metalogfile->records;
+ struct metalog_record_block *record_block = records->iterator.current;
+
+ if (unlikely(NULL == record_block)) {
+ return NULL;
+ }
+ if (++records->iterator.record_i >= record_block->records_nr) {
+ record_block = record_block->next;
+ if (unlikely(NULL == record_block || !record_block->records_nr)) {
+ return NULL;
+ }
+ records->iterator.current = record_block;
+ records->iterator.record_i = 0;
+ return &record_block->record_array[0];
+ }
+ return &record_block->record_array[records->iterator.record_i];
+}
+
+static void flush_records_buffer_cb(uv_fs_t* req)
+{
+ struct generic_io_descriptor *io_descr = req->data;
+ struct metalog_worker_config *wc = req->loop->data;
+ struct metalog_instance *ctx = wc->ctx;
+
+ debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
+ if (req->result < 0) {
+ ++ctx->stats.io_errors;
+ rrd_stat_atomic_add(&global_io_errors, 1);
+ error("%s: uv_fs_write: %s", __func__, uv_strerror((int)req->result));
+ } else {
+ debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
+ }
+
+ uv_fs_req_cleanup(req);
+ free(io_descr->buf);
+ freez(io_descr);
+}
+
+/* Careful to always call this before creating a new metadata log file to finish writing the old one */
+void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+ struct metadata_logfile_list *metadata_logfiles)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ int ret;
+ struct generic_io_descriptor *io_descr;
+ unsigned pos, size;
+ struct metadata_logfile *metalogfile;
+
+ if (unlikely(NULL == records_log->buf || 0 == records_log->buf_pos)) {
+ return;
+ }
+ /* care with outstanding records when switching metadata log files */
+ metalogfile = metadata_logfiles->last;
+
+ io_descr = mallocz(sizeof(*io_descr));
+ pos = records_log->buf_pos;
+ size = pos; /* no need to align the I/O when doing buffered writes */
+ io_descr->buf = records_log->buf;
+ io_descr->bytes = size;
+ io_descr->pos = metalogfile->pos;
+ io_descr->req.data = io_descr;
+ io_descr->completion = NULL;
+
+ io_descr->iov = uv_buf_init((void *)io_descr->buf, size);
+ ret = uv_fs_write(wc->loop, &io_descr->req, metalogfile->file, &io_descr->iov, 1,
+ metalogfile->pos, flush_records_buffer_cb);
+ assert (-1 != ret);
+ metalogfile->pos += size;
+ rrd_atomic_fetch_add(&ctx->disk_space, size);
+ records_log->buf = NULL;
+ ctx->stats.io_write_bytes += size;
+ ++ctx->stats.io_write_requests;
+}
+
+void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+ struct metadata_logfile_list *metadata_logfiles, unsigned size)
+{
+ int ret;
+ unsigned buf_pos, buf_size;
+
+ assert(size);
+ if (records_log->buf) {
+ unsigned remaining;
+
+ buf_pos = records_log->buf_pos;
+ buf_size = records_log->buf_size;
+ remaining = buf_size - buf_pos;
+ if (size > remaining) {
+ /* we need a new buffer */
+ mlf_flush_records_buffer(wc, records_log, metadata_logfiles);
+ }
+ }
+ if (NULL == records_log->buf) {
+ buf_size = ALIGN_BYTES_CEILING(size);
+ ret = posix_memalign((void *)&records_log->buf, RRDFILE_ALIGNMENT, buf_size);
+ if (unlikely(ret)) {
+ fatal("posix_memalign:%s", strerror(ret));
+ }
+ buf_pos = records_log->buf_pos = 0;
+ records_log->buf_size = buf_size;
+ }
+ records_log->buf_pos += size;
+
+ return records_log->buf + buf_pos;
+}
+
+
+void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
+{
+ if (likely(NULL != metadata_logfiles->last)) {
+ metadata_logfiles->last->next = metalogfile;
+ }
+ if (unlikely(NULL == metadata_logfiles->first)) {
+ metadata_logfiles->first = metalogfile;
+ }
+ metadata_logfiles->last = metalogfile;
+}
+
+void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
+{
+ struct metadata_logfile *next;
+
+ next = metalogfile->next;
+ assert((NULL != next) && (metadata_logfiles->first == metalogfile) &&
+ (metadata_logfiles->last != metalogfile));
+ metadata_logfiles->first = next;
+}
+
+void generate_metadata_logfile_path(struct metadata_logfile *metalogfile, char *str, size_t maxlen)
+{
+ (void) snprintf(str, maxlen, "%s/" METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION,
+ metalogfile->ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+}
+
+void metadata_logfile_init(struct metadata_logfile *metalogfile, struct metalog_instance *ctx, unsigned starting_fileno,
+ unsigned fileno)
+{
+ metalogfile->starting_fileno = starting_fileno;
+ metalogfile->fileno = fileno;
+ metalogfile->file = (uv_file)0;
+ metalogfile->pos = 0;
+ metalogfile->records.first = metalogfile->records.last = NULL;
+ metalogfile->next = NULL;
+ metalogfile->ctx = ctx;
+}
+
+int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno, unsigned new_fileno)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ uv_fs_t req;
+ int ret;
+ char oldpath[RRDENG_PATH_MAX], newpath[RRDENG_PATH_MAX];
+ unsigned backup_starting_fileno, backup_fileno;
+
+ backup_starting_fileno = metalogfile->starting_fileno;
+ backup_fileno = metalogfile->fileno;
+ generate_metadata_logfile_path(metalogfile, oldpath, sizeof(oldpath));
+ metalogfile->starting_fileno = new_starting_fileno;
+ metalogfile->fileno = new_fileno;
+ generate_metadata_logfile_path(metalogfile, newpath, sizeof(newpath));
+
+ info("Renaming metadata log file \"%s\" to \"%s\".", oldpath, newpath);
+ ret = uv_fs_rename(NULL, &req, oldpath, newpath, NULL);
+ if (ret < 0) {
+ error("uv_fs_rename(%s): %s", oldpath, uv_strerror(ret));
+ ++ctx->stats.fs_errors; /* this is racy, may miss some errors */
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ /* restore previous values */
+ metalogfile->starting_fileno = backup_starting_fileno;
+ metalogfile->fileno = backup_fileno;
+ }
+ uv_fs_req_cleanup(&req);
+
+ return ret;
+}
+
+int close_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ uv_fs_t req;
+ int ret;
+ char path[RRDENG_PATH_MAX];
+
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+ ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
+ if (ret < 0) {
+ error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+
+ return ret;
+}
+
+int unlink_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ uv_fs_t req;
+ int ret;
+ char path[RRDENG_PATH_MAX];
+
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+ ret = uv_fs_unlink(NULL, &req, path, NULL);
+ if (ret < 0) {
+ error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+
+ return ret;
+}
+
+int destroy_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ uv_fs_t req;
+ int ret;
+ char path[RRDENG_PATH_MAX];
+
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+ ret = uv_fs_ftruncate(NULL, &req, metalogfile->file, 0, NULL);
+ if (ret < 0) {
+ error("uv_fs_ftruncate(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+
+ ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
+ if (ret < 0) {
+ error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+
+ ret = uv_fs_unlink(NULL, &req, path, NULL);
+ if (ret < 0) {
+ error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+
+// ++ctx->stats.metadata_logfile_deletions;
+
+ return ret;
+}
+
+int create_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ uv_fs_t req;
+ uv_file file;
+ int ret, fd;
+ struct rrdeng_metalog_sb *superblock;
+ uv_buf_t iov;
+ char path[RRDENG_PATH_MAX];
+
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+ fd = open_file_buffered_io(path, O_CREAT | O_RDWR | O_TRUNC, &file);
+ if (fd < 0) {
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ return fd;
+ }
+ metalogfile->file = file;
+// ++ctx->stats.metadata_logfile_creations;
+
+ ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
+ if (unlikely(ret)) {
+ fatal("posix_memalign:%s", strerror(ret));
+ }
+ (void) strncpy(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ);
+ superblock->version = RRDENG_METALOG_VER;
+
+ iov = uv_buf_init((void *)superblock, sizeof(*superblock));
+
+ ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);
+ if (ret < 0) {
+ assert(req.result < 0);
+ error("uv_fs_write: %s", uv_strerror(ret));
+ ++ctx->stats.io_errors;
+ rrd_stat_atomic_add(&global_io_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+ free(superblock);
+ if (ret < 0) {
+ destroy_metadata_logfile(metalogfile);
+ return ret;
+ }
+
+ metalogfile->pos = sizeof(*superblock);
+ ctx->stats.io_write_bytes += sizeof(*superblock);
+ ++ctx->stats.io_write_requests;
+
+ return 0;
+}
+
+static int check_metadata_logfile_superblock(uv_file file)
+{
+ int ret;
+ struct rrdeng_metalog_sb *superblock;
+ uv_buf_t iov;
+ uv_fs_t req;
+
+ ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
+ if (unlikely(ret)) {
+ fatal("posix_memalign:%s", strerror(ret));
+ }
+ iov = uv_buf_init((void *)superblock, sizeof(*superblock));
+
+ ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);
+ if (ret < 0) {
+ error("uv_fs_read: %s", uv_strerror(ret));
+ uv_fs_req_cleanup(&req);
+ goto error;
+ }
+ assert(req.result >= 0);
+ uv_fs_req_cleanup(&req);
+
+ if (strncmp(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ)) {
+ error("File has invalid superblock.");
+ ret = UV_EINVAL;
+ } else {
+ ret = 0;
+ }
+ if (superblock->version > RRDENG_METALOG_VER) {
+ error("File has unknown version %"PRIu16". Compatibility is not guaranteed.", superblock->version);
+ }
+error:
+ free(superblock);
+ return ret;
+}
+
+void replay_record(struct metadata_logfile *metalogfile, struct rrdeng_metalog_record_header *header, void *payload)
+{
+ struct metalog_instance *ctx = metalogfile->ctx;
+ char *line, *nextline, *record_end;
+ int ret;
+
+ debug(D_METADATALOG, "RECORD contents: %.*s", (int)header->payload_length, (char *)payload);
+ record_end = (char *)payload + header->payload_length - 1;
+ *record_end = '\0';
+
+ for (line = payload ; line ; line = nextline) {
+ nextline = strchr(line, '\n');
+ if (nextline) {
+ *nextline++ = '\0';
+ }
+ ret = parser_action(ctx->metalog_parser_object->parser, line);
+ debug(D_METADATALOG, "parser_action ret:%d", ret);
+ if (ret)
+ return; /* skip record due to error */
+ };
+}
+
+/* This function only works with buffered I/O */
+static inline int metalogfile_read(struct metadata_logfile *metalogfile, void *buf, size_t len, uint64_t offset)
+{
+ struct metalog_instance *ctx;
+ uv_file file;
+ uv_buf_t iov;
+ uv_fs_t req;
+ int ret;
+
+ ctx = metalogfile->ctx;
+ file = metalogfile->file;
+ iov = uv_buf_init(buf, len);
+ ret = uv_fs_read(NULL, &req, file, &iov, 1, offset, NULL);
+ if (unlikely(ret < 0 && ret != req.result)) {
+ fatal("uv_fs_read: %s", uv_strerror(ret));
+ }
+ if (req.result < 0) {
+ ++ctx->stats.io_errors;
+ rrd_stat_atomic_add(&global_io_errors, 1);
+ error("%s: uv_fs_read - %s - record at offset %"PRIu64"(%u) in metadata logfile %u-%u.", __func__,
+ uv_strerror((int)req.result), offset, (unsigned)len, metalogfile->starting_fileno, metalogfile->fileno);
+ }
+ uv_fs_req_cleanup(&req);
+ ctx->stats.io_read_bytes += len;
+ ++ctx->stats.io_read_requests;
+
+ return ret;
+}
+
+/* Return 0 on success */
+static int metadata_record_integrity_check(void *record)
+{
+ int ret;
+ uint32_t data_size;
+ struct rrdeng_metalog_record_header *header;
+ struct rrdeng_metalog_record_trailer *trailer;
+ uLong crc;
+
+ header = record;
+ data_size = header->header_length + header->payload_length;
+ trailer = record + data_size;
+
+ crc = crc32(0L, Z_NULL, 0);
+ crc = crc32(crc, record, data_size);
+ ret = crc32cmp(trailer->checksum, crc);
+
+ return ret;
+}
+
+#define MAX_READ_BYTES (RRDENG_BLOCK_SIZE * 32) /* no record should be over 128KiB in this version */
+
+/*
+ * Iterates metadata log file records and creates database objects (host/chart/dimension)
+ */
+static void iterate_records(struct metadata_logfile *metalogfile)
+{
+ uint32_t file_size, pos, bytes_remaining, record_size;
+ void *buf;
+ struct rrdeng_metalog_record_header *header;
+ struct metalog_instance *ctx = metalogfile->ctx;
+ struct metalog_pluginsd_state *state = ctx->metalog_parser_object->private;
+ const size_t min_header_size = offsetof(struct rrdeng_metalog_record_header, header_length) +
+ sizeof(header->header_length);
+
+ file_size = metalogfile->pos;
+ state->metalogfile = metalogfile;
+
+ buf = mallocz(MAX_READ_BYTES);
+
+ for (pos = sizeof(struct rrdeng_metalog_sb) ; pos < file_size ; pos += record_size) {
+ bytes_remaining = file_size - pos;
+ if (bytes_remaining < min_header_size) {
+ error("%s: unexpected end of file in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+ metalogfile->fileno);
+ break;
+ }
+ if (metalogfile_read(metalogfile, buf, min_header_size, pos) < 0)
+ break;
+ header = (struct rrdeng_metalog_record_header *)buf;
+ if (METALOG_STORE_PADDING == header->type) {
+ info("%s: Skipping padding in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+ metalogfile->fileno);
+ record_size = ALIGN_BYTES_FLOOR(pos + RRDENG_BLOCK_SIZE) - pos;
+ continue;
+ }
+ if (metalogfile_read(metalogfile, buf + min_header_size, sizeof(*header) - min_header_size,
+ pos + min_header_size) < 0)
+ break;
+ record_size = header->header_length + header->payload_length + sizeof(struct rrdeng_metalog_record_trailer);
+ if (header->header_length < min_header_size || record_size > bytes_remaining) {
+ error("%s: Corrupted record in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+ metalogfile->fileno);
+ break;
+ }
+ if (record_size > MAX_READ_BYTES) {
+ error("%s: Record is too long (%u bytes) in metadata logfile %u-%u.", __func__, record_size,
+ metalogfile->starting_fileno, metalogfile->fileno);
+ continue;
+ }
+ if (metalogfile_read(metalogfile, buf + sizeof(*header), record_size - sizeof(*header),
+ pos + sizeof(*header)) < 0)
+ break;
+ if (metadata_record_integrity_check(buf)) {
+ error("%s: Record at offset %"PRIu32" was read from disk. CRC32 check: FAILED", __func__, pos);
+ continue;
+ }
+ debug(D_METADATALOG, "%s: Record at offset %"PRIu32" was read from disk. CRC32 check: SUCCEEDED", __func__,
+ pos);
+
+ replay_record(metalogfile, header, buf + header->header_length);
+ if (!uuid_is_null(state->uuid)) { /* It's a valid object */
+ struct metalog_record record;
+
+ uuid_copy(record.uuid, state->uuid);
+ mlf_record_insert(metalogfile, &record);
+ uuid_clear(state->uuid); /* Clear state for parsing of next record */
+ }
+ }
+
+ freez(buf);
+}
+
+int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
+{
+ uv_fs_t req;
+ uv_file file;
+ int ret, fd, error;
+ uint64_t file_size;
+ char path[RRDENG_PATH_MAX];
+
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+ fd = open_file_buffered_io(path, O_RDWR, &file);
+ if (fd < 0) {
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ return fd;
+ }
+ info("Loading metadata log \"%s\".", path);
+
+ ret = check_file_properties(file, &file_size, sizeof(struct rrdeng_metalog_sb));
+ if (ret)
+ goto error;
+
+ ret = check_metadata_logfile_superblock(file);
+ if (ret)
+ goto error;
+ ctx->stats.io_read_bytes += sizeof(struct rrdeng_jf_sb);
+ ++ctx->stats.io_read_requests;
+
+ metalogfile->file = file;
+ metalogfile->pos = file_size;
+
+ iterate_records(metalogfile);
+
+ info("Metadata log \"%s\" loaded (size:%"PRIu64").", path, file_size);
+ return 0;
+
+error:
+ error = ret;
+ ret = uv_fs_close(NULL, &req, file, NULL);
+ if (ret < 0) {
+ error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ }
+ uv_fs_req_cleanup(&req);
+ return error;
+}
+
+void init_metadata_record_log(struct metadata_record_commit_log *records_log)
+{
+ records_log->buf = NULL;
+ records_log->buf_pos = 0;
+ records_log->record_id = 1;
+}
+
+static int scan_metalog_files_cmp(const void *a, const void *b)
+{
+ struct metadata_logfile *file1, *file2;
+ char path1[RRDENG_PATH_MAX], path2[RRDENG_PATH_MAX];
+
+ file1 = *(struct metadata_logfile **)a;
+ file2 = *(struct metadata_logfile **)b;
+ generate_metadata_logfile_path(file1, path1, sizeof(path1));
+ generate_metadata_logfile_path(file2, path2, sizeof(path2));
+ return strcmp(path1, path2);
+}
+
+/* Returns number of metadata logfiles that were loaded or < 0 on error */
+static int scan_metalog_files(struct metalog_instance *ctx)
+{
+ int ret;
+ unsigned starting_no, no, matched_files, i, failed_to_load;
+ static uv_fs_t req;
+ uv_dirent_t dent;
+ struct metadata_logfile **metalogfiles, *metalogfile;
+ char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+ ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
+ if (ret < 0) {
+ assert(req.result < 0);
+ uv_fs_req_cleanup(&req);
+ error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
+ ++ctx->stats.fs_errors;
+ rrd_stat_atomic_add(&global_fs_errors, 1);
+ return ret;
+ }
+ info("Found %d files in path %s", ret, dbfiles_path);
+
+ metalogfiles = callocz(MIN(ret, MAX_DATAFILES), sizeof(*metalogfiles));
+ for (matched_files = 0 ; UV_EOF != uv_fs_scandir_next(&req, &dent) && matched_files < MAX_DATAFILES ; ) {
+ info("Scanning file \"%s/%s\"", dbfiles_path, dent.name);
+ ret = sscanf(dent.name, METALOG_PREFIX METALOG_FILE_NUMBER_SCAN_TMPL METALOG_EXTENSION, &starting_no, &no);
+ if (2 == ret) {
+ info("Matched file \"%s/%s\"", dbfiles_path, dent.name);
+ metalogfile = mallocz(sizeof(*metalogfile));
+ metadata_logfile_init(metalogfile, ctx, starting_no, no);
+ metalogfiles[matched_files++] = metalogfile;
+ }
+ }
+ uv_fs_req_cleanup(&req);
+
+ if (0 == matched_files) {
+ freez(metalogfiles);
+ return 0;
+ }
+ if (matched_files == MAX_DATAFILES) {
+ error("Warning: hit maximum database engine file limit of %d files", MAX_DATAFILES);
+ }
+ qsort(metalogfiles, matched_files, sizeof(*metalogfiles), scan_metalog_files_cmp);
+ ret = compaction_failure_recovery(ctx, metalogfiles, &matched_files);
+ if (ret) { /* If the files are corrupted fail */
+ for (i = 0 ; i < matched_files ; ++i) {
+ freez(metalogfiles[i]);
+ }
+ freez(metalogfiles);
+ return UV_EINVAL;
+ }
+ ctx->last_fileno = metalogfiles[matched_files - 1]->fileno;
+
+ struct plugind cd = {
+ .enabled = 1,
+ .update_every = 0,
+ .pid = 0,
+ .serial_failures = 0,
+ .successful_collections = 0,
+ .obsolete = 0,
+ .started_t = INVALID_TIME,
+ .next = NULL,
+ .version = 0,
+ };
+
+ struct metalog_pluginsd_state metalog_parser_state;
+ metalog_pluginsd_state_init(&metalog_parser_state, ctx);
+
+ PARSER_USER_OBJECT metalog_parser_object;
+ metalog_parser_object.enabled = cd.enabled;
+ metalog_parser_object.host = ctx->rrdeng_ctx->host;
+ metalog_parser_object.cd = &cd;
+ metalog_parser_object.trust_durations = 0;
+ metalog_parser_object.private = &metalog_parser_state;
+
+ PARSER *parser = parser_init(metalog_parser_object.host, &metalog_parser_object, NULL, PARSER_INPUT_SPLIT);
+ if (unlikely(!parser)) {
+ error("Failed to initialize metadata log parser.");
+ failed_to_load = matched_files;
+ goto after_failed_to_parse;
+ }
+ parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
+ parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
+ parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
+ parser->plugins_action->dimension_action = &metalog_pluginsd_dimension_action;
+ parser->plugins_action->chart_action = &metalog_pluginsd_chart_action;
+ parser->plugins_action->guid_action = &metalog_pluginsd_guid_action;
+ parser->plugins_action->context_action = &metalog_pluginsd_context_action;
+ parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
+
+ metalog_parser_object.parser = parser;
+ ctx->metalog_parser_object = &metalog_parser_object;
+
+ for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
+ metalogfile = metalogfiles[i];
+ ret = load_metadata_logfile(ctx, metalogfile);
+ if (0 != ret) {
+ freez(metalogfile);
+ ++failed_to_load;
+ break;
+ }
+ metadata_logfile_list_insert(&ctx->metadata_logfiles, metalogfile);
+ rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
+ }
+ debug(D_METADATALOG, "PARSER ended");
+
+ parser_destroy(parser);
+
+ size_t count = metalog_parser_object.count;
+
+ debug(D_METADATALOG, "Parsing count=%u", (unsigned)count);
+after_failed_to_parse:
+
+ freez(metalogfiles);
+ if (failed_to_load) {
+ error("%u metadata log files failed to load.", failed_to_load);
+ finalize_metalog_files(ctx);
+ return UV_EIO;
+ }
+
+ return matched_files;
+}
+
+/* Creates a metadata log file */
+int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
+ unsigned starting_fileno, unsigned fileno)
+{
+ struct metadata_logfile *metalogfile;
+ int ret;
+ char path[RRDENG_PATH_MAX];
+
+ info("Creating new metadata log file in path %s", ctx->rrdeng_ctx->dbfiles_path);
+ metalogfile = mallocz(sizeof(*metalogfile));
+ metadata_logfile_init(metalogfile, ctx, starting_fileno, fileno);
+ ret = create_metadata_logfile(metalogfile);
+ if (!ret) {
+ generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+ info("Created metadata log file \"%s\".", path);
+ } else {
+ freez(metalogfile);
+ return ret;
+ }
+ metadata_logfile_list_insert(logfile_list, metalogfile);
+ rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
+
+ return 0;
+}
+
+/* Return 0 on success. */
+int init_metalog_files(struct metalog_instance *ctx)
+{
+ int ret;
+ char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+ ret = scan_metalog_files(ctx);
+ if (ret < 0) {
+ error("Failed to scan path \"%s\".", dbfiles_path);
+ return ret;
+ } else if (0 == ret) {
+ info("Metadata log files not found, creating in path \"%s\".", dbfiles_path);
+ ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, 1);
+ if (ret) {
+ error("Failed to create metadata log file in path \"%s\".", dbfiles_path);
+ return ret;
+ }
+ ctx->last_fileno = 1;
+ }
+
+ return 0;
+}
+
+void finalize_metalog_files(struct metalog_instance *ctx)
+{
+ struct metadata_logfile *metalogfile, *next_metalogfile;
+ struct metalog_record_block *record_block, *next_record_block;
+
+ for (metalogfile = ctx->metadata_logfiles.first ; metalogfile != NULL ; metalogfile = next_metalogfile) {
+ next_metalogfile = metalogfile->next;
+
+ for (record_block = metalogfile->records.first ; record_block != NULL ; record_block = next_record_block) {
+ next_record_block = record_block->next;
+ freez(record_block);
+ }
+ close_metadata_logfile(metalogfile);
+ freez(metalogfile);
+ }
+}
diff --git a/database/engine/metadata_log/logfile.h b/database/engine/metadata_log/logfile.h
new file mode 100644
index 0000000000..d4e26da883
--- /dev/null
+++ b/database/engine/metadata_log/logfile.h
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOGFILE_H
+#define NETDATA_LOGFILE_H
+
+#include "metadatalogprotocol.h"
+#include "../rrdengine.h"
+
+/* Forward declarations */
+struct metadata_logfile;
+struct metalog_worker_config;
+
+#define METALOG_PREFIX "metadatalog-"
+#define METALOG_EXTENSION ".mlf"
+
+#define MAX_METALOGFILE_SIZE (524288LU)
+
+/* Deletions are ignored during compaction, so only creation UUIDs are stored */
+struct metalog_record {
+ uuid_t uuid;
+};
+
+#define MAX_METALOG_RECORDS_PER_BLOCK (1024LU)
+struct metalog_record_block {
+ uint64_t file_offset;
+ uint32_t io_size;
+
+ struct metalog_record record_array[MAX_METALOG_RECORDS_PER_BLOCK];
+ uint16_t records_nr;
+
+ struct metalog_record_block *next;
+};
+
+struct metalog_records {
+ /* the record block list is sorted based on disk offset */
+ struct metalog_record_block *first;
+ struct metalog_record_block *last;
+ struct {
+ struct metalog_record_block *current;
+ uint16_t record_i;
+ } iterator;
+};
+
+/* only one event loop is supported for now */
+struct metadata_logfile {
+ unsigned fileno; /* Starts at 1 */
+ unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
+ uv_file file;
+ uint64_t pos;
+ struct metalog_instance *ctx;
+ struct metalog_records records;
+ struct metadata_logfile *next;
+};
+
+struct metadata_logfile_list {
+ struct metadata_logfile *first; /* oldest */
+ struct metadata_logfile *last; /* newest */
+};
+
+struct metadata_record_commit_log {
+ uint64_t record_id;
+
+ /* outstanding record buffer */
+ void *buf;
+ unsigned buf_pos;
+ unsigned buf_size;
+};
+
+extern void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record);
+extern struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile);
+extern struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile);
+extern void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+ struct metadata_logfile_list *metadata_logfiles);
+extern void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+ struct metadata_logfile_list *metadata_logfiles, unsigned size);
+extern void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles,
+ struct metadata_logfile *metalogfile);
+extern void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles,
+ struct metadata_logfile *metalogfile);
+extern void generate_metadata_logfile_path(struct metadata_logfile *metadatalog, char *str, size_t maxlen);
+extern void metadata_logfile_init(struct metadata_logfile *metadatalog, struct metalog_instance *ctx,
+ unsigned tier, unsigned fileno);
+extern int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno,
+ unsigned new_fileno);
+extern int close_metadata_logfile(struct metadata_logfile *metadatalog);
+extern int unlink_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int destroy_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int create_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *logfile);
+extern void init_metadata_record_log(struct metadata_record_commit_log *records_log);
+extern int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
+ unsigned tier, unsigned fileno);
+extern int init_metalog_files(struct metalog_instance *ctx);
+extern void finalize_metalog_files(struct metalog_instance *ctx);
+
+
+#endif /* NETDATA_LOGFILE_H */
diff --git a/database/engine/metadata_log/metadatalog.c b/database/engine/metadata_log/metadatalog.c
new file mode 100644
index 0000000000..b436cb105a
--- /dev/null
+++ b/database/engine/metadata_log/metadatalog.c
@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+static void sanity_check(void)
+{
+ /* Magic numbers must fit in the super-blocks */
+ BUILD_BUG_ON(strlen(RRDENG_METALOG_MAGIC) > RRDENG_MAGIC_SZ);
+
+ /* Metadata log file super-block cannot be larger than RRDENG_BLOCK_SIZE */
+ BUILD_BUG_ON(RRDENG_METALOG_SB_PADDING_SZ < 0);
+
+ /* Object duplication factor cannot be less than 1, or too close to 1 */
+ BUILD_BUG_ON(MAX_DUPLICATION_PERCENTAGE < 110);
+}
+
+char *get_metalog_statistics(struct metalog_instance *ctx, char *str, size_t size)
+{
+ snprintfz(str, size,
+ "io_write_bytes: %ld\n"
+ "io_write_requests: %ld\n"
+ "io_read_bytes: %ld\n"
+ "io_read_requests: %ld\n"
+ "io_write_record_bytes: %ld\n"
+ "io_write_records: %ld\n"
+ "io_read_record_bytes: %ld\n"
+ "io_read_records: %ld\n"
+ "metadata_logfile_creations: %ld\n"
+ "metadata_logfile_deletions: %ld\n"
+ "io_errors: %ld\n"
+ "fs_errors: %ld\n",
+ (long)ctx->stats.io_write_bytes,
+ (long)ctx->stats.io_write_requests,
+ (long)ctx->stats.io_read_bytes,
+ (long)ctx->stats.io_read_requests,
+ (long)ctx->stats.io_write_record_bytes,
+ (long)ctx->stats.io_write_records,
+ (long)ctx->stats.io_read_record_bytes,
+ (long)ctx->stats.io_read_records,
+ (long)ctx->stats.metadata_logfile_creations,
+ (long)ctx->stats.metadata_logfile_deletions,
+ (long)ctx->stats.io_errors,
+ (long)ctx->stats.fs_errors
+ );
+ return str;
+}
+
+/* The buffer must not be empty */
+void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode, uuid_t *uuid,
+ int compacting)
+{
+ struct metalog_cmd cmd;
+
+ assert(buffer_strlen(buffer));
+ assert(opcode == METALOG_COMMIT_CREATION_RECORD || opcode == METALOG_COMMIT_DELETION_RECORD);
+
+ cmd.opcode = opcode;
+ cmd.record_io_descr.buffer = buffer;
+ cmd.record_io_descr.compacting = compacting;
+ if (!uuid)
+ uuid_clear(cmd.record_io_descr.uuid);
+ else
+ uuid_copy(cmd.record_io_descr.uuid, *uuid);
+ metalog_enq_cmd(&ctx->worker_config, &cmd);
+}
+
+static void commit_record(struct metalog_worker_config* wc, struct metalog_record_io_descr *io_descr, uint8_t type)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ unsigned payload_length, size_bytes;
+ void *buf, *mlf_payload;
+ /* persistent structures */
+ struct rrdeng_metalog_record_header *mlf_header;
+ struct rrdeng_metalog_record_trailer *mlf_trailer;
+ uLong crc;
+
+ payload_length = buffer_strlen(io_descr->buffer);
+ size_bytes = sizeof(*mlf_header) + payload_length + sizeof(*mlf_trailer);
+
+ if (io_descr->compacting)
+ buf = mlf_get_records_buffer(wc, &ctx->compaction_state.records_log,
+ &ctx->compaction_state.new_metadata_logfiles, size_bytes);
+ else
+ buf = mlf_get_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles, size_bytes);
+
+ mlf_header = buf;
+ mlf_header->type = type;
+ mlf_header->header_length = sizeof(*mlf_header);
+ mlf_header->payload_length = payload_length;
+
+ mlf_payload = buf + sizeof(*mlf_header);
+ memcpy(mlf_payload, buffer_tostring(io_descr->buffer), payload_length);
+
+ mlf_trailer = buf + sizeof(*mlf_header) + payload_length;
+ crc = crc32(0L, Z_NULL, 0);
+ crc = crc32(crc, buf, sizeof(*mlf_header) + payload_length);
+ crc32set(mlf_trailer->checksum, crc);
+
+ buffer_free(io_descr->buffer);
+}
+
+static void do_commit_record(struct metalog_worker_config* wc, uint8_t type, void *data)
+{
+ struct metalog_record_io_descr *io_descr = (struct metalog_record_io_descr *)data;
+ switch (type) {
+ case METALOG_CREATE_OBJECT:
+ if (!uuid_is_null(io_descr->uuid)) { /* It's a valid object */
+ struct metalog_record record;
+
+ uuid_copy(record.uuid, io_descr->uuid);
+ if (io_descr->compacting)
+ mlf_record_insert(wc->ctx->compaction_state.new_metadata_logfiles.last, &record);
+ else
+ mlf_record_insert(wc->ctx->metadata_logfiles.last, &record);
+ } /* fall through */
+ case METALOG_DELETE_OBJECT:
+ commit_record(wc, (struct metalog_record_io_descr *)data, type);
+ break;
+ default:
+ fatal("Unknown metadata log file record type, possible memory corruption.");
+ break;
+ }
+}
+
+/* Only creates a new metadata file and links it to the metadata log if the last one is non empty. */
+void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ struct metadata_logfile *metalogfile;
+ int ret;
+
+ metalogfile = ctx->metadata_logfiles.last;
+ if (metalogfile->records.first) { /* it has records */
+ /* Finalize metadata log file and create a new one */
+ mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+ ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, ctx->last_fileno + 1);
+ if (likely(!ret)) {
+ ++ctx->last_fileno;
+ }
+ }
+}
+
+void metalog_test_quota(struct metalog_worker_config *wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+ struct metadata_logfile *metalogfile;
+ unsigned current_size;
+ uint8_t only_one_metalogfile;
+
+ metalogfile = ctx->metadata_logfiles.last;
+ current_size = metalogfile->pos;
+ if (unlikely(current_size >= MAX_METALOGFILE_SIZE)) {
+ metalog_try_link_new_metadata_logfile(wc);
+ }
+
+ metalogfile = ctx->metadata_logfiles.last;
+ only_one_metalogfile = (metalogfile == ctx->metadata_logfiles.first) ? 1 : 0;
+ debug(D_METADATALOG, "records=%lu objects=%lu", (long unsigned)ctx->records_nr,
+ (long unsigned)ctx->rrdeng_ctx->host->objects_nr);
+ if (unlikely(!only_one_metalogfile &&
+ ctx->records_nr > (ctx->rrdeng_ctx->host->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&
+ NO_QUIESCE == ctx->quiesce) {
+ metalog_do_compaction(wc);
+ }
+}
+
+static inline int metalog_threads_alive(struct metalog_worker_config* wc)
+{
+ if (wc->cleanup_thread_compacting_files) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static void metalog_cleanup_finished_threads(struct metalog_worker_config *wc)
+{
+ struct metalog_instance *ctx = wc->ctx;
+
+ if (unlikely(wc->cleanup_thread_compacting_files)) {
+ after_compact_old_records(wc);
+ }
+ if (unlikely(SET_QUIESCE == ctx->quiesce && !metalog_threads_alive(wc))) {
+ ctx->quiesce = QUIESCED;
+ complete(&ctx->metalog_completion);
+ }
+}
+
+static void metalog_init_cmd_queue(struct metalog_worker_config *wc)
+{
+ wc->cmd_queue.head = wc->cmd_queue.tail = 0;
+ wc->queue_size = 0;
+ assert(0 == uv_cond_init(&wc->cmd_cond));
+ assert(0 == uv_mutex_init(&wc->cmd_mutex));
+}
+
+void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd)
+{
+ unsigned queue_size;
+
+ /* wait for free space in queue */
+ uv_mutex_lock(&wc->cmd_mutex);
+ while ((queue_size = wc->queue_size) == METALOG_CMD_Q_MAX_SIZE) {
+ uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
+ }
+ assert(queue_size < METALOG_CMD_Q_MAX_SIZE);
+ /* enqueue command */
+ wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
+ wc->cmd_queue.tail = wc->cmd_queue.tail != METALOG_CMD_Q_MAX_SIZE - 1 ?
+ wc->cmd_queue.tail + 1 : 0;
+ wc->queue_size = queue_size + 1;
+ uv_mutex_unlock(&wc->cmd_mutex);
+
+ /* wake up event loop */
+ assert(0 == uv_async_send(&wc->async));
+}
+
+struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc)
+{
+ struct metalog_cmd ret;
+ unsigned queue_size;
+
+ uv_mutex_lock(&wc->cmd_mutex);
+ queue_size = wc->queue_size;
+ if (queue_size == 0) {
+ ret.opcode = METALOG_NOOP;
+ } else {
+ /* dequeue command */
+ ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
+ if (queue_size == 1) {
+ wc->cmd_queue.head = wc->cmd_queue.tail = 0;
+ } else {
+ wc->cmd_queue.head = wc->cmd_queue.head != RRDENG_CMD_Q_MAX_SIZE - 1 ?
+ wc->cmd_queue.head + 1 : 0;
+ }
+ wc->queue_size = queue_size - 1;
+
+ /* wake up producers */
+ uv_cond_signal(&wc->cmd_cond);
+ }
+ uv_mutex_unlock(&wc->cmd_mutex);
+
+ return ret;
+}
+
+static void async_cb(uv_async_t *handle)
+{
+ uv_stop(handle->loop);
+ uv_update_time(handle->loop);
+ debug(D_METADATALOG, "%s called, active=%d.", __func__, uv_is_active((uv_handle_t *)handle));
+}
+
+/* Flushes metadata log when timer expires */
+#define TIMER_PERIOD_MS (5000)
+
+static void timer_cb(uv_timer_t* handle)
+{
+ struct metalog_worker_config* wc = handle->data;
+ struct metalog_instance *ctx = wc->ctx;
+
+ uv_stop(handle->loop);
+ uv_update_time(handle->loop);
+ metalog_test_quota(wc);
+ debug(D_METADATALOG, "%s: timeout reached.", __func__);
+#ifdef NETDATA_INTERNAL_CHECKS
+ {
+ char buf[4096];
+ debug(D_METADATALOG, "%s", get_metalog_statistics(wc->ctx, buf, sizeof(buf)));
+ }
+#endif
+ mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+}
+
+#define MAX_CMD_BATCH_SIZE (256)
+
+void metalog_worker(void* arg)
+{
+ struct metalog_worker_config *wc = arg;
+ struct metalog_instance *ctx = wc->ctx;
+ uv_loop_t* loop;
+ int shutdown, ret;
+ enum metalog_opcode opcode;
+ uv_timer_t timer_req;
+ struct metalog_cmd cmd;
+ unsigned cmd_batch_size;
+
+ metalog_init_cmd_queue(wc);
+
+ loop = wc->loop = mallocz(sizeof(uv_loop_t));
+ ret = uv_loop_init(loop);
+ if (ret) {
+ error("uv_loop_init(): %s", uv_strerror(ret));
+ goto error_after_loop_init;
+ }
+ loop->data = wc;
+
+ ret = uv_async_init(wc->loop, &wc->async, async_cb);
+ if (ret) {
+ error("uv_async_init(): %s", uv_strerror(ret));
+ goto error_after_async_init;
+ }
+ wc->async.data = wc;
+
+ wc->now_compacting_files = NULL;
+ wc->cleanup_thread_compacting_files = 0;
+
+ /* quota check timer */
+ ret = uv_timer_init(loop, &timer_req);
+ if (ret) {
+ error("uv_timer_init(): %s", uv_strerror(ret));
+ goto error_after_timer_init;
+ }
+ timer_req.data = wc;
+
+ wc->error = 0;
+ /* wake up initialization thread */
+ complete(&ctx->metalog_completion);
+
+ assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
+ shutdown = 0;
+ while (likely(shutdown == 0 || metalog_threads_alive(wc))) {
+ uv_run(loop, UV_RUN_DEFAULT);
+ metalog_cleanup_finished_threads(wc);
+
+ /* wait for commands */
+ cmd_batch_size = 0;
+ do {
+ /*
+ * Avoid starving the loop when there are too many commands coming in.
+ * timer_cb will interrupt the loop again to allow serving more commands.
+ */
+ if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))
+ break;
+
+ cmd = metalog_deq_cmd(wc);
+ opcode = cmd.opcode;
+ ++cmd_batch_size;
+
+ switch (opcode) {
+ case METALOG_NOOP:
+ /* the command queue was empty, do nothing */
+ break;
+ case METALOG_SHUTDOWN:
+ shutdown = 1;
+ break;
+ case METALOG_QUIESCE:
+ ctx->quiesce = SET_QUIESCE;
+ assert(0 == uv_timer_stop(&timer_req));
+ uv_close((uv_handle_t *)&timer_req, NULL);
+ mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+ if (!metalog_threads_alive(wc)) {
+ ctx->quiesce = QUIESCED;
+ complete(&ctx->metalog_completion);
+ }
+ break;
+ case METALOG_COMMIT_CREATION_RECORD:
+ do_commit_record(wc, METALOG_CREATE_OBJECT, &cmd.record_io_descr);
+ break;
+ case METALOG_COMMIT_DELETION_RECORD:
+ do_commit_record(wc, METALOG_DELETE_OBJECT, &cmd.record_io_descr);
+ break;
+ case METALOG_COMPACTION_FLUSH:
+ mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log,
+ &ctx->compaction_state.new_metadata_logfiles);
+ complete(cmd.record_io_descr.completion);
+ break;
+ default:
+ debug(D_METADATALOG, "%s: default.", __func__);
+ break;
+ }
+ } while (opcode != METALOG_NOOP);
+ }
+
+ /* cleanup operations of the event loop */
+ info("Shutting down RRD metadata log event loop.");
+
+ /*
+ * uv_async_send after uv_close does not seem to crash in linux at the moment,
+ * it is however undocumented behaviour and we need to be aware if this becomes
+ * an issue in the future.
+ */
+ uv_close((uv_handle_t *)&wc->async, NULL);
+
+ mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+ uv_run(loop, UV_RUN_DEFAULT);
+
+ info("Shutting down RRD metadata log loop complete.");
+ /* TODO: don't let the API block by waiting to enqueue commands */
+ uv_cond_destroy(&wc->cmd_cond);
+/* uv_mutex_destroy(&wc->cmd_mutex); */
+ assert(0 == uv_loop_close(loop));
+ freez(loop);
+
+ return;
+
+error_after_timer_init:
+ uv_close((uv_handle_t *)&wc->async, NULL);
+error_after_async_init:
+ assert(0 == uv_loop_close(loop));
+error_after_loop_init:
+ freez(loop);
+
+ wc->error = UV_EAGAIN;
+ /* wake up initialization thread */
+ complete(&ctx->metalog_completion);
+}
diff --git a/database/engine/metadata_log/metadatalog.h b/database/engine/metadata_log/metadatalog.h
new file mode 100644
index 0000000000..004eec7ee3
--- /dev/null
+++ b/database/engine/metadata_log/metadatalog.h
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOG_H
+#define NETDATA_METADATALOG_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include "../rrdengine.h"
+#include "metadatalogprotocol.h"
+#include "logfile.h"
+#include "metadatalogapi.h"
+#include "compaction.h"
+
+/* Forward declerations */
+struct metalog_instance;
+struct parser_user_object;
+
+#define MAX_PAGES_PER_EXTENT (64) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
+
+#define METALOG_FILE_NUMBER_SCAN_TMPL "%5u-%5u"
+#define METALOG_FILE_NUMBER_PRINT_TMPL "%5.5u-%5.5u"
+
+#define MAX_DUPLICATION_PERCENTAGE 150 /* the maximum duplication factor of objects in metadata log records */
+
+typedef enum {
+ METALOG_STATUS_UNINITIALIZED = 0,
+ METALOG_STATUS_INITIALIZING,
+ METALOG_STATUS_INITIALIZED
+} metalog_state_t;
+
+struct metalog_record_io_descr {
+ BUFFER *buffer;
+ struct completion *completion;
+ int compacting; /* When 0 append at the end of the metadata log file list.
+ When 1 append to the temporary compaction metadata log file list. */
+ uuid_t uuid;
+};
+
+enum metalog_opcode {
+ /* can be used to return empty status or flush the command queue */
+ METALOG_NOOP = 0,
+
+ METALOG_SHUTDOWN,
+ METALOG_COMMIT_CREATION_RECORD,
+ METALOG_COMMIT_DELETION_RECORD,
+ METALOG_COMPACTION_FLUSH,
+ METALOG_QUIESCE,
+
+ METALOG_MAX_OPCODE
+};
+
+struct metalog_cmd {
+ enum metalog_opcode opcode;
+ struct metalog_record_io_descr record_io_descr;
+};
+
+#define METALOG_CMD_Q_MAX_SIZE (2048)
+
+struct metalog_cmdqueue {
+ unsigned head, tail;
+ struct metalog_cmd cmd_array[METALOG_CMD_Q_MAX_SIZE];
+};
+
+struct metalog_worker_config {
+ struct metalog_instance *ctx;
+
+ uv_thread_t thread;
+ uv_loop_t *loop;
+ uv_async_t async;
+
+ /* metadata log file comapaction thread */
+ uv_thread_t *now_compacting_files;
+ unsigned long cleanup_thread_compacting_files; /* set to 0 when now_compacting_files is still running */
+
+ /* FIFO command queue */
+ uv_mutex_t cmd_mutex;
+ uv_cond_t cmd_cond;
+ volatile unsigned queue_size;
+ struct metalog_cmdqueue cmd_queue;
+
+ int error;
+};
+
+/*
+ * Debug statistics not used by code logic.
+ * They only describe operations since DB engine instance load time.
+ */
+struct metalog_statistics {
+ rrdeng_stats_t io_write_bytes;
+ rrdeng_stats_t io_write_requests;
+ rrdeng_stats_t io_read_bytes;
+ rrdeng_stats_t io_read_requests;
+ rrdeng_stats_t io_write_record_bytes;
+ rrdeng_stats_t io_write_records;
+ rrdeng_stats_t io_read_record_bytes;
+ rrdeng_stats_t io_read_records;
+ rrdeng_stats_t metadata_logfile_creations;
+ rrdeng_stats_t metadata_logfile_deletions;
+ rrdeng_stats_t io_errors;
+ rrdeng_stats_t fs_errors;
+};
+
+struct metalog_instance {
+ struct rrdengine_instance *rrdeng_ctx;
+ struct metalog_worker_config worker_config;
+ struct completion metalog_completion;
+ struct metadata_record_commit_log records_log;
+ struct metadata_logfile_list metadata_logfiles;
+ struct parser_user_object *metalog_parser_object;
+ struct logfile_compaction_state compaction_state;
+ uint32_t current_compaction_id; /* Every compaction run increments this by 1 */
+ unsigned long disk_space;
+ unsigned long records_nr;
+ unsigned last_fileno; /* newest index of metadata log file */
+
+ uint8_t quiesce; /*
+ * 0 initial state when all operations function normally
+ * 1 set it before shutting down the instance, quiesce long running operations
+ * 2 is set after all threads have finished running
+ */
+
+ struct metalog_statistics stats;
+};
+
+extern void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode,
+ uuid_t *uuid, int compacting);
+ extern int init_metadata_logfiles(struct metalog_instance *ctx);
+extern void finalize_metadata_logfiles(struct metalog_instance *ctx);
+extern void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc);
+extern void metalog_test_quota(struct metalog_worker_config *wc);
+extern void metalog_worker(void* arg);
+extern void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd);
+extern struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc);
+
+#endif /* NETDATA_METADATALOG_H */
diff --git a/database/engine/metadata_log/metadatalogapi.c b/database/engine/metadata_log/metadatalogapi.c
new file mode 100755
index 0000000000..6bd09b4d9e
--- /dev/null
+++ b/database/engine/metadata_log/metadatalogapi.c
@@ -0,0 +1,444 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+static inline int metalog_is_initialized(struct metalog_instance *ctx)
+{
+ return ctx->rrdeng_ctx->metalog_ctx != NULL;
+}
+
+static inline void metalog_commit_creation_record(struct metalog_instance *ctx, BUFFER *buffer, uuid_t *uuid)
+{
+ metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 0);
+}
+
+static inline void metalog_commit_deletion_record(struct metalog_instance *ctx, BUFFER *buffer)
+{
+ metalog_commit_record(ctx, buffer, METALOG_COMMIT_DELETION_RECORD, NULL, 0);
+}
+
+BUFFER *metalog_update_host_buffer(RRDHOST *host)
+{
+ BUFFER *buffer;
+ buffer = buffer_create(4096); /* This will be freed after it has been committed to the metadata log buffer */
+
+ rrdhost_rdlock(host);
+
+ buffer_sprintf(buffer,
+ "HOST \"%s\" \"%s\" \"%s\" %d \"%s\" \"%s\" \"%s\"\n",
+// "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"" /* system */
+// "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"", /* info */
+ host->machine_guid,
+ host->hostname,
+ host->registry_hostname,
+ default_rrd_update_every,
+ host->os,
+ host->timezone,
+ (host->tags) ? host->tags : "");
+
+ netdata_rwlock_rdlock(&host->labels_rwlock);
+ struct label *labels = host->labels;
+ while (labels) {
+ buffer_sprintf(buffer
+ , "LABEL \"%s\" = %d %s\n"
+ , labels->key
+ , (int)labels->label_source
+ , labels->value);
+
+ labels = labels->next;
+ }
+ netdata_rwlock_unlock(&host->labels_rwlock);
+
+ buffer_strcat(buffer, "OVERWRITE labels\n");
+
+ rrdhost_unlock(host);
+ return buffer;
+}
+
+void metalog_commit_update_host(RRDHOST *host)
+{
+ struct metalog_instance *ctx;
+ BUFFER *buffer;
+
+ /* Metadata are only available with dbengine */
+ if (!host->rrdeng_ctx)
+ return;
+
+ ctx = host->rrdeng_ctx->metalog_ctx;
+ if (!ctx) /* metadata log has not been initialized yet */
+ return;
+
+ buffer = metalog_update_host_buffer(host);
+
+ metalog_commit_creation_record(ctx, buffer, &host->host_uuid);
+}
+
+/* compaction_id 0 means it was not called by compaction logic */
+BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id)
+{
+ BUFFER *buffer;
+ RRDHOST *host = st->rrdhost;
+
+ buffer = buffer_create(1024); /* This will be freed after it has been committed to the metadata log buffer */
+
+ rrdset_rdlock(st);
+
+ buffer_sprintf(buffer, "CONTEXT %s\n", host->machine_guid);
+
+ char uuid_str[37];
+ uuid_unparse_lower(*st->chart_uuid, uuid_str);
+ buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+ // properly set the name for the remote end to parse it
+ char *name = "";
+ if(likely(st->name)) {
+ if(unlikely(strcmp(st->id, st->name))) {
+ // they differ
+ name = strchr(st->name, '.');
+ if(name)
+ name++;
+ else
+ name = "";
+ }
+ }
+
+ // send the chart
+ buffer_sprintf(
+ buffer
+ , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
+ , st->id
+ , name
+ , st->title
+ , st->units
+ , st->family
+ , st->context
+ , rrdset_type_name(st->chart_type)
+ , st->priority
+ , st->update_every
+ , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
+ , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
+ , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
+ , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
+ , (st->plugin_name)?st->plugin_name:""
+ , (st->module_name)?st->module_name:""
+ );
+
+ // send the dimensions
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st) {
+ char uuid_str[37];
+
+ uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+ buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+ buffer_sprintf(
+ buffer
+ , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
+ , rd->id
+ , rd->name
+ , rrd_algorithm_name(rd->algorithm)
+ , rd->multiplier
+ , rd->divisor
+ , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
+ , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
+ , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
+ );
+ if (compaction_id && compaction_id > rd->state->compaction_id) {
+ /* No need to use this dimension again during this compaction cycle */
+ rd->state->compaction_id = compaction_id;
+ }
+ }
+ rrdset_unlock(st);
+ return buffer;
+}
+
+void metalog_commit_update_chart(RRDSET *st)
+{
+ struct metalog_instance *ctx;
+ BUFFER *buffer;
+ RRDHOST *host = st->rrdhost;
+
+ /* Metadata are only available with dbengine */
+ if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+ return;
+
+ ctx = host->rrdeng_ctx->metalog_ctx;
+ if (!ctx) /* metadata log has not been initialized yet */
+ return;
+
+ buffer = metalog_update_chart_buffer(st, 0);
+
+ metalog_commit_creation_record(ctx, buffer, st->chart_uuid);
+}
+
+void metalog_commit_delete_chart(RRDSET *st)
+{
+ struct metalog_instance *ctx;
+ BUFFER *buffer;
+ RRDHOST *host = st->rrdhost;
+ char uuid_str[37];
+
+ /* Metadata are only available with dbengine */
+ if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+ return;
+
+ ctx = host->rrdeng_ctx->metalog_ctx;
+ if (!ctx) /* metadata log has not been initialized yet */
+ return;
+ buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
+
+ uuid_unparse_lower(*st->chart_uuid, uuid_str);
+ buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
+
+ metalog_commit_deletion_record(ctx, buffer);
+}
+
+BUFFER *metalog_update_dimension_buffer(RRDDIM *rd)
+{
+ BUFFER *buffer;
+ RRDSET *st = rd->rrdset;
+ char uuid_str[37];
+
+ buffer = buffer_create(128); /* This will be freed after it has been committed to the metadata log buffer */
+
+ uuid_unparse_lower(*st->chart_uuid, uuid_str);
+ buffer_sprintf(buffer, "CONTEXT %s\n", uuid_str);
+ // Activate random GUID
+ uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+ buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+ buffer_sprintf(
+ buffer
+ , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
+ , rd->id
+ , rd->name
+ , rrd_algorithm_name(rd->algorithm)
+ , rd->multiplier
+ , rd->divisor
+ , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
+ , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
+ , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
+ );
+ return buffer;
+}
+
+void metalog_commit_update_dimension(RRDDIM *rd)
+{
+ struct metalog_instance *ctx;
+ BUFFER *buffer;
+ RRDSET *st = rd->rrdset;
+ RRDHOST *host = st->rrdhost;
+
+ /* Metadata are only available with dbengine */
+ if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+ return;
+
+ ctx = host->rrdeng_ctx->metalog_ctx;
+ if (!ctx) /* metadata log has not been initialized yet */
+ return;
+
+ buffer = metalog_update_dimension_buffer(rd);
+
+ metalog_commit_creation_record(ctx, buffer, rd->state->metric_uuid);
+}
+
+void metalog_commit_delete_dimension(RRDDIM *rd)
+{
+ struct metalog_instance *ctx;
+ BUFFER *buffer;
+ RRDSET *st = rd->rrdset;
+ RRDHOST *host = st->rrdhost;
+ char uuid_str[37];
+
+ /* Metadata are only available with dbengine */
+ if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+ return;
+
+ ctx = host->rrdeng_ctx->metalog_ctx;
+ if (!ctx) /* metadata log has not been initialized yet */
+ return;
+ buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
+
+ uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+ buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
+
+ metalog_commit_deletion_record(ctx, buffer);
+}
+
+RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
+{
+ GUID_TYPE ret;
+ char chart_object[33], chart_fullid[RRD_ID_LENGTH_MAX + 1];
+ uuid_t *machine_guid, *chart_char_guid;
+
+ ret = find_object_by_guid(chart_uuid, chart_object, 33);
+ assert(GUID_TYPE_CHART == ret);
+
+ machine_guid = (uuid_t *)chart_object;
+ RRDHOST *host = ctx->rrdeng_ctx->host;
+ assert(!uuid_compare(host->host_uuid, *machine_guid));
+
+ chart_char_guid = (uuid_t *)(chart_object + 16);
+
+ ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
+ assert(GUID_TYPE_CHAR == ret);
+ RRDSET *st = rrdset_find(host, chart_fullid);
+
+ return st;
+}
+
+RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
+{
+ GUID_TYPE ret;
+ char dim_object[49], chart_object[33], id_str[PLUGINSD_LINE_MAX], chart_fullid[RRD_ID_LENGTH_MAX + 1];
+ uuid_t *machine_guid, *chart_guid, *chart_char_guid, *dim_char_guid;
+
+ ret = find_object_by_guid(metric_uuid, dim_object, sizeof(dim_object));
+ if (GUID_TYPE_DIMENSION != ret) /* not found */
+ return NULL;
+
+ machine_guid = (uuid_t *)dim_object;
+ RRDHOST *host = ctx->rrdeng_ctx->host;
+ assert(!uuid_compare(host->host_uuid, *machine_guid));
+
+ chart_guid = (uuid_t *)(dim_object + 16);
+ dim_char_guid = (uuid_t *)(dim_object + 16 + 16);
+
+ ret = find_object_by_guid(dim_char_guid, id_str, sizeof(id_str));
+ assert(GUID_TYPE_CHAR == ret);
+
+ ret = find_object_by_guid(chart_guid, chart_object, sizeof(chart_object));
+ assert(GUID_TYPE_CHART == ret);
+ chart_char_guid = (uuid_t *)(chart_object + 16);
+
+ ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
+ assert(GUID_TYPE_CHAR == ret);
+ RRDSET *st = rrdset_find(host, chart_fullid);
+ assert(st);
+
+ RRDDIM *rd = rrddim_find(st, id_str);
+
+ return rd;
+}
+
+/* This function is called by dbengine rotation logic when the metric has no writers */
+void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
+{
+ RRDDIM *rd;
+ RRDSET *st;
+ RRDHOST *host;
+ uint8_t empty_chart;
+
+ rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
+ if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
+ uuid_t multihost_uuid;
+
+ rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->host->machine_guid, metric_uuid, &multihost_uuid);
+ rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);
+ }
+ if(!rd) {
+ info("Rotated unknown archived metric.");
+ return;
+ }
+ st = rd->rrdset;
+ host = st->rrdhost;
+
+ /* Since the metric has no writer it will not be commited to the metadata log by rrddim_free_custom().
+ * It must be commited explicitly before calling rrddim_free_custom(). */
+ metalog_commit_delete_dimension(rd);
+
+ rrdset_wrlock(st);
+ rrddim_free_custom(st, rd, 1);
+ empty_chart = (NULL == st->dimensions);
+ rrdset_unlock(st);
+
+ if (empty_chart) {
+ rrdhost_wrlock(host);
+ rrdset_rdlock(st);
+ rrdset_delete_custom(st, 1);
+ rrdset_unlock(st);
+ rrdset_free(st);
+ rrdhost_unlock(host);
+ }
+}
+
+/*
+ * Returns 0 on success, negative on error
+ */
+int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
+{
+ struct metalog_instance *ctx;
+ int error;
+
+ ctx = callocz(1, sizeof(*ctx));
+ ctx->records_nr = 0;
+ ctx->current_compaction_id = 0;
+ ctx->quiesce = NO_QUIESCE;
+
+ memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
+ ctx->rrdeng_ctx = rrdeng_parent_ctx;
+ ctx->worker_config.ctx = ctx;
+ init_metadata_record_log(&ctx->records_log);
+ error = init_metalog_files(ctx);
+ if (error) {
+ goto error_after_init_rrd_files;
+ }
+
+ init_completion(&ctx->metalog_completion);
+ assert(0 == uv_thread_create(&ctx->worker_config.thread, metalog_worker, &ctx->worker_config));
+ /* wait for worker thread to initialize */
+ wait_for_completion(&ctx->metalog_completion);
+ destroy_completion(&ctx->metalog_completion);
+ uv_thread_set_name_np(ctx->worker_config.thread, "METALOG");
+ if (ctx->worker_config.error) {
+ goto error_after_rrdeng_worker;
+ }
+ rrdeng_parent_ctx->metalog_ctx = ctx; /* notify dbengine that the metadata log has finished initializing */
+ return 0;
+
+error_after_rrdeng_worker:
+ finalize_metalog_files(ctx);
+error_after_init_rrd_files:
+ freez(ctx);
+ return UV_EIO;
+}
+
+/*
+ * Returns 0 on success, 1 on error
+ */
+int metalog_exit(struct metalog_instance *ctx)
+{
+ struct metalog_cmd cmd;
+
+ if (NULL == ctx) {
+ return 1;
+ }
+
+ cmd.opcode = METALOG_SHUTDOWN;
+ metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+ assert(0 == uv_thread_join(&ctx->worker_config.thread));
+
+ finalize_metalog_files(ctx);
+ freez(ctx);
+
+ return 0;
+}
+
+void metalog_prepare_exit(struct metalog_instance *ctx)
+{
+ struct metalog_cmd cmd;
+
+ if (NULL == ctx) {
+ return;
+ }
+
+ init_completion(&ctx->metalog_completion);
+ cmd.opcode = METALOG_QUIESCE;
+ metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+ /* wait for metadata log to quiesce */
+ wait_for_completion(&ctx->metalog_completion);
+ destroy_completion(&ctx->metalog_completion);
+}
diff --git a/database/engine/metadata_log/metadatalogapi.h b/database/engine/metadata_log/metadatalogapi.h
new file mode 100644
index 0000000000..d9dd589848
--- /dev/null
+++ b/database/engine/metadata_log/metadatalogapi.h
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOGAPI_H
+#define NETDATA_METADATALOGAPI_H
+
+#include "metadatalog.h"
+
+extern BUFFER *metalog_update_host_buffer(RRDHOST *host);
+extern void metalog_commit_update_host(RRDHOST *host);
+extern BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id);
+extern void metalog_commit_update_chart(RRDSET *st);
+extern void metalog_commit_delete_chart(RRDSET *st);
+extern BUFFER *metalog_update_dimension_buffer(RRDDIM *rd);
+extern void metalog_commit_update_dimension(RRDDIM *rd);
+extern void metalog_commit_delete_dimension(RRDDIM *rd);
+
+extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
+extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
+extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
+
+/* must call once before using anything */
+extern int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx);
+extern int metalog_exit(struct metalog_instance *ctx);
+extern void metalog_prepare_exit(struct metalog_instance *ctx);
+
+#endif /* NETDATA_METADATALOGAPI_H */
diff --git a/database/engine/metadata_log/metadatalogprotocol.h b/database/engine/metadata_log/metadatalogprotocol.h
new file mode 100644
index 0000000000..1017213ae2
--- /dev/null
+++ b/database/engine/metadata_log/metadatalogprotocol.h
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOGPROTOCOL_H
+#define NETDATA_METADATALOGPROTOCOL_H
+
+#include "../rrddiskprotocol.h"
+
+#define RRDENG_METALOG_MAGIC "netdata-metadata-log"
+
+#define RRDENG_METALOG_VER (1)
+
+#define RRDENG_METALOG_SB_PADDING_SZ (RRDENG_BLOCK_SIZE - (RRDENG_MAGIC_SZ + sizeof(uint16_t)))
+/*
+ * Metadata log persistent super-block
+ */
+struct rrdeng_metalog_sb {
+ char magic_number[RRDENG_MAGIC_SZ];
+ uint16_t version;
+ uint8_t padding[RRDENG_METALOG_SB_PADDING_SZ];
+} __attribute__ ((packed));
+
+/*
+ * Metadata log record types
+ */
+#define METALOG_STORE_PADDING (0)
+#define METALOG_CREATE_OBJECT (1)
+#define METALOG_DELETE_OBJECT (2)
+#define METALOG_OTHER (3) /* reserved */
+
+/*
+ * Metadata log record header
+ */
+struct rrdeng_metalog_record_header {
+ /* when set to METALOG_STORE_PADDING jump to start of next block */
+ uint8_t type;
+
+ uint16_t header_length;
+ uint32_t payload_length;
+ /******************************************************
+ * No fields above this point can ever change. *
+ ******************************************************
+ * All fields below this point are subject to change. *
+ ******************************************************/
+} __attribute__ ((packed));
+
+/*
+ * Metadata log record trailer
+ */
+struct rrdeng_metalog_record_trailer {
+ uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */
+} __attribute__ ((packed));
+
+#endif /* NETDATA_METADATALOGPROTOCOL_H */
diff --git a/database/engine/metadata_log/metalogpluginsd.c b/database/engine/metadata_log/metalogpluginsd.c
new file mode 100755
index 0000000000..23027af7b2
--- /dev/null
+++ b/database/engine/metadata_log/metalogpluginsd.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+#include "metalogpluginsd.h"
+
+PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+ char *title, char *units, char *plugin, char *module, int priority,
+ int update_every, RRDSET_TYPE chart_type, char *options)
+{
+ struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+ RRDSET *st = NULL;
+ RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
+ uuid_t *chart_uuid;
+
+ chart_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
+ st = rrdset_create_custom(
+ host, type, id, name, family, context, title, units,
+ plugin, module, priority, update_every,
+ chart_type, RRD_MEMORY_MODE_DBENGINE, (host)->rrd_history_entries, 1, chart_uuid);
+
+ if (options && *options) {
+ if (strstr(options, "obsolete"))
+ rrdset_is_obsolete(st);
+ else
+ rrdset_isnot_obsolete(st);
+
+ if (strstr(options, "detail"))
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+
+ if (strstr(options, "hidden"))
+ rrdset_flag_set(st, RRDSET_FLAG_HIDDEN);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_HIDDEN);
+
+ if (strstr(options, "store_first"))
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+ } else {
+ rrdset_isnot_obsolete(st);
+ rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+ }
+ ((PARSER_USER_OBJECT *)user)->st = st;
+
+ if (chart_uuid) { /* It's a valid object */
+ struct metalog_record record;
+ struct metadata_logfile *metalogfile = state->metalogfile;
+
+ uuid_copy(record.uuid, state->uuid);
+ mlf_record_insert(metalogfile, &record);
+ uuid_clear(state->uuid); /* Consume UUID */
+ }
+ return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+ long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type)
+{
+ struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+ UNUSED(user);
+ UNUSED(algorithm);
+ uuid_t *dim_uuid;
+
+ dim_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
+
+ RRDDIM *rd = rrddim_add_custom(st, id, name, multiplier, divisor, algorithm_type, RRD_MEMORY_MODE_DBENGINE, 1,
+ dim_uuid);
+ rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
+ rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if (options && *options) {
+ if (strstr(options, "obsolete") != NULL)
+ rrddim_is_obsolete(st, rd);
+ else
+ rrddim_isnot_obsolete(st, rd);
+ if (strstr(options, "hidden") != NULL)
+ rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
+ if (strstr(options, "noreset") != NULL)
+ rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if (strstr(options, "nooverflow") != NULL)
+ rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ } else {
+ rrddim_isnot_obsolete(st, rd);
+ }
+ if (dim_uuid) { /* It's a valid object */
+ struct metalog_record record;
+ struct metadata_logfile *metalogfile = state->metalogfile;
+
+ uuid_copy(record.uuid, state->uuid);
+ mlf_record_insert(metalogfile, &record);
+ uuid_clear(state->uuid); /* Consume UUID */
+ }
+ return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)
+{
+ struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+
+ uuid_copy(state->uuid, *uuid);
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
+{
+ GUID_TYPE ret;
+ struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+ struct metalog_instance *ctx = state->ctx;
+ char object[49], chart_object[33], id_str[1024];
+ uuid_t *chart_guid, *chart_char_guid;
+ RRDHOST *host;
+
+ ret = find_object_by_guid(uuid, object, 49);
+ switch (ret) {
+ case GUID_TYPE_CHAR:
+ assert(0);
+ break;
+ case GUID_TYPE_CHART:
+ case GUID_TYPE_DIMENSION:
+ host = ctx->rrdeng_ctx->host;
+ switch (ret) {
+ case GUID_TYPE_CHART:
+ chart_char_guid = (uuid_t *)(object + 16);
+
+ ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
+ assert(GUID_TYPE_CHAR == ret);
+ ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
+ break;
+ case GUID_TYPE_DIMENSION:
+ chart_guid = (uuid_t *)(object + 16);
+
+ ret = find_object_by_guid(chart_guid, chart_object, 33);
+ assert(GUID_TYPE_CHART == ret);
+ chart_char_guid = (uuid_t *)(chart_object + 16);
+
+ ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
+ assert(GUID_TYPE_CHAR == ret);
+ ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ break;
+ case GUID_TYPE_HOST:
+ /* Ignore for now */
+ break;
+ default:
+ break;
+ }
+
+ return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
+{
+ GUID_TYPE ret;
+ struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+ struct metalog_instance *ctx = state->ctx;
+ RRDHOST *host = ctx->rrdeng_ctx->host;
+ RRDSET *st;
+ RRDDIM *rd;
+
+ ret = find_object_by_guid(uuid, NULL, 0);
+ switch (ret) {
+ case GUID_TYPE_CHAR:
+ assert(0);
+ break;
+ case GUID_TYPE_CHART:
+ st = metalog_get_chart_from_uuid(ctx, uuid);
+ if (st) {
+ rrdhost_wrlock(host);
+ rrdset_free(st);
+ rrdhost_unlock(host);
+ }
+ break;
+ case GUID_TYPE_DIMENSION:
+ rd = metalog_get_dimension_from_uuid(ctx, uuid);
+ if (rd) {
+ st = rd->rrdset;
+ rrdset_wrlock(st);
+ rrddim_free_custom(st, rd, 0);
+ rrdset_unlock(st);
+ }
+ break;
+ case GUID_TYPE_HOST:
+ /* Ignore for now */
+ break;
+ default:
+ break;
+ }
+
+ return PARSER_RC_OK;
+}
+
+void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx)
+{
+ state->ctx = ctx;
+ state->skip_record = 0;
+ uuid_clear(state->uuid);
+ state->metalogfile = NULL;
+}
\ No newline at end of file
diff --git a/database/engine/metadata_log/metalogpluginsd.h b/database/engine/metadata_log/metalogpluginsd.h
new file mode 100644
index 0000000000..b29a9db33a
--- /dev/null
+++ b/database/engine/metadata_log/metalogpluginsd.h
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METALOGPLUGINSD_H
+#define NETDATA_METALOGPLUGINSD_H
+
+#include "../../../collectors/plugins.d/pluginsd_parser.h"
+#include "../../../collectors/plugins.d/plugins_d.h"
+#include "../../../parser/parser.h"
+
+struct metalog_pluginsd_state {
+ struct metalog_instance *ctx;
+ uuid_t uuid;
+ uint8_t skip_record; /* skip this record due to errors in parsing */
+ struct metadata_logfile *metalogfile; /* current metadata log file being replayed */
+};
+
+extern void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx);
+
+extern PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family,
+ char *context, char *title, char *units, char *plugin, char *module,
+ int priority, int update_every, RRDSET_TYPE chart_type, char *options);
+extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+ long multiplier, long divisor, char *options,
+ RRD_ALGORITHM algorithm_type);
+extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
+extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
+extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
+
+#endif /* NETDATA_METALOGPLUGINSD_H */
diff --git a/database/engine/pagecache.c b/database/engine/pagecache.c
index 4534921267..f1f3a5be85 100644
--- a/database/engine/pagecache.c
+++ b/database/engine/pagecache.c
@@ -383,17 +383,26 @@ static int pg_cache_try_evict_one_page_unsafe(struct rrdengine_instance *ctx)
return 0;
}
-/*
- * Callers of this function need to make sure they're not deleting the same descriptor concurrently
+/**
+ * Deletes a page from the database.
+ * Callers of this function need to make sure they're not deleting the same descriptor concurrently.
+ * @param ctx is the database instance.
+ * @param descr is the page descriptor.
+ * @param remove_dirty must be non-zero if the page to be deleted is dirty.
+ * @param is_exclusive_holder must be non-zero if the caller holds an exclusive page reference.
+ * @param metric_id is set to the metric the page belongs to, if it's safe to delete the metric and metric_id is not
+ * NULL. Otherwise, metric_id is not set.
+ * @return 1 if it's safe to delete the metric, 0 otherwise.
*/
-void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
- uint8_t is_exclusive_holder)
+uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
+ uint8_t is_exclusive_holder, uuid_t *metric_id)
{
struct page_cache *pg_cache = &ctx->pg_cache;
struct page_cache_descr *pg_cache_descr = NULL;
Pvoid_t *PValue;
struct pg_cache_page_index *page_index;
int ret;
+ uint8_t can_delete_metric = 0;
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, descr->id, sizeof(uuid_t));
@@ -403,14 +412,22 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
uv_rwlock_wrlock(&page_index->lock);
ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
- uv_rwlock_wrunlock(&page_index->lock);
if (unlikely(0 == ret)) {
+ uv_rwlock_wrunlock(&page_index->lock);
error("Page under deletion was not in index.");
if (unlikely(debug_flags & D_RRDENGINE)) {
print_page_descr(descr);
}
goto destroy;
}
+ --page_index->page_count;
+ if (!page_index->writers && !page_index->page_count) {
+ can_delete_metric = 1;
+ if (metric_id) {
+ memcpy(metric_id, page_index->id, sizeof(uuid_t));
+ }
+ }
+ uv_rwlock_wrunlock(&page_index->lock);
assert(1 == ret);
uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);
@@ -459,6 +476,8 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
destroy:
freez(descr);
pg_cache_update_metric_times(page_index);
+
+ return can_delete_metric;
}
static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t start_time, usec_t end_time)
@@ -588,6 +607,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
uv_rwlock_wrlock(&page_index->lock);
PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
*PValue = descr;
+ ++page_index->page_count;
pg_cache_add_new_metric_time(page_index, descr);
uv_rwlock_wrunlock(&page_index->lock);
@@ -1032,6 +1052,8 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
page_index->oldest_time = INVALID_TIME;
page_index->latest_time = INVALID_TIME;
page_index->prev = NULL;
+ page_index->page_count = 0;
+ page_index->writers = 0;
return page_index;
}
diff --git a/database/engine/pagecache.h b/database/engine/pagecache.h
index 3722a7e1cc..4ca57cd667 100644
--- a/database/engine/pagecache.h
+++ b/database/engine/pagecache.h
@@ -85,6 +85,8 @@ struct pg_cache_page_index {
* TODO: examine if we want to support better granularity than seconds
*/
Pvoid_t JudyL_array;
+ Word_t page_count;
+ unsigned short writers;
uv_rwlock_t lock;
/*
@@ -163,8 +165,8 @@ extern void pg_cache_put_unsafe(struct rrdeng_page_descr *descr);
extern void pg_cache_put(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr);
extern void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index *index,
struct rrdeng_page_descr *descr);
-extern void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
- uint8_t is_exclusive_holder);
+extern uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
+ uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
extern usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
usec_t start_time, usec_t end_time);
extern void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c
index b15239e6c4..bf59242711 100644
--- a/database/engine/rrdengine.c
+++ b/database/engine/rrdengine.c
@@ -304,7 +304,7 @@ static void invalidate_oldest_committed(void *arg)
goto out;
}
- pg_cache_punch_hole(ctx, descr, 1, 1);
+ pg_cache_punch_hole(ctx, descr, 1, 1, NULL);
uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
nr_committed_pages = --pg_cache->committed_page_index.nr_committed_pages;
@@ -326,6 +326,9 @@ void rrdeng_invalidate_oldest_committed(struct rrdengine_worker_config* wc)
unsigned nr_committed_pages;
int error;
+ if (unlikely(ctx->quiesce != NO_QUIESCE)) /* Shutting down */
+ return;
+
uv_rwlock_rdlock(&pg_cache->committed_page_index.lock);
nr_committed_pages = pg_cache->committed_page_index.nr_committed_pages;
uv_rwlock_rdunlock(&pg_cache->committed_page_index.lock);
@@ -630,6 +633,8 @@ static void delete_old_data(void *arg)
struct extent_info *extent, *next;
struct rrdeng_page_descr *descr;
unsigned count, i;
+ uint8_t can_delete_metric;
+ uuid_t metric_id;
/* Safe to use since it will be deleted after we are done */
datafile = ctx->datafiles.first;
@@ -638,7 +643,14 @@ static void delete_old_data(void *arg)
count = extent->number_of_pages;
for (i = 0 ; i < count ; ++i) {
descr = extent->pages[i];
- pg_cache_punch_hole(ctx, descr, 0, 0);
+ can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);
+ if (unlikely(can_delete_metric && ctx->metalog_ctx)) {
+ /*
+ * If the metric is empty, has no active writers and if the metadata log has been initialized then
+ * attempt to delete the corresponding netdata dimension.
+ */
+ metalog_delete_dimension_by_uuid(ctx->metalog_ctx, &metric_id);
+ }
}
next = extent->next;
freez(extent);
@@ -674,7 +686,7 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
++ctx->last_fileno;
}
}
- if (unlikely(out_of_space)) {
+ if (unlikely(out_of_space && NO_QUIESCE == ctx->quiesce)) {
/* delete old data */
if (wc->now_deleting_files) {
/* already deleting data */
@@ -710,12 +722,18 @@ static inline int rrdeng_threads_alive(struct rrdengine_worker_config* wc)
static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
{
+ struct rrdengine_instance *ctx = wc->ctx;
+
if (unlikely(wc->cleanup_thread_invalidating_dirty_pages)) {
after_invalidate_oldest_committed(wc);
}
if (unlikely(wc->cleanup_thread_deleting_files)) {
after_delete_old_data(wc);
}
+ if (unlikely(SET_QUIESCE == ctx->quiesce && !rrdeng_threads_alive(wc))) {
+ ctx->quiesce = QUIESCED;
+ complete(&ctx->rrdengine_completion);
+ }
}
/* return 0 on success */
@@ -799,14 +817,16 @@ void async_cb(uv_async_t *handle)
void timer_cb(uv_timer_t* handle)
{
struct rrdengine_worker_config* wc = handle->data;
+ struct rrdengine_instance *ctx = wc->ctx;
uv_stop(handle->loop);
uv_update_time(handle->loop);
+ if (unlikely(!ctx->metalog_ctx))
+ return; /* Wait for the metadata log to initialize */
rrdeng_test_quota(wc);
debug(D_RRDENGINE, "%s: timeout reached.", __func__);
if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
/* There is free space so we can write to disk and we are not actively deleting dirty buffers */
- struct rrdengine_instance *ctx = wc->ctx;
struct page_cache *pg_cache = &ctx->pg_cache;
unsigned long total_bytes, bytes_written, nr_committed_pages, bytes_to_write = 0, producers, low_watermark,
high_watermark;
@@ -920,7 +940,20 @@ void rrdeng_worker(void* arg)
break;
case RRDENG_SHUTDOWN:
shutdown = 1;
+ break;
+ case RRDENG_QUIESCE:
ctx->drop_metrics_under_page_cache_pressure = 0;
+ ctx->quiesce = SET_QUIESCE;
+ assert(0 == uv_timer_stop(&timer_req));
+ uv_close((uv_handle_t *)&timer_req, NULL);
+ while (do_flush_pages(wc, 1, NULL)) {
+ ; /* Force flushing of all committed pages. */
+ }
+ wal_flush_transaction_buffer(wc);
+ if (!rrdeng_threads_alive(wc)) {
+ ctx->quiesce = QUIESCED;
+ complete(&ctx->rrdengine_completion);
+ }
break;
case RRDENG_READ_PAGE:
do_read_extent(wc, &cmd.read_page.page_cache_descr, 1, 0);
@@ -959,8 +992,6 @@ void rrdeng_worker(void* arg)
* an issue in the future.
*/
uv_close((uv_handle_t *)&wc->async, NULL);
- assert(0 == uv_timer_stop(&timer_req));
- uv_close((uv_handle_t *)&timer_req, NULL);
while (do_flush_pages(wc, 1, NULL)) {
; /* Force flushing of all committed pages. */
@@ -998,7 +1029,7 @@ void rrdengine_main(void)
struct rrdengine_instance *ctx;
sanity_check();
- ret = rrdeng_init(&ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
+ ret = rrdeng_init(NULL, &ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
if (ret) {
exit(ret);
}
diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h
index d97a34aa30..e5d474a74b 100644
--- a/database/engine/rrdengine.h
+++ b/database/engine/rrdengine.h
@@ -17,6 +17,7 @@
#include "rrdenginelib.h"
#include "datafile.h"
#include "journalfile.h"
+#include "metadata_log/metadatalog.h"
#include "rrdengineapi.h"
#include "pagecache.h"
#include "rrdenglocking.h"
@@ -50,6 +51,7 @@ enum rrdeng_opcode {
RRDENG_FLUSH_PAGES,
RRDENG_SHUTDOWN,
RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE,
+ RRDENG_QUIESCE,
RRDENG_MAX_OPCODE
};
@@ -169,7 +171,13 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;
extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */
+#define NO_QUIESCE (0) /* initial state when all operations function normally */
+#define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
+#define QUIESCED (2) /* is set after all threads have finished running */
+
struct rrdengine_instance {
+ RRDHOST *host;
+ struct metalog_instance *metalog_ctx;
struct rrdengine_worker_config worker_config;
struct completion rrdengine_completion;
struct page_cache pg_cache;
@@ -185,6 +193,8 @@ struct rrdengine_instance {
unsigned long cache_pages_low_watermark;
unsigned long metric_API_max_producers;
+ uint8_t quiesce; /* set to SET_QUIESCE before shutdown of the engine */
+
struct rrdengine_statistics stats;
};
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index d8b5539795..b09d12c0e6 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -9,6 +9,114 @@ int default_rrdeng_disk_quota_mb = 256;
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
+/* This UUID is not unique across hosts */
+void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
+{
+ EVP_MD_CTX *evpctx;
+ unsigned char hash_value[EVP_MAX_MD_SIZE];
+ unsigned int hash_len;
+
+ evpctx = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
+ EVP_DigestUpdate(evpctx, dim_id, strlen(dim_id));
+ EVP_DigestUpdate(evpctx, chart_id, strlen(chart_id));
+ EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
+ EVP_MD_CTX_destroy(evpctx);
+ assert(hash_len > sizeof(uuid_t));
+ memcpy(ret_uuid, hash_value, sizeof(uuid_t));
+}
+
+/* Transform legacy UUID to be unique across hosts deterministacally */
+void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid, uuid_t *ret_uuid)
+{
+ EVP_MD_CTX *evpctx;
+ unsigned char hash_value[EVP_MAX_MD_SIZE];
+ unsigned int hash_len;
+
+ evpctx = EVP_MD_CTX_create();
+ EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
+ EVP_DigestUpdate(evpctx, machine_guid, GUID_LEN);
+ EVP_DigestUpdate(evpctx, *legacy_uuid, sizeof(uuid_t));
+ EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
+ EVP_MD_CTX_destroy(evpctx);
+ assert(hash_len > sizeof(uuid_t));
+ memcpy(ret_uuid, hash_value, sizeof(uuid_t));
+}
+
+void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)
+{
+ struct page_cache *pg_cache;
+ struct rrdengine_instance *ctx;
+ uuid_t legacy_uuid;
+ Pvoid_t *PValue;
+ struct pg_cache_page_index *page_index;
+ int replace_instead_of_generate = 0;
+
+ ctx = rd->rrdset->rrdhost->rrdeng_ctx;
+ pg_cache = &ctx->pg_cache;
+
+ rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
+ rd->state->metric_uuid = callocz(1, sizeof(uuid_t));
+
+ uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+ PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &legacy_uuid, sizeof(uuid_t));
+ if (likely(NULL != PValue)) {
+ page_index = *PValue;
+ }
+ uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+ if (NULL == PValue) {
+ /* First time we see the legacy UUID, drop legacy support, normal path */
+
+ if (NULL != dim_uuid) {
+ replace_instead_of_generate = 1;
+ uuid_copy(*rd->state->metric_uuid, *dim_uuid);
+ }
+ if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION,
+ replace_instead_of_generate))) {
+ errno = 0;
+ error("FAILED to generate GUID for %s", rd->id);
+ freez(rd->state->metric_uuid);
+ rd->state->metric_uuid = NULL;
+ assert(0);
+ }
+
+ uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+ PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t));
+ if (likely(NULL != PValue)) {
+ page_index = *PValue;
+ }
+ uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+ if (NULL == PValue) {
+ uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
+ PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t), PJE0);
+ assert(NULL == *PValue); /* TODO: figure out concurrency model */
+ *PValue = page_index = create_page_index(rd->state->metric_uuid);
+ page_index->prev = pg_cache->metrics_index.last_page_index;
+ pg_cache->metrics_index.last_page_index = page_index;
+ uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
+ }
+ } else {
+ /* There are legacy UUIDs in the database, implement backward compatibility */
+
+
+ rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
+ rd->state->metric_uuid);
+ if (dim_uuid && uuid_compare(*rd->state->metric_uuid, *dim_uuid)) {
+ error("Mismatch of metadata log DIMENSION GUID with dbengine metric GUID.");
+ }
+ if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION, 1))) {
+ errno = 0;
+ error("FAILED to generate GUID for %s", rd->id);
+ freez(rd->state->metric_uuid);
+ rd->state->metric_uuid = NULL;
+ assert(0);
+ }
+ }
+ rd->state->rrdeng_uuid = &page_index->id;
+ rd->state->page_index = page_index;
+ rd->state->compaction_id = 0;
+}
+
/*
* Gets a handle for storing metrics to the database.
* The handle must be released with rrdeng_store_metric_final().
@@ -16,53 +124,21 @@ uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
void rrdeng_store_metric_init(RRDDIM *rd)
{
struct rrdeng_collect_handle *handle;
- struct page_cache *pg_cache;
struct rrdengine_instance *ctx;
- uuid_t temp_id;
- Pvoid_t *PValue;
struct pg_cache_page_index *page_index;
- EVP_MD_CTX *evpctx;
- unsigned char hash_value[EVP_MAX_MD_SIZE];
- unsigned int hash_len;
-
- //&default_global_ctx; TODO: test this use case or remove it?
ctx = rd->rrdset->rrdhost->rrdeng_ctx;
- pg_cache = &ctx->pg_cache;
handle = &rd->state->handle.rrdeng;
handle->ctx = ctx;
- evpctx = EVP_MD_CTX_create();
- EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
- EVP_DigestUpdate(evpctx, rd->id, strlen(rd->id));
- EVP_DigestUpdate(evpctx, rd->rrdset->id, strlen(rd->rrdset->id));
- EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
- EVP_MD_CTX_destroy(evpctx);
- assert(hash_len > sizeof(temp_id));
- memcpy(&temp_id, hash_value, sizeof(temp_id));
-
handle->descr = NULL;
handle->prev_descr = NULL;
handle->unaligned_page = 0;
- uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
- PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t));
- if (likely(NULL != PValue)) {
- page_index = *PValue;
- }
- uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
- if (NULL == PValue) {
- /* First time we see the UUID */
- uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
- PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t), PJE0);
- assert(NULL == *PValue); /* TODO: figure out concurrency model */
- *PValue = page_index = create_page_index(&temp_id);
- page_index->prev = pg_cache->metrics_index.last_page_index;
- pg_cache->metrics_index.last_page_index = page_index;
- uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
- }
- rd->state->rrdeng_uuid = &page_index->id;
- handle->page_index = page_index;
+ page_index = rd->state->page_index;
+ uv_rwlock_wrlock(&page_index->lock);
+ ++page_index->writers;
+ uv_rwlock_wrunlock(&page_index->lock);
}
/* The page must be populated and referenced */
@@ -109,7 +185,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
if (unlikely(debug_flags & D_RRDENGINE))
print_page_cache_descr(descr);
pg_cache_put(ctx, descr);
- pg_cache_punch_hole(ctx, descr, 1, 0);
+ pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
handle->prev_descr = NULL;
} else {
/* added 1 extra reference to keep 2 dirty pages pinned per metric, expected refcnt = 2 */
@@ -170,7 +246,7 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
must_flush_unaligned_page)) {
rrdeng_store_metric_flush_current_page(rd);
- page = rrdeng_create_page(ctx, &handle->page_index->id, &descr);
+ page = rrdeng_create_page(ctx, &rd->state->page_index->id, &descr);
assert(page);
handle->descr = descr;
@@ -204,27 +280,38 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
}
}
- pg_cache_insert(ctx, handle->page_index, descr);
+ pg_cache_insert(ctx, rd->state->page_index, descr);
} else {
- pg_cache_add_new_metric_time(handle->page_index, descr);
+ pg_cache_add_new_metric_time(rd->state->page_index, descr);
}
}
/*
* Releases the database reference from the handle for storing metrics.
+ * Returns 1 if it's safe to delete the dimension.
*/
-void rrdeng_store_metric_finalize(RRDDIM *rd)
+int rrdeng_store_metric_finalize(RRDDIM *rd)
{
struct rrdeng_collect_handle *handle;
struct rrdengine_instance *ctx;
+ struct pg_cache_page_index *page_index;
+ uint8_t can_delete_metric = 0;
handle = &rd->state->handle.rrdeng;
ctx = handle->ctx;
+ page_index = rd->state->page_index;
rrdeng_store_metric_flush_current_page(rd);
if (handle->prev_descr) {
/* unpin old second page */
pg_cache_put(ctx, handle->prev_descr);
}
+ uv_rwlock_wrlock(&page_index->lock);
+ if (!--page_index->writers && !page_index->page_count) {
+ can_delete_metric = 1;
+ }
+ uv_rwlock_wrunlock(&page_index->lock);
+
+ return can_delete_metric;
}
/* Returns 1 if the data collection interval is well defined, 0 otherwise */
@@ -577,21 +664,17 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
time_t rrdeng_metric_latest_time(RRDDIM *rd)
{
- struct rrdeng_collect_handle *handle;
struct pg_cache_page_index *page_index;
- handle = &rd->state->handle.rrdeng;
- page_index = handle->page_index;
+ page_index = rd->state->page_index;
return page_index->latest_time / USEC_PER_SEC;
}
time_t rrdeng_metric_oldest_time(RRDDIM *rd)
{
- struct rrdeng_collect_handle *handle;
struct pg_cache_page_index *page_index;
- handle = &rd->state->handle.rrdeng;
- page_index = handle->page_index;
+ page_index = rd->state->page_index;
return page_index->oldest_time / USEC_PER_SEC;
}
@@ -765,7 +848,8 @@ void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle)
/*
* Returns 0 on success, negative on error
*/
-int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb, unsigned disk_space_mb)
+int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
+ unsigned disk_space_mb)
{
struct rrdengine_instance *ctx;
int error;
@@ -776,8 +860,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
/* reserve RRDENG_FD_BUDGET_PER_INSTANCE file descriptors for this instance */
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, RRDENG_FD_BUDGET_PER_INSTANCE);
if (rrdeng_reserved_file_descriptors > max_open_files) {
- error("Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
- (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
+ error(
+ "Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
+ (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
rrd_stat_atomic_add(&global_fs_errors, 1);
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
@@ -804,6 +889,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';
ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
ctx->metric_API_max_producers = 0;
+ ctx->quiesce = NO_QUIESCE;
+ ctx->metalog_ctx = NULL; /* only set this after the metadata log has finished initializing */
+ ctx->host = host;
memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
ctx->worker_config.ctx = ctx;
@@ -823,6 +911,11 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
if (ctx->worker_config.error) {
goto error_after_rrdeng_worker;
}
+ error = metalog_init(ctx);
+ if(error) {
+ error("Failed to initialize metadata log file event loop.");
+ goto error_after_rrdeng_worker;
+ }
return 0;
error_after_rrdeng_worker:
@@ -855,6 +948,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
assert(0 == uv_thread_join(&ctx->worker_config.thread));
finalize_rrd_files(ctx);
+ metalog_exit(ctx->metalog_ctx);
free_page_cache(ctx);
if (ctx != &default_global_ctx) {
@@ -863,3 +957,23 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
return 0;
}
+
+void rrdeng_prepare_exit(struct rrdengine_instance *ctx)
+{
+ struct rrdeng_cmd cmd;
+
+ if (NULL == ctx) {
+ return;
+ }
+
+ init_completion(&ctx->rrdengine_completion);
+ cmd.opcode = RRDENG_QUIESCE;
+ rrdeng_enq_cmd(&ctx->worker_config, &cmd);
+
+ /* wait for dbengine to quiesce */
+ wait_for_completion(&ctx->rrdengine_completion);
+ destroy_completion(&ctx->rrdengine_completion);
+
+ metalog_prepare_exit(ctx->metalog_ctx);
+}
+
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index 1f5be09d77..42382717ea 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -28,10 +28,17 @@ extern void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_pag
extern void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **handle);
extern void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle);
extern void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle);
+
+extern void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid);
+extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
+ uuid_t *ret_uuid);
+
+
+extern void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid);
extern void rrdeng_store_metric_init(RRDDIM *rd);
extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number number);
-extern void rrdeng_store_metric_finalize(RRDDIM *rd);
+extern int rrdeng_store_metric_finalize(RRDDIM *rd);
extern unsigned
rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp);
@@ -45,9 +52,10 @@ extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
extern void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
/* must call once before using anything */
-extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
+extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
unsigned disk_space_mb);
extern int rrdeng_exit(struct rrdengine_instance *ctx);
+extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
#endif /* NETDATA_RRDENGINEAPI_H */
\ No newline at end of file
diff --git a/database/engine/rrdenginelib.c b/database/engine/rrdenginelib.c
index dffcf9c347..26696fb25d 100644
--- a/database/engine/rrdenginelib.c
+++ b/database/engine/rrdenginelib.c
@@ -78,17 +78,22 @@ int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size)
return 0;
}
-/*
- * Tries to open a file in direct I/O mode, falls back to buffered mode if not possible.
- * Returns UV error number that is < 0 on failure.
- * On success sets (*file) to be the uv_file that was opened.
+/**
+ * Open file for I/O.
+ *
+ * @param path The full path of the file.
+ * @param flags Same flags as the open() system call uses.
+ * @param file On success sets (*file) to be the uv_file that was opened.
+ * @param direct Tries to open a file in direct I/O mode when direct=1, falls back to buffered mode if not possible.
+ * @return Returns UV error number that is < 0 on failure. 0 on success.
*/
-int open_file_direct_io(char *path, int flags, uv_file *file)
+int open_file_for_io(char *path, int flags, uv_file *file, int direct)
{
uv_fs_t req;
- int fd, current_flags, direct;
+ int fd, current_flags;
- for (direct = 1 ; direct >= 0 ; --direct) {
+ assert(0 == direct || 1 == direct);
+ for ( ; direct >= 0 ; --direct) {
#ifdef __APPLE__
/* Apple OS does not support O_DIRECT */
direct = 0;
diff --git a/database/engine/rrdenginelib.h b/database/engine/rrdenginelib.h
index 91836b63b9..7962cbc018 100644
--- a/database/engine/rrdenginelib.h
+++ b/database/engine/rrdenginelib.h
@@ -100,7 +100,15 @@ static inline void crc32set(void *crcp, uLong crc)
extern void print_page_cache_descr(struct rrdeng_page_descr *page_cache_descr);
extern void print_page_descr(struct rrdeng_page_descr *descr);
extern int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size);
-extern int open_file_direct_io(char *path, int flags, uv_file *file);
+extern int open_file_for_io(char *path, int flags, uv_file *file, int direct);
+static inline int open_file_direct_io(char *path, int flags, uv_file *file)
+{
+ return open_file_for_io(path, flags, file, 1);
+}
+static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
+{
+ return open_file_for_io(path, flags, file, 0);
+}
extern char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t size);
#endif /* NETDATA_RRDENGINELIB_H */
\ No newline at end of file
diff --git a/database/rrd.h b/database/rrd.h
index 7e1df252a8..b18c828b03 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -16,6 +16,7 @@ typedef struct alarm_entry ALARM_ENTRY;
// forward declarations
struct rrddim_volatile;
+struct rrdset_volatile;
#ifdef ENABLE_DBENGINE
struct rrdeng_page_descr;
struct rrdengine_instance;
@@ -31,6 +32,11 @@ struct pg_cache_page_index;
#include "rrdcalctemplate.h"
#include "../streaming/rrdpush.h"
+#define META_CHART_UPDATED 1
+#define META_PLUGIN_UPDATED 2
+#define META_MODULE_UPDATED 4
+#define META_CHART_ACTIVATED 8
+
#define UPDATE_EVERY 1
#define UPDATE_EVERY_MAX 3600
@@ -136,7 +142,10 @@ typedef enum rrddim_flags {
RRDDIM_FLAG_NONE = 0,
RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers
RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
- RRDDIM_FLAG_OBSOLETE = (1 << 2) // this is marked by the collector/module as obsolete
+ RRDDIM_FLAG_OBSOLETE = (1 << 2), // this is marked by the collector/module as obsolete
+ // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at
+ // least rrdset_free_obsolete_time seconds ago.
+ RRDDIM_FLAG_ARCHIVED = (1 << 3)
} RRDDIM_FLAGS;
#ifdef HAVE_C___ATOMIC
@@ -274,7 +283,6 @@ union rrddim_collect_handle {
struct rrdeng_page_descr *descr, *prev_descr;
unsigned long page_correlation_id;
struct rrdengine_instance *ctx;
- struct pg_cache_page_index *page_index;
// set to 1 when this dimension is not page aligned with the other dimensions in the chart
uint8_t unaligned_page;
} rrdeng; // state the database engine uses
@@ -312,6 +320,9 @@ struct rrddim_query_handle {
struct rrddim_volatile {
#ifdef ENABLE_DBENGINE
uuid_t *rrdeng_uuid; // database engine metric UUID
+ uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts)
+ struct pg_cache_page_index *page_index;
+ uint32_t compaction_id; // The last metadata log compaction procedure that has processed this object.
#endif
union rrddim_collect_handle handle;
// ------------------------------------------------------------------------
@@ -324,7 +335,8 @@ struct rrddim_volatile {
void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);
// an finalization function to run after collection is over
- void (*finalize)(RRDDIM *rd);
+ // returns 1 if it's safe to delete the dimension
+ int (*finalize)(RRDDIM *rd);
} collect_ops;
// function pointers that handle database queries
@@ -349,6 +361,14 @@ struct rrddim_volatile {
} query_ops;
};
+// ----------------------------------------------------------------------------
+// volatile state per chart
+struct rrdset_volatile {
+ char *old_title;
+ char *old_family;
+ char *old_context;
+};
+
// ----------------------------------------------------------------------------
// these loop macros make sure the linked list is accessed with the right lock
@@ -382,7 +402,10 @@ typedef enum rrdset_flags {
RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous
RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for backends
RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
- RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14 // this is marked by the collector/module when a chart has obsolete dimensions
+ RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions
+ // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
+ // least rrdset_free_obsolete_time seconds ago.
+ RRDSET_FLAG_ARCHIVED = 1 << 15
} RRDSET_FLAGS;
#ifdef HAVE_C___ATOMIC
@@ -459,8 +482,11 @@ struct rrdset {
char *plugin_name; // the name of the plugin that generated this
char *module_name; // the name of the plugin module that generated this
-
- size_t unused[5];
+ uuid_t *chart_uuid; // Store the global GUID for this chart
+ size_t compaction_id; // The last metadata log compaction procedure that has processed
+ // this object.
+ struct rrdset_volatile *state; // volatile state that is not persistently stored
+ size_t unused[2];
size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
@@ -783,6 +809,10 @@ struct rrdhost {
#ifdef ENABLE_DBENGINE
struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
+ uuid_t host_uuid; // Global GUID for this host
+ unsigned long objects_nr; // Number of charts and dimensions in this host
+ uint32_t compaction_id; // The last metadata log compaction procedure that has processed
+ // this object.
#endif
#ifdef ENABLE_HTTPS
@@ -847,6 +877,26 @@ extern RRDHOST *rrdhost_find_or_create(
, struct rrdhost_system_info *system_info
);
+extern void rrdhost_update(RRDHOST *host
+ , const char *hostname
+ , const char *registry_hostname
+ , const char *guid
+ , const char *os
+ , const char *timezone
+ , const char *tags
+ , const char *program_name
+ , const char *program_version
+ , int update_every
+ , long history
+ , RRD_MEMORY_MODE mode
+ , unsigned int health_enabled
+ , unsigned int rrdpush_enabled
+ , char *rrdpush_destination
+ , char *rrdpush_api_key
+ , char *rrdpush_send_charts_matching
+ , struct rrdhost_system_info *system_info
+);
+
extern int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);
#if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)
@@ -892,10 +942,12 @@ extern RRDSET *rrdset_create_custom(RRDHOST *host
, int update_every
, RRDSET_TYPE chart_type
, RRD_MEMORY_MODE memory_mode
- , long history_entries);
+ , long history_entries
+ , int is_archived
+ , uuid_t *chart_uuid);
#define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
- rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
+ rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries, 0, NULL)
#define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
@@ -919,6 +971,14 @@ extern RRDSET *rrdset_find(RRDHOST *host, const char *id);
extern RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
+/* This will not return charts that are archived */
+static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
+{
+ RRDSET *st = rrdset_find_bytype_localhost(type, id);
+ if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
+ return NULL;
+ return st;
+}
extern RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
@@ -933,8 +993,9 @@ extern void rrdset_is_obsolete(RRDSET *st);
extern void rrdset_isnot_obsolete(RRDSET *st);
// checks if the RRDSET should be offered to viewers
-#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
-#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions)
+#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
+#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
+#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
// get the total duration in seconds of the round robin database
#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
@@ -1062,8 +1123,11 @@ static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
// RRD DIMENSION functions
extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host);
-extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);
-#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
+extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
+ collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
+ int is_archived, uuid_t *dim_uuid);
+#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \
+ algorithm, (st)->rrd_memory_mode, 0, NULL)
extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
@@ -1071,6 +1135,15 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
extern RRDDIM *rrddim_find(RRDSET *st, const char *id);
+/* This will not return dimensions that are archived */
+static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id)
+{
+ RRDDIM *rd = rrddim_find(st, id);
+ if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
+ return NULL;
+ return rd;
+}
+
extern int rrddim_hide(RRDSET *st, const char *id);
extern int rrddim_unhide(RRDSET *st, const char *id);
@@ -1099,7 +1172,8 @@ extern avl_tree_lock rrdhost_root_index;
extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
extern char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section);
-extern void rrddim_free(RRDSET *st, RRDDIM *rd);
+#define rrddim_free(st, rd) rrddim_free_custom(st, rd, 0)
+extern void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated);
extern int rrddim_compare(void* a, void* b);
extern int rrdset_compare(void* a, void* b);
@@ -1116,7 +1190,8 @@ extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
extern void rrdset_free(RRDSET *st);
extern void rrdset_reset(RRDSET *st);
extern void rrdset_save(RRDSET *st);
-extern void rrdset_delete(RRDSET *st);
+#define rrdset_delete(st) rrdset_delete_custom(st, 0)
+extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
diff --git a/database/rrddim.c b/database/rrddim.c
index 3e94021e78..27769b99d5 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -100,10 +100,10 @@ static void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storag
rd->values[rd->rrdset->current_entry] = number;
}
-static void rrddim_collect_finalize(RRDDIM *rd) {
+static int rrddim_collect_finalize(RRDDIM *rd) {
(void)rd;
- return;
+ return 0;
}
// ----------------------------------------------------------------------------
@@ -189,7 +189,9 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
#endif
}
-RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) {
+RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
+ collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
+ int is_archived, uuid_t *dim_uuid) {
RRDHOST *host = st->rrdhost;
rrdset_wrlock(st);
@@ -200,11 +202,21 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
if(unlikely(rd)) {
debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
- rrddim_set_name(st, rd, name);
- rrddim_set_algorithm(st, rd, algorithm);
- rrddim_set_multiplier(st, rd, multiplier);
- rrddim_set_divisor(st, rd, divisor);
-
+ int rc = rrddim_set_name(st, rd, name);
+ rc += rrddim_set_algorithm(st, rd, algorithm);
+ rc += rrddim_set_multiplier(st, rd, multiplier);
+ rc += rrddim_set_divisor(st, rd, divisor);
+ if (!is_archived && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
+ rd->state->collect_ops.init(rd);
+ rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
+ }
+ // DBENGINE available and activated?
+#ifdef ENABLE_DBENGINE
+ if (likely(!is_archived && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) {
+ debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
+ metalog_commit_update_dimension(rd);
+ }
+#endif
rrdset_unlock(st);
return rd;
}
@@ -301,7 +313,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
else
rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
-
rd->memsize = size;
strcpy(rd->magic, RRDDIMENSION_MAGIC);
@@ -350,15 +361,16 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rd->state = mallocz(sizeof(*rd->state));
if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
- rd->state->collect_ops.init = rrdeng_store_metric_init;
+ rrdeng_metric_init(rd, dim_uuid);
+ rd->state->collect_ops.init = rrdeng_store_metric_init;
rd->state->collect_ops.store_metric = rrdeng_store_metric_next;
- rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
- rd->state->query_ops.init = rrdeng_load_metric_init;
- rd->state->query_ops.next_metric = rrdeng_load_metric_next;
- rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
- rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
- rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
- rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
+ rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
+ rd->state->query_ops.init = rrdeng_load_metric_init;
+ rd->state->query_ops.next_metric = rrdeng_load_metric_next;
+ rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
+ rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
+ rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
+ rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
#endif
} else {
rd->state->collect_ops.init = rrddim_collect_init;
@@ -371,7 +383,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rd->state->query_ops.latest_time = rrddim_query_latest_time;
rd->state->query_ops.oldest_time = rrddim_query_oldest_time;
}
- rd->state->collect_ops.init(rd);
+ if (is_archived)
+ rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
+ else
+ rd->state->collect_ops.init(rd); // only initialize if a collector created this dimension
// append this dimension
if(!st->dimensions)
st->dimensions = rd;
@@ -432,17 +447,30 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
if (netdata_cloud_setting)
aclk_update_chart(host, st->id, ACLK_CMD_CHART);
#endif
+#ifdef ENABLE_DBENGINE
+ rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
+ metalog_commit_update_dimension(rd);
+#endif
+
return(rd);
}
// ----------------------------------------------------------------------------
// RRDDIM remove / free a dimension
-void rrddim_free(RRDSET *st, RRDDIM *rd)
+void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
{
debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
- rd->state->collect_ops.finalize(rd);
+ if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
+ uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
+ if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+#ifdef ENABLE_DBENGINE
+ /* This metric has no data and no references */
+ metalog_commit_delete_dimension(rd);
+#endif
+ }
+ }
freez(rd->state);
if(rd == st->dimensions)
@@ -486,9 +514,12 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
break;
}
#ifdef ENABLE_ACLK
- if (netdata_cloud_setting)
+ if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != rd->rrd_memory_mode))
aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
#endif
+#ifdef ENABLE_DBENGINE
+ rrd_atomic_fetch_add(&st->rrdhost->objects_nr, -1);
+#endif
}
@@ -541,6 +572,10 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
if (netdata_cloud_setting)
aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
#endif
+#ifdef ENABLE_DBENGINE
+ metalog_commit_update_dimension(rd);
+#endif
+
}
inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
@@ -551,6 +586,9 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
if (netdata_cloud_setting)
aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
#endif
+#ifdef ENABLE_DBENGINE
+ metalog_commit_update_dimension(rd);
+#endif
}
// ----------------------------------------------------------------------------
diff --git a/database/rrdhost.c b/database/rrdhost.c
index c4613e3642..9c51f361ad 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -245,6 +245,9 @@ RRDHOST *rrdhost_create(const char *hostname,
}
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
+ uuid_parse(host->machine_guid, host->host_uuid);
+ host->objects_nr = 1;
+ host->compaction_id = 0;
char dbenginepath[FILENAME_MAX + 1];
int ret;
@@ -253,7 +256,7 @@ RRDHOST *rrdhost_create(const char *hostname,
if(ret != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
else
- ret = rrdeng_init(&host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
+ ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
if(ret) {
error("Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
host->hostname, host->machine_guid, host->cache_dir);
@@ -361,9 +364,91 @@ RRDHOST *rrdhost_create(const char *hostname,
rrd_hosts_available++;
+#ifdef ENABLE_DBENGINE
+ if (likely(!is_localhost && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
+ metalog_commit_update_host(host);
+#endif
return host;
}
+void rrdhost_update(RRDHOST *host
+ , const char *hostname
+ , const char *registry_hostname
+ , const char *guid
+ , const char *os
+ , const char *timezone
+ , const char *tags
+ , const char *program_name
+ , const char *program_version
+ , int update_every
+ , long history
+ , RRD_MEMORY_MODE mode
+ , unsigned int health_enabled
+ , unsigned int rrdpush_enabled
+ , char *rrdpush_destination
+ , char *rrdpush_api_key
+ , char *rrdpush_send_charts_matching
+ , struct rrdhost_system_info *system_info
+)
+{
+ UNUSED(guid);
+ UNUSED(rrdpush_enabled);
+ UNUSED(rrdpush_destination);
+ UNUSED(rrdpush_api_key);
+ UNUSED(rrdpush_send_charts_matching);
+
+ host->health_enabled = health_enabled;
+ //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
+
+ rrdhost_system_info_free(host->system_info);
+ host->system_info = system_info;
+
+ rrdhost_init_os(host, os);
+ rrdhost_init_timezone(host, timezone);
+
+ freez(host->registry_hostname);
+ host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
+
+ if(strcmp(host->hostname, hostname) != 0) {
+ info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
+ char *t = host->hostname;
+ host->hostname = strdupz(hostname);
+ host->hash_hostname = simple_hash(host->hostname);
+ freez(t);
+ }
+
+ if(strcmp(host->program_name, program_name) != 0) {
+ info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
+ char *t = host->program_name;
+ host->program_name = strdupz(program_name);
+ freez(t);
+ }
+
+ if(strcmp(host->program_version, program_version) != 0) {
+ info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
+ char *t = host->program_version;
+ host->program_version = strdupz(program_version);
+ freez(t);
+ }
+
+ if(host->rrd_update_every != update_every)
+ error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
+
+ if(host->rrd_history_entries < history)
+ error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
+
+ if(host->rrd_memory_mode != mode)
+ error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
+
+ // update host tags
+ rrdhost_init_tags(host, tags);
+#ifdef ENABLE_DBENGINE
+ if (likely(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
+ metalog_commit_update_host(host);
+#endif
+ return;
+}
+
RRDHOST *rrdhost_find_or_create(
const char *hostname
, const char *registry_hostname
@@ -410,42 +495,24 @@ RRDHOST *rrdhost_find_or_create(
);
}
else {
- host->health_enabled = health_enabled;
- //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
-
- if(strcmp(host->hostname, hostname) != 0) {
- info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
- char *t = host->hostname;
- host->hostname = strdupz(hostname);
- host->hash_hostname = simple_hash(host->hostname);
- freez(t);
- }
-
- if(strcmp(host->program_name, program_name) != 0) {
- info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
- char *t = host->program_name;
- host->program_name = strdupz(program_name);
- freez(t);
- }
-
- if(strcmp(host->program_version, program_version) != 0) {
- info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
- char *t = host->program_version;
- host->program_version = strdupz(program_version);
- freez(t);
- }
-
- if(host->rrd_update_every != update_every)
- error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
-
- if(host->rrd_history_entries < history)
- error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
-
- if(host->rrd_memory_mode != mode)
- error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
-
- // update host tags
- rrdhost_init_tags(host, tags);
+ rrdhost_update(host
+ , hostname
+ , registry_hostname
+ , guid
+ , os
+ , timezone
+ , tags
+ , program_name
+ , program_version
+ , update_every
+ , history
+ , mode
+ , health_enabled
+ , rrdpush_enabled
+ , rrdpush_destination
+ , rrdpush_api_key
+ , rrdpush_send_charts_matching
+ , system_info);
}
rrdhost_cleanup_orphan_hosts_nolock(host);
@@ -454,7 +521,6 @@ RRDHOST *rrdhost_find_or_create(
return host;
}
-
inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) {
if(host != protected
&& host != localhost
@@ -618,6 +684,9 @@ void rrdhost_free(RRDHOST *host) {
// ------------------------------------------------------------------------
// release its children resources
+#ifdef ENABLE_DBENGINE
+ rrdeng_prepare_exit(host->rrdeng_ctx);
+#endif
while(host->rrdset_root)
rrdset_free(host->rrdset_root);
@@ -1322,7 +1391,17 @@ restart_after_removal:
&& st->last_updated.tv_sec + rrdset_free_obsolete_time < now
&& st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now
)) {
-
+#ifdef ENABLE_DBENGINE
+ if(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
+ rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+ if (st->dimensions) {
+ /* If the chart still has dimensions don't delete it from the metadata log */
+ continue;
+ }
+ metalog_commit_delete_chart(st);
+ }
+#endif
rrdset_rdlock(st);
if(rrdhost_delete_obsolete_charts)
diff --git a/database/rrdset.c b/database/rrdset.c
index d0554f0f30..7819c2a62e 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -2,6 +2,7 @@
#define NETDATA_RRD_INTERNALS
#include "rrd.h"
+#include <sched.h>
void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id);
@@ -260,7 +261,7 @@ void rrdset_reset(RRDSET *st) {
rd->collections_counter = 0;
// memset(rd->values, 0, rd->entries * sizeof(storage_number));
#ifdef ENABLE_DBENGINE
- if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode) {
+ if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode && !rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
rrdeng_store_metric_flush_current_page(rd);
}
#endif
@@ -366,6 +367,10 @@ void rrdset_free(RRDSET *st) {
freez(st->config_section);
freez(st->plugin_name);
freez(st->module_name);
+ freez(st->state->old_title);
+ freez(st->state->old_family);
+ freez(st->state->old_context);
+ freez(st->state);
switch(st->rrd_memory_mode) {
case RRD_MEMORY_MODE_SAVE:
@@ -381,6 +386,10 @@ void rrdset_free(RRDSET *st) {
freez(st);
break;
}
+#ifdef ENABLE_DBENGINE
+ rrd_atomic_fetch_add(&host->objects_nr, -1);
+#endif
+
}
void rrdset_save(RRDSET *st) {
@@ -402,7 +411,7 @@ void rrdset_save(RRDSET *st) {
}
}
-void rrdset_delete(RRDSET *st) {
+void rrdset_delete_custom(RRDSET *st, int db_rotated) {
RRDDIM *rd;
rrdset_check_rdlock(st);
@@ -425,11 +434,12 @@ void rrdset_delete(RRDSET *st) {
recursively_delete_dir(st->cache_dir, "left-over chart");
#ifdef ENABLE_ACLK
- if (netdata_cloud_setting) {
+ if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)) {
aclk_del_collector(st->rrdhost->hostname, st->plugin_name, st->module_name);
aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHARTDEL);
}
#endif
+
}
void rrdset_delete_obsolete_dimensions(RRDSET *st) {
@@ -480,6 +490,8 @@ RRDSET *rrdset_create_custom(
, RRDSET_TYPE chart_type
, RRD_MEMORY_MODE memory_mode
, long history_entries
+ , int is_archived
+ , uuid_t *chart_uuid
) {
if(!type || !type[0]) {
fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
@@ -516,15 +528,139 @@ RRDSET *rrdset_create_custom(
snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
RRDSET *st = rrdset_find_on_create(host, fullid);
- if(st) {
+ if (st) {
+ int mark_rebuild = 0;
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+ if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
+ rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
+ mark_rebuild |= META_CHART_ACTIVATED;
+ }
+ char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
+ *old_title_v = NULL, *old_family_v = NULL, *old_context_v = NULL;
+ const char *new_name = name ? name : id;
- if(unlikely(name))
- rrdset_set_name(st, name);
- else
- rrdset_set_name(st, id);
+ if (unlikely((st->name && !strcmp(st->name, new_name)) || !st->name)) {
+ mark_rebuild |= META_CHART_UPDATED;
+ rrdset_set_name(st, new_name);
+ }
+ if (unlikely(st->priority != priority)) {
+ st->priority = priority;
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+ if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != update_every)) {
+ st->update_every = update_every;
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+
+ if (plugin && st->plugin_name) {
+ if (unlikely(strcmp(plugin, st->plugin_name))) {
+ old_plugin = st->plugin_name;
+ st->plugin_name = strdupz(plugin);
+ mark_rebuild |= META_PLUGIN_UPDATED;
+ }
+ } else {
+ if (plugin != st->plugin_name) { // one is NULL?
+ old_plugin = st->plugin_name;
+ st->plugin_name = plugin ? strdupz(plugin) : NULL;
+ mark_rebuild |= META_PLUGIN_UPDATED;
+ }
+ }
+
+ if (module && st->module_name) {
+ if (unlikely(strcmp(module, st->module_name))) {
+ old_module = st->module_name;
+ st->module_name = strdupz(module);
+ mark_rebuild |= META_MODULE_UPDATED;
+ }
+ } else {
+ if (module != st->module_name) {
+ if (st->module_name && *st->module_name) {
+ old_module = st->module_name;
+ st->module_name = module ? strdupz(module) : NULL;
+ mark_rebuild |= META_MODULE_UPDATED;
+ }
+ }
+ }
+
+ if (unlikely(title && st->state->old_title && strcmp(st->state->old_title, title))) {
+ char *new_title = strdupz(title);
+ old_title_v = st->state->old_title;
+ st->state->old_title = strdupz(title);
+ json_fix_string(new_title);
+ old_title = st->title;
+ st->title = new_title;
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+
+ RRDSET_TYPE new_chart_type =
+ rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
+ if (st->chart_type != new_chart_type) {
+ st->chart_type = new_chart_type;
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+
+ if (unlikely(family && st->state->old_family && strcmp(st->state->old_family, family))) {
+ char *new_family = strdupz(family);
+ old_family_v = st->state->old_family;
+ st->state->old_family = strdupz(family);
+ json_fix_string(new_family);
+ old_family = st->family;
+ rrdfamily_free(host, st->rrdfamily);
+ st->family = new_family;
+ st->rrdfamily = rrdfamily_create(host, st->family);
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+
+ if (unlikely(context && st->state->old_context && strcmp(st->state->old_context, context))) {
+ char *new_context = strdupz(context);
+ old_context_v = st->state->old_context;
+ st->state->old_context = strdupz(context);
+ json_fix_string(new_context);
+ old_context = st->context;
+ st->context = new_context;
+ st->hash_context = simple_hash(st->context);
+ mark_rebuild |= META_CHART_UPDATED;
+ }
+
+ if (mark_rebuild) {
+#ifdef ENABLE_ACLK
+ if (netdata_cloud_setting) {
+ if (mark_rebuild & META_CHART_ACTIVATED) {
+ aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
+ }
+ else {
+ if (mark_rebuild & (META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
+ aclk_del_collector(
+ host->hostname, mark_rebuild & META_PLUGIN_UPDATED ? old_plugin : st->plugin_name,
+ mark_rebuild & META_MODULE_UPDATED ? old_module : st->module_name);
+ aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
+ }
+ }
+ aclk_update_chart(host, st->id, ACLK_CMD_CHART);
+ }
+#endif
+ freez(old_plugin);
+ freez(old_module);
+ freez(old_title);
+ freez(old_family);
+ freez(old_context);
+ freez(old_title_v);
+ freez(old_family_v);
+ freez(old_context_v);
+ if (mark_rebuild != META_CHART_ACTIVATED) {
+ info("Collector updated metadata for chart %s", st->id);
+ sched_yield();
+ }
+ }
+#ifdef ENABLE_DBENGINE
+ if (is_archived == 0 && st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
+ (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) {
+ debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
+ metalog_commit_update_chart(st);
+ }
+#endif
return st;
}
@@ -535,6 +671,9 @@ RRDSET *rrdset_create_custom(
rrdhost_unlock(host);
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+ if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
+ rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
+ }
return st;
}
@@ -664,6 +803,8 @@ RRDSET *rrdset_create_custom(
else
st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
+ if (is_archived)
+ rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
@@ -687,13 +828,16 @@ RRDSET *rrdset_create_custom(
st->chart_type = rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
st->type = config_get(st->config_section, "type", type);
+ st->state = mallocz(sizeof(*st->state));
st->family = config_get(st->config_section, "family", family?family:st->type);
+ st->state->old_family = strdupz(st->family);
json_fix_string(st->family);
st->units = config_get(st->config_section, "units", units?units:"");
json_fix_string(st->units);
st->context = config_get(st->config_section, "context", context?context:st->id);
+ st->state->old_context = strdupz(st->context);
json_fix_string(st->context);
st->hash_context = simple_hash(st->context);
@@ -745,6 +889,7 @@ RRDSET *rrdset_create_custom(
rrdset_set_name(st, id);
st->title = config_get(st->config_section, "title", title);
+ st->state->old_title = strdupz(st->title);
json_fix_string(st->title);
st->rrdfamily = rrdfamily_create(host, st->family);
@@ -765,6 +910,26 @@ RRDSET *rrdset_create_custom(
rrdsetcalc_link_matching(st);
rrdcalctemplate_link_matching(st);
+#ifdef ENABLE_DBENGINE
+ if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ int replace_instead_of_generate = 0;
+
+ st->chart_uuid = callocz(1, sizeof(uuid_t));
+ if (NULL != chart_uuid) {
+ replace_instead_of_generate = 1;
+ uuid_copy(*st->chart_uuid, *chart_uuid);
+ }
+ if (unlikely(
+ find_or_generate_guid((void *) st, st->chart_uuid, GUID_TYPE_CHART, replace_instead_of_generate))) {
+ errno = 0;
+ error("FAILED to generate GUID for %s", st->id);
+ freez(st->chart_uuid);
+ st->chart_uuid = NULL;
+ assert(0);
+ }
+ st->compaction_id = 0;
+ }
+#endif
rrdhost_cleanup_obsolete_charts(host);
@@ -775,6 +940,11 @@ RRDSET *rrdset_create_custom(
aclk_update_chart(host, st->id, ACLK_CMD_CHART);
}
#endif
+#ifdef ENABLE_DBENGINE
+ rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
+ metalog_commit_update_chart(st);
+#endif
+
return(st);
}
@@ -1006,6 +1176,9 @@ static inline size_t rrdset_done_interpolate(
last_ut = next_store_ut;
rrddim_foreach_read(rd, st) {
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+ continue;
+
calculated_number new_value;
switch(rd->algorithm) {
@@ -1374,6 +1547,8 @@ void rrdset_done(RRDSET *st) {
int dimensions = 0;
st->collected_total = 0;
rrddim_foreach_read(rd, st) {
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+ continue;
dimensions++;
if(likely(rd->updated))
st->collected_total += rd->collected_value;
@@ -1385,6 +1560,8 @@ void rrdset_done(RRDSET *st) {
// based on the collected figures only
// at this stage we do not interpolate anything
rrddim_foreach_read(rd, st) {
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+ continue;
if(unlikely(!rd->updated)) {
rd->calculated_value = 0;
@@ -1630,6 +1807,8 @@ void rrdset_done(RRDSET *st) {
st->last_collected_total = st->collected_total;
rrddim_foreach_read(rd, st) {
+ if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+ continue;
if(unlikely(!rd->updated))
continue;
@@ -1692,7 +1871,7 @@ void rrdset_done(RRDSET *st) {
// find if there are any obsolete dimensions
time_t now = now_realtime_sec();
- if(unlikely(rrddim_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
rrddim_foreach_read(rd, st)
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)))
break;
@@ -1714,6 +1893,21 @@ void rrdset_done(RRDSET *st) {
error("Cannot delete dimension file '%s'", rd->cache_filename);
}
+#ifdef ENABLE_DBENGINE
+ if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
+ rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
+ /* only a collector can mark a chart as obsolete, so we must remove the reference */
+ uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
+ if (can_delete_metric) {
+ /* This metric has no data and no references */
+ metalog_commit_delete_dimension(rd);
+ } else {
+ /* Do not delete this dimension */
+ continue;
+ }
+ }
+#endif
if(unlikely(!last)) {
rrddim_free(st, rd);
rd = st->dimensions;
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index f4da7769f6..f24c746618 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -100,7 +100,9 @@ RRDSET *rrdset_create_custom(
int update_every,
RRDSET_TYPE chart_type,
RRD_MEMORY_MODE memory_mode,
- long history_entries)
+ long history_entries,
+ int is_archived,
+ uuid_t *chart_uuid)
{
check_expected_ptr(host);
check_expected_ptr(type);
@@ -117,6 +119,8 @@ RRDSET *rrdset_create_custom(
check_expected(chart_type);
UNUSED(memory_mode);
UNUSED(history_entries);
+ UNUSED(is_archived);
+ UNUSED(chart_uuid);
function_called();
diff --git a/health/health.c b/health/health.c
index 0bf79e27d1..a0cbd884bc 100644
--- a/health/health.c
+++ b/health/health.c
@@ -488,6 +488,11 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
return 0;
}
+ if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) {
+ debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rc->chart?rc->chart:"NOCHART", rc->name);
+ return 0;
+ }
+
if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
return 0;
diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h
index 582ebafe03..8b54f28de1 100644
--- a/libnetdata/log/log.h
+++ b/libnetdata/log/log.h
@@ -38,6 +38,8 @@
#define D_STREAM 0x0000000040000000
#define D_RRDENGINE 0x0000000100000000
#define D_ACLK 0x0000000200000000
+#define D_METADATALOG 0x0000000400000000
+#define D_GUIDLOG 0x0000000800000000
#define D_SYSTEM 0x8000000000000000
//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
diff --git a/parser/parser.h b/parser/parser.h
index 9c16fbf12a..64127a71d1 100644
--- a/parser/parser.h
+++ b/parser/parser.h
@@ -31,6 +31,10 @@ typedef struct pluginsd_action {
PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
PARSER_RC (*label_action)(void *user, char *key, char *value, LABEL_SOURCE source);
PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, struct label *new_labels);
+
+ PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
+ PARSER_RC (*context_action)(void *user, uuid_t *uuid);
+ PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid);
} PLUGINSD_ACTION;
typedef enum parser_input_type {
@@ -101,5 +105,8 @@ extern PARSER_RC pluginsd_flush(char **words, void *user, PLUGINSD_ACTION *plug
extern PARSER_RC pluginsd_disable(char **words, void *user, PLUGINSD_ACTION *plugins_action);
extern PARSER_RC pluginsd_label(char **words, void *user, PLUGINSD_ACTION *plugins_action);
extern PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins_action);
+extern PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action);
+extern PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action);
+extern PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action);
#endif
diff --git a/streaming/receiver.c b/streaming/receiver.c
index 380d49137c..c8e10330e1 100644
--- a/streaming/receiver.c
+++ b/streaming/receiver.c
@@ -279,6 +279,25 @@ static int rrdpush_receive(struct receiver_state *rpt)
}
netdata_mutex_unlock(&rpt->host->receiver_lock);
}
+ else rrdhost_update(rpt->host
+ , rpt->hostname
+ , rpt->registry_hostname
+ , rpt->machine_guid
+ , rpt->os
+ , rpt->timezone
+ , rpt->tags
+ , program_name
+ , program_version
+ , rpt->update_every
+ , history
+ , mode
+ , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
+ , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
+ , rrdpush_destination
+ , rrdpush_api_key
+ , rrdpush_send_charts_matching
+ , rpt->system_info);
+
int ssl = 0;
#ifdef ENABLE_HTTPS
diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c
index 864387ed51..9ea74df0ba 100644
--- a/web/api/formatters/charts2json.c
+++ b/web/api/formatters/charts2json.c
@@ -36,7 +36,7 @@ static inline const char* get_release_channel() {
return (use_stable)?"stable":"nightly";
}
-void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
+void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived) {
static char *custom_dashboard_info_js_filename = NULL;
size_t c, dimensions = 0, memory = 0, alarms = 0;
RRDSET *st;
@@ -71,7 +71,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
c = 0;
rrdhost_rdlock(host);
rrdset_foreach_read(st, host) {
- if(rrdset_is_available_for_viewers(st)) {
+ if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
if(c) buffer_strcat(wb, ",");
buffer_strcat(wb, "\n\t\t\"");
buffer_strcat(wb, st->id);
diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h
index 4f0d57422a..8e3ff1ab42 100644
--- a/web/api/formatters/charts2json.h
+++ b/web/api/formatters/charts2json.h
@@ -5,7 +5,7 @@
#include "rrd2json.h"
-extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile);
+extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
extern void chartcollectors2json(RRDHOST *host, BUFFER *wb);
#endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index b2a3028afa..be358e66bb 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -349,7 +349,16 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
buffer_flush(w->response.data);
w->response.data->contenttype = CT_APPLICATION_JSON;
- charts2json(host, w->response.data, 0);
+ charts2json(host, w->response.data, 0, 0);
+ return HTTP_RESP_OK;
+}
+
+inline int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url) {
+ (void)url;
+
+ buffer_flush(w->response.data);
+ w->response.data->contenttype = CT_APPLICATION_JSON;
+ charts2json(host, w->response.data, 0, 1);
return HTTP_RESP_OK;
}
@@ -915,6 +924,7 @@ static struct api_command {
{ "data", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_data },
{ "chart", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_chart },
{ "charts", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_charts },
+ { "archivedcharts", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_archivedcharts },
// registry checks the ACL by itself, so we allow everything
{ "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },
diff --git a/web/api/web_api_v1.h b/web/api/web_api_v1.h
index 2748023851..445b0e4a5c 100644
--- a/web/api/web_api_v1.h
+++ b/web/api/web_api_v1.h
@@ -19,6 +19,7 @@ extern int web_client_api_request_single_chart(RRDHOST *host, struct web_client
extern int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
extern int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
extern int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
extern int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
extern int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
extern int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);