Add support for persistent metadata (#9324)

* Implemented collector metadata logging 
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update

Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>

CMakeLists.txt

@@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES
         database/engine/pagecache.h
         database/engine/rrdenglocking.c
         database/engine/rrdenglocking.h
+        database/engine/metadata_log/metadatalog.c
+        database/engine/metadata_log/metadatalog.h
+        database/engine/metadata_log/metadatalogapi.c
+        database/engine/metadata_log/metadatalogapi.h
+        database/engine/metadata_log/logfile.h
+        database/engine/metadata_log/logfile.c
+        database/engine/metadata_log/metadatalogprotocol.h
+        database/engine/metadata_log/metalogpluginsd.c
+        database/engine/metadata_log/metalogpluginsd.h
+        database/engine/metadata_log/compaction.c
+        database/engine/metadata_log/compaction.h
+        database/engine/global_uuid_map/global_uuid_map.c
+        database/engine/global_uuid_map/global_uuid_map.h
         )
 
 set(WEB_PLUGIN_FILES

Makefile.am

@@ -375,6 +375,19 @@ if ENABLE_DBENGINE
         database/engine/pagecache.h \
         database/engine/rrdenglocking.c \
         database/engine/rrdenglocking.h \
+        database/engine/metadata_log/metadatalog.c \
+        database/engine/metadata_log/metadatalog.h \
+        database/engine/metadata_log/metadatalogapi.c \
+        database/engine/metadata_log/metadatalogapi.h \
+        database/engine/metadata_log/logfile.h \
+        database/engine/metadata_log/logfile.c \
+        database/engine/metadata_log/metadatalogprotocol.h \
+        database/engine/metadata_log/metalogpluginsd.c \
+        database/engine/metadata_log/metalogpluginsd.h \
+        database/engine/metadata_log/compaction.c \
+        database/engine/metadata_log/compaction.h \
+        database/engine/global_uuid_map/global_uuid_map.c \
+        database/engine/global_uuid_map/global_uuid_map.h \
         $(NULL)
 endif
 

aclk/agent_cloud_link.c

@@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi
 void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha
 void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -1752,7 +1758,7 @@ int aclk_send_info_metadata()
     debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len);
 
     buffer_sprintf(local_buffer, ", \n\t \"charts\" : ");
-    charts2json(localhost, local_buffer, 1);
+    charts2json(localhost, local_buffer, 1, 0);
     buffer_sprintf(local_buffer, "\n}\n}");
     debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len);
 
@@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd)
     UNUSED(chart_name);
     return 0;
 #else
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (!netdata_cloud_setting)
         return 0;
 
@@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
 {
     BUFFER *local_buffer = NULL;
 
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (host != localhost)
         return 0;
 

collectors/diskspace.plugin/plugin_diskspace.c

@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                               netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_space)) {
             m->do_space = CONFIG_BOOLEAN_YES;
-            m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
+            m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
             if(unlikely(!m->st_space)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_inodes)) {
             m->do_inodes = CONFIG_BOOLEAN_YES;
-            m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
+            m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
             if(unlikely(!m->st_inodes)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);

collectors/freebsd.plugin/plugin_freebsd.c

@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(

collectors/macos.plugin/macos_fw.c

@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         total_disk_writes += diskstat.bytes_write;
                     }
 
-                    st = rrdset_find_bytype_localhost("disk", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
     }
 
     if (likely(do_io)) {
-        st = rrdset_find_bytype_localhost("system", "io");
+        st = rrdset_find_active_bytype_localhost("system", "io");
         if (unlikely(!st)) {
             st = rrdset_create_localhost(
                     "system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_space)) {
-                    st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_inodes)) {
-                    st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_events"

collectors/macos.plugin/macos_mach_smi.c

@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
                 error("DISABLED: system.cpu");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "cpu");
+                st = rrdset_find_active_bytype_localhost("system", "cpu");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"

collectors/macos.plugin/macos_sysctl.c

@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
                 error("DISABLED: system.load");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "load");
+                st = rrdset_find_active_bytype_localhost("system", "load");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"

collectors/plugins.d/plugins_d.h

@@ -31,10 +31,10 @@
 #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
 #define PLUGINSD_KEYWORD_LABEL "LABEL"
 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_GUID "GUID"
-#define PLUGINSD_KEYWORD_HOST "HOST"
+#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
+#define PLUGINSD_KEYWORD_HOST "HOST"
 
 
 #define PLUGINSD_LINE_MAX 1024

collectors/plugins.d/pluginsd_parser.c

@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION  *plugins
     return PARSER_RC_OK;
 }
 
+PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a GUID, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a GUID, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->guid_action) {
+        return plugins_action->guid_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a CONTEXT, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a CONTEXT, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->context_action) {
+        return plugins_action->context_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a TOMBSTONE, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a TOMBSTONE, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->tombstone_action) {
+        return plugins_action->tombstone_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
 
 // New plugins.d parser
 

collectors/plugins.d/pluginsd_parser.h

@@ -16,19 +16,23 @@ typedef struct parser_user_object {
     struct label *new_labels;
     size_t count;
     int enabled;
+    void *private; // the user can set this for private use
 } PARSER_USER_OBJECT;
 
-PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
-PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
-PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
-PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
-           char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
-PARSER_RC pluginsd_disable_action(void *user);
-PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
-PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
-                                    RRD_ALGORITHM algorithm_type);
-PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
-PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
+extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
+extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
+extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+                                       char *title, char *units, char *plugin, char *module, int priority,
+                                       int update_every, RRDSET_TYPE chart_type, char *options);
+extern PARSER_RC pluginsd_disable_action(void *user);
+extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global,
+                                          calculated_number value);
+extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+                                           long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
+extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
+extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
+
 
 #endif //NETDATA_PLUGINSD_PARSER_H

collectors/proc.plugin/plugin_proc.c

@@ -148,7 +148,7 @@ void *proc_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(

collectors/proc.plugin/proc_net_softnet_stat.c

@@ -81,7 +81,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
 
     // --------------------------------------------------------------------
 
-    st = rrdset_find_bytype_localhost("system", "softnet_stat");
+    st = rrdset_find_active_bytype_localhost("system", "softnet_stat");
     if(unlikely(!st)) {
         st = rrdset_create_localhost(
                 "system"
@@ -114,7 +114,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
             char id[50+1];
             snprintfz(id, 50, "cpu%zu_softnet_stat", l);
 
-            st = rrdset_find_bytype_localhost("cpu", id);
+            st = rrdset_find_active_bytype_localhost("cpu", id);
             if(unlikely(!st)) {
                 char title[100+1];
                 snprintfz(title, 100, "CPU%zu softnet_stat", l);

collectors/statsd.plugin/statsd.c

@@ -1461,6 +1461,8 @@ static inline RRDSET *statsd_private_rrdset_create(
             , chart_type      // chart type
             , memory_mode     // memory mode
             , history         // history
+            , 0               // not archived
+            , NULL            // no known UUID
     );
     rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
 
@@ -1999,6 +2001,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
                 , chart->chart_type         // chart type
                 , app->rrd_memory_mode      // memory mode
                 , app->rrd_history_entries  // history
+                , 0                         // not archived
+                , NULL                      // no known UUID
         );
 
         rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);

configure.ac

@@ -1437,6 +1437,8 @@ AC_CONFIG_FILES([
     daemon/Makefile
     database/Makefile
     database/engine/Makefile
+    database/engine/metadata_log/Makefile
+    database/engine/global_uuid_map/Makefile
     diagrams/Makefile
     exporting/Makefile
     exporting/graphite/Makefile

daemon/common.h

@@ -68,9 +68,15 @@
 // netdata agent cloud link
 #include "aclk/agent_cloud_link.h"
 
+// global GUID map functions
+
 // netdata agent spawn server
 #include "spawn/spawn.h"
 
+#ifdef ENABLE_DBENGINE
+#include "database/engine/global_uuid_map/global_uuid_map.h"
+#endif
+
 // the netdata deamon
 #include "daemon.h"
 #include "main.h"

daemon/main.c

@@ -1400,6 +1400,9 @@ int main(int argc, char **argv) {
     struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
     get_system_info(system_info);
 
+#ifdef ENABLE_DBENGINE
+    init_global_guid_map();
+#endif
     if(rrd_init(netdata_configured_hostname, system_info))
         fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);
 
@@ -1417,6 +1420,9 @@ int main(int argc, char **argv) {
 
     // Load host labels
     reload_host_labels();
+#ifdef ENABLE_DBENGINE
+    metalog_commit_update_host(localhost);
+#endif
 
     // ------------------------------------------------------------------------
     // spawn the threads

daemon/unit_test.c

@@ -1833,9 +1833,10 @@ int test_dbengine(void)
         }
     }
 error_out:
-    rrdeng_exit(host->rrdeng_ctx);
     rrd_wrlock();
+    rrdeng_prepare_exit(host->rrdeng_ctx);
     rrdhost_delete_charts(host);
+    rrdeng_exit(host->rrdeng_ctx);
     rrd_unlock();
 
     return errors;
@@ -2222,9 +2223,10 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
         freez(query_threads[i]);
     }
     freez(query_threads);
-    rrdeng_exit(host->rrdeng_ctx);
     rrd_wrlock();
+    rrdeng_prepare_exit(host->rrdeng_ctx);
     rrdhost_delete_charts(host);
+    rrdeng_exit(host->rrdeng_ctx);
     rrd_unlock();
 }
 

database/engine/Makefile.am

@@ -3,6 +3,11 @@
 AUTOMAKE_OPTIONS = subdir-objects
 MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
+SUBDIRS = \
+    metadata_log \
+    global_uuid_map \
+    $(NULL)
+
 dist_noinst_DATA = \
     README.md \
     $(NULL)

database/engine/global_uuid_map/Makefile.am

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)

database/engine/global_uuid_map/global_uuid_map.c

@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "global_uuid_map.h"
+
+static Pvoid_t JGUID_map = (Pvoid_t) NULL;
+static Pvoid_t JGUID_object_map = (Pvoid_t) NULL;
+static uv_rwlock_t guid_lock;
+static uv_rwlock_t object_lock;
+static uv_rwlock_t global_lock;
+
+
+void   dump_object(uuid_t *index, void *object)
+{
+    char uuid_s[36 + 1];
+    uuid_unparse_lower(*index, uuid_s);
+    char local_object[3 * 36 + 2 + 1];
+
+    switch (*(char *) object) {
+        case GUID_TYPE_CHAR:
+            debug(D_GUIDLOG, "OBJECT GUID %s on [%s]", uuid_s, (char *)object + 1);
+            break;
+        case GUID_TYPE_CHART:
+            uuid_unparse_lower((const unsigned char *)object + 1, local_object);
+            uuid_unparse_lower((const unsigned char *)object + 17, local_object+37);
+            local_object[36] = ':';
+            local_object[74] = '\0';
+            debug(D_GUIDLOG, "CHART GUID %s on [%s]", uuid_s, local_object);
+            break;
+        case GUID_TYPE_DIMENSION:
+            uuid_unparse_lower((const unsigned char *)object + 1, local_object);
+            uuid_unparse_lower((const unsigned char *)object + 17, local_object + 37);
+            uuid_unparse_lower((const unsigned char *)object + 33, local_object + 74);
+            local_object[36] = ':';
+            local_object[73] = ':';
+            local_object[110] = '\0';
+            debug(D_GUIDLOG, "DIM GUID %s on [%s]", uuid_s, local_object);
+            break;
+        default:
+            debug(D_GUIDLOG, "Unknown object");
+    }
+}
+
+/* Returns 0 if it successfully stores the uuid-object mapping or if an identical mapping already exists */
+static inline int guid_store_nolock(uuid_t *uuid, void *object, GUID_TYPE object_type)
+{
+    char *existing_object;
+    GUID_TYPE existing_object_type;
+
+    if (unlikely(!object) || uuid == NULL)
+        return 0;
+
+    Pvoid_t *PValue;
+
+    PValue = JudyHSIns(&JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t), PJE0);
+    if (PPJERR == PValue)
+        fatal("JudyHSIns() fatal error.");
+    if (*PValue) {
+        existing_object = *PValue;
+        existing_object_type = existing_object[0];
+        if (existing_object_type != object_type)
+            return 1;
+        switch (existing_object_type) {
+            case GUID_TYPE_DIMENSION:
+                if (memcmp(existing_object, object, 1 + 16 + 16 + 16))
+                    return 1;
+                break;
+            case GUID_TYPE_CHART:
+                if (memcmp(existing_object, object, 1 + 16 + 16))
+                    return 1;
+                break;
+            case GUID_TYPE_CHAR:
+                if (strcmp(existing_object + 1, (char *)object))
+                    return 1;
+                break;
+            default:
+                return 1;
+        }
+        freez(existing_object);
+    }
+
+    *PValue = (Pvoid_t *) object;
+
+    PValue = JudyHSIns(&JGUID_object_map, (void *)object, (Word_t) object_type?(object_type * 16)+1:strlen((char *) object+1)+2, PJE0);
+    if (PPJERR == PValue)
+        fatal("JudyHSIns() fatal error.");
+    if (*PValue == NULL) {
+        uuid_t *value = (uuid_t *) mallocz(sizeof(uuid_t));
+        uuid_copy(*value, *uuid);
+        *PValue = value;
+    }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+    static uint32_t count = 0;
+    count++;
+    char uuid_s[36 + 1];
+    uuid_unparse_lower(*uuid, uuid_s);
+    debug(D_GUIDLOG,"GUID added item %" PRIu32" [%s] as:", count, uuid_s);
+    dump_object(uuid, object);
+#endif
+    return 0;
+}
+
+
+inline int guid_store(uuid_t *uuid, char *object, GUID_TYPE object_type)
+{
+    uv_rwlock_wrlock(&global_lock);
+    int rc = guid_store_nolock(uuid, object, object_type);
+    uv_rwlock_wrunlock(&global_lock);
+    return rc;
+}
+
+/*
+ * This can be used to bulk load entries into the global map
+ *
+ * A lock must be aquired since it will call guid_store_nolock
+ * with a "no lock" parameter.
+ *
+ * Note: object memory must be allocated by caller and not released
+ */
+int guid_bulk_load(char *uuid, char *object)
+{
+    uuid_t target_uuid;
+    if (likely(!uuid_parse(uuid, target_uuid))) {
+#ifdef NETDATA_INTERNAL_CHECKS
+        debug(D_GUIDLOG,"Mapping GUID [%s] on [%s]", uuid, object);
+#endif
+        return guid_store_nolock(&target_uuid, object, GUID_TYPE_CHAR);
+    }
+    return 1;
+}
+
+/*
+ * Given a GUID, find if an object is stored
+ *   - Optionally return the object
+ */
+
+GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
+{
+    Pvoid_t *PValue;
+    GUID_TYPE   value_type;
+
+    uv_rwlock_rdlock(&global_lock);
+    PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t));
+    if (unlikely(!PValue)) {
+        uv_rwlock_rdunlock(&global_lock);
+        return GUID_TYPE_NOTFOUND;
+    }
+
+    value_type = *(char *) *PValue;
+
+    if (likely(object && max_bytes)) {
+        switch (value_type) {
+            case GUID_TYPE_CHAR:
+                if (unlikely(max_bytes - 1 < strlen((char *) *PValue+1)))
+                    return GUID_TYPE_NOSPACE;
+                strncpyz(object, (char *) *PValue+1, max_bytes - 1);
+                break;
+            case GUID_TYPE_CHART:
+            case GUID_TYPE_DIMENSION:
+                if (unlikely(max_bytes < (size_t) value_type * 16))
+                    return GUID_TYPE_NOSPACE;
+                memcpy(object, *PValue+1, value_type * 16);
+                break;
+            default:
+                uv_rwlock_rdunlock(&global_lock);
+                return GUID_TYPE_NOTFOUND;
+        }
+    }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+    dump_object(uuid, *PValue);
+#endif
+    uv_rwlock_rdunlock(&global_lock);
+    return value_type;
+}
+
+/*
+ * Find a GUID of an object
+ *   - Optionally return the GUID
+ *
+ */
+
+int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE object_type)
+{
+    Pvoid_t *PValue;
+
+    uv_rwlock_rdlock(&global_lock);
+    PValue = JudyHSGet(JGUID_object_map, (void *)object, (Word_t)object_type?object_type*16+1:strlen(object+1)+2);
+    if (unlikely(!PValue)) {
+        uv_rwlock_rdunlock(&global_lock);
+        return 1;
+    }
+
+    if (likely(uuid))
+        uuid_copy(*uuid, *PValue);
+    uv_rwlock_rdunlock(&global_lock);
+    return 0;
+}
+
+int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate)
+{
+    char  *target_object;
+    uuid_t temp_uuid;
+    int rc;
+
+    switch (object_type) {
+        case GUID_TYPE_DIMENSION:
+            if (unlikely(find_or_generate_guid((void *) ((RRDDIM *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
+                return 1;
+            target_object = mallocz(49);
+            target_object[0] = object_type;
+            memcpy(target_object + 1, ((RRDDIM *)object)->rrdset->rrdhost->host_uuid, 16);
+            memcpy(target_object + 17, ((RRDDIM *)object)->rrdset->chart_uuid, 16);
+            memcpy(target_object + 33, temp_uuid, 16);
+            break;
+        case GUID_TYPE_CHART:
+            if (unlikely(find_or_generate_guid((void *) ((RRDSET *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
+                return 1;
+            target_object = mallocz(33);
+            target_object[0] = object_type;
+            memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
+            memcpy(target_object + 17, temp_uuid, 16);
+            break;
+        case GUID_TYPE_CHAR:
+            target_object = mallocz(strlen((char *) object)+2);
+            target_object[0] = object_type;
+            strcpy(target_object+1, (char *) object);
+            break;
+        default:
+            return 1;
+    }
+    rc = find_guid_by_object(target_object, uuid, object_type);
+    if (rc) {
+        if (!replace_instead_of_generate) /* else take *uuid as user input */
+            uuid_generate(*uuid);
+        uv_rwlock_wrlock(&global_lock);
+        int rc = guid_store_nolock(uuid, target_object, object_type);
+        uv_rwlock_wrunlock(&global_lock);
+        return rc;
+    }
+    //uv_rwlock_wrunlock(&global_lock);
+#ifdef NETDATA_INTERNAL_CHECKS
+    dump_object(uuid, target_object);
+#endif
+    return 0;
+}
+
+void init_global_guid_map()
+{
+    static int init = 0;
+
+    if (init)
+        return;
+
+    init = 1;
+    info("Configuring locking mechanism for global GUID map");
+    assert(0 == uv_rwlock_init(&guid_lock));
+    assert(0 == uv_rwlock_init(&object_lock));
+    assert(0 == uv_rwlock_init(&global_lock));
+
+//    int rc = guid_bulk_load("6fc56a64-05d7-47a7-bc82-7f3235d8cbda","d6b37186-74db-11ea-88b2-0bf5095b1f9e/cgroup_qemu_ubuntu18.04.cpu_per_core/cpu3");
+//    rc = guid_bulk_load("75c6fa02-97cc-40c1-aacd-a0132190472e","d6b37186-74db-11ea-88b2-0bf5095b1f9e/services.throttle_io_ops_write/system.slice_setvtrgb.service");
+//    if (rc == 0)
+//        info("BULK GUID load successful");
+
+    return;
+}
+
+

database/engine/global_uuid_map/global_uuid_map.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_GLOBAL_UUID_MAP_H
+#define NETDATA_GLOBAL_UUID_MAP_H
+
+#include "libnetdata/libnetdata.h"
+#include <Judy.h>
+#include "../../rrd.h"
+
+typedef enum guid_type {
+    GUID_TYPE_CHAR,
+    GUID_TYPE_HOST,
+    GUID_TYPE_CHART,
+    GUID_TYPE_DIMENSION,
+    GUID_TYPE_NOTFOUND,
+    GUID_TYPE_NOSPACE
+} GUID_TYPE;
+
+extern int guid_store(uuid_t *uuid, char *object, GUID_TYPE);
+extern GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes);
+extern int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE);
+extern void init_global_guid_map();
+extern int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate);
+
+
+#endif //NETDATA_GLOBAL_UUID_MAP_H

database/engine/metadata_log/Makefile.am

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)

database/engine/metadata_log/compaction.c

@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+void after_compact_old_records(struct metalog_worker_config* wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    int error;
+
+    mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log, &ctx->compaction_state.new_metadata_logfiles);
+    uv_run(wc->loop, UV_RUN_DEFAULT);
+
+    error = uv_thread_join(wc->now_compacting_files);
+    if (error) {
+        error("uv_thread_join(): %s", uv_strerror(error));
+    }
+    freez(wc->now_compacting_files);
+    /* unfreeze command processing */
+    wc->now_compacting_files = NULL;
+
+    wc->cleanup_thread_compacting_files = 0;
+
+    /* interrupt event loop */
+    uv_stop(wc->loop);
+
+    info("Finished metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
+}
+
+static void metalog_flush_compaction_records(struct metalog_instance *ctx)
+{
+    struct metalog_cmd cmd;
+    struct completion compaction_completion;
+
+    init_completion(&compaction_completion);
+
+    cmd.opcode = METALOG_COMPACTION_FLUSH;
+    cmd.record_io_descr.completion = &compaction_completion;
+    metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+    wait_for_completion(&compaction_completion);
+    destroy_completion(&compaction_completion);
+}
+
+/* The caller must have called metalog_flush_compaction_records() before to synchronize and quiesce the event loop. */
+static void compaction_test_quota(struct metalog_worker_config *wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    struct logfile_compaction_state *compaction_state;
+    struct metadata_logfile *oldmetalogfile, *newmetalogfile;
+    unsigned current_size;
+    int ret;
+
+    compaction_state = &ctx->compaction_state;
+    newmetalogfile = compaction_state->new_metadata_logfiles.last;
+
+    oldmetalogfile = ctx->metadata_logfiles.first;
+
+    current_size = newmetalogfile->pos;
+    if (unlikely(current_size >= MAX_METALOGFILE_SIZE && newmetalogfile->starting_fileno < oldmetalogfile->fileno)) {
+        /* It's safe to finalize the compacted metadata log file and create a new one since it has already replaced
+         * an older one. */
+
+        /* Finalize as the immediately previous file than the currently compacted one. */
+        ret = rename_metadata_logfile(newmetalogfile, 0, newmetalogfile->fileno - 1);
+        if (ret < 0)
+            return;
+
+        ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles,
+                                       ctx->metadata_logfiles.first->fileno, ctx->metadata_logfiles.first->fileno);
+
+        if (likely(!ret)) {
+            compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
+        }
+    }
+}
+
+
+static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
+{
+    GUID_TYPE ret;
+    RRDHOST *host = ctx->rrdeng_ctx->host;
+    RRDSET *st;
+    RRDDIM *rd;
+    BUFFER *buffer;
+
+    ret = find_object_by_guid(uuid, NULL, 0);
+    switch (ret) {
+        case GUID_TYPE_CHAR:
+            assert(0);
+            break;
+        case GUID_TYPE_CHART:
+            st = metalog_get_chart_from_uuid(ctx, uuid);
+            if (st) {
+                if (ctx->current_compaction_id > st->compaction_id) {
+                    st->compaction_id = ctx->current_compaction_id;
+                    buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
+                    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+                } else {
+                    debug(D_METADATALOG, "Chart has already been compacted, ignoring record.");
+                }
+            } else {
+                debug(D_METADATALOG, "Ignoring nonexistent chart metadata record.");
+            }
+            break;
+        case GUID_TYPE_DIMENSION:
+            rd = metalog_get_dimension_from_uuid(ctx, uuid);
+            if (rd) {
+                if (ctx->current_compaction_id > rd->state->compaction_id) {
+                    rd->state->compaction_id = ctx->current_compaction_id;
+                    buffer = metalog_update_dimension_buffer(rd);
+                    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+                } else {
+                    debug(D_METADATALOG, "Dimension has already been compacted, ignoring record.");
+                }
+            } else {
+                debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record.");
+            }
+            break;
+        case GUID_TYPE_HOST:
+            if (ctx->current_compaction_id > host->compaction_id) {
+                host->compaction_id = ctx->current_compaction_id;
+                buffer = metalog_update_host_buffer(host);
+                metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
+            } else {
+                debug(D_METADATALOG, "Host has already been compacted, ignoring record.");
+            }
+            break;
+        case GUID_TYPE_NOTFOUND:
+            debug(D_METADATALOG, "Ignoring nonexistent metadata record.");
+            break;
+        default:
+            assert(0);
+            break;
+    }
+}
+
+/* Returns 0 on success. */
+static int compact_metadata_logfile_records(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
+{
+    struct metalog_worker_config* wc = &ctx->worker_config;
+    struct logfile_compaction_state *compaction_state;
+    struct metalog_record *record;
+    struct metalog_record_block *record_block, *prev_record_block;
+    int ret;
+    unsigned iterated_records;
+#define METADATA_LOG_RECORD_BATCH 128 /* Flush I/O and check sizes whenever this many records have been iterated */
+
+    info("Compacting metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+         ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+
+    compaction_state = &ctx->compaction_state;
+    record_block = prev_record_block = NULL;
+    iterated_records = 0;
+    for (record = mlf_record_get_first(metalogfile) ; record != NULL ; record = mlf_record_get_next(metalogfile)) {
+        if ((record_block = metalogfile->records.iterator.current) != prev_record_block) {
+            if (prev_record_block) { /* Deallocate iterated record blocks */
+                rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
+                freez(prev_record_block);
+            }
+            prev_record_block = record_block;
+        }
+        compact_record_by_uuid(ctx, &record->uuid);
+        if (0 == ++iterated_records % METADATA_LOG_RECORD_BATCH) {
+            metalog_flush_compaction_records(ctx);
+            if (compaction_state->throttle) {
+                (void)sleep_usec(10000); /* 10 msec throttle compaction */
+            }
+            compaction_test_quota(wc);
+        }
+    }
+    if (prev_record_block) { /* Deallocate iterated record blocks */
+        rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
+        freez(prev_record_block);
+    }
+
+    info("Compacted metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+         ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+
+    metadata_logfile_list_delete(&ctx->metadata_logfiles, metalogfile);
+    ret = destroy_metadata_logfile(metalogfile);
+    if (!ret) {
+        info("Deleted file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+             ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+        rrd_atomic_fetch_add(&ctx->disk_space, -metalogfile->pos);
+    } else {
+        error("Failed to delete file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+             ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+    }
+    freez(metalogfile);
+
+    return ret;
+}
+
+static void compact_old_records(void *arg)
+{
+    struct metalog_instance *ctx = arg;
+    struct metalog_worker_config* wc = &ctx->worker_config;
+    struct logfile_compaction_state *compaction_state;
+    struct metadata_logfile *metalogfile, *nextmetalogfile, *newmetalogfile;
+    int ret;
+
+    compaction_state = &ctx->compaction_state;
+
+    nextmetalogfile = NULL;
+    for (metalogfile = ctx->metadata_logfiles.first ;
+         metalogfile != compaction_state->last_original_logfile ;
+         metalogfile = nextmetalogfile) {
+        nextmetalogfile = metalogfile->next;
+
+        newmetalogfile = compaction_state->new_metadata_logfiles.last;
+        ret = rename_metadata_logfile(newmetalogfile, newmetalogfile->starting_fileno, metalogfile->fileno);
+        if (ret < 0) {
+            error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+                  ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
+        }
+
+        ret = compact_metadata_logfile_records(ctx, metalogfile);
+        if (ret) {
+            error("Metadata log compaction failed, cancelling.");
+            break;
+        }
+    }
+    assert(nextmetalogfile); /* There are always more than 1 metadata log files during compaction */
+
+    newmetalogfile = compaction_state->new_metadata_logfiles.last;
+    if (newmetalogfile->starting_fileno != 0) { /* Must rename the last compacted file */
+        ret = rename_metadata_logfile(newmetalogfile, 0, nextmetalogfile->fileno - 1);
+        if (ret < 0) {
+            error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
+                  ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
+        }
+    }
+    /* Connect the compacted files to the metadata log */
+    newmetalogfile->next = nextmetalogfile;
+    ctx->metadata_logfiles.first = compaction_state->new_metadata_logfiles.first;
+
+    wc->cleanup_thread_compacting_files = 1;
+    /* wake up event loop */
+    assert(0 == uv_async_send(&wc->async));
+}
+
+/* Returns 0 on success. */
+static int init_compaction_state(struct metalog_instance *ctx)
+{
+    struct metadata_logfile *newmetalogfile;
+    struct logfile_compaction_state *compaction_state;
+    int ret;
+
+    compaction_state = &ctx->compaction_state;
+    compaction_state->new_metadata_logfiles.first = NULL;
+    compaction_state->new_metadata_logfiles.last = NULL;
+    compaction_state->starting_fileno = ctx->metadata_logfiles.first->fileno;
+    compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
+    compaction_state->last_original_logfile = ctx->metadata_logfiles.last;
+    compaction_state->throttle = 0;
+
+    ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, compaction_state->starting_fileno,
+                                   compaction_state->fileno);
+    if (unlikely(ret)) {
+        error("Cannot create new metadata log files, compaction aborted.");
+        return ret;
+    }
+    newmetalogfile = compaction_state->new_metadata_logfiles.first;
+    assert(newmetalogfile == compaction_state->new_metadata_logfiles.last);
+    init_metadata_record_log(&compaction_state->records_log);
+
+    return 0;
+}
+
+void metalog_do_compaction(struct metalog_worker_config *wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    int error;
+
+    if (wc->now_compacting_files) {
+        /* already compacting metadata log files */
+        return;
+    }
+    wc->now_compacting_files = mallocz(sizeof(*wc->now_compacting_files));
+    wc->cleanup_thread_compacting_files = 0;
+    metalog_try_link_new_metadata_logfile(wc);
+
+    error = init_compaction_state(ctx);
+    if (unlikely(error)) {
+        error("Cannot create new metadata log files, compaction aborted.");
+        return;
+    }
+    ++ctx->current_compaction_id; /* Signify a new compaction */
+
+    info("Starting metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
+    error = uv_thread_create(wc->now_compacting_files, compact_old_records, ctx);
+    if (error) {
+        error("uv_thread_create(): %s", uv_strerror(error));
+        freez(wc->now_compacting_files);
+        wc->now_compacting_files = NULL;
+    }
+
+}
+
+/* Return 0 on success. */
+int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
+                                 unsigned *matched_files)
+{
+    int ret;
+    unsigned starting_fileno, fileno, i, j, recovered_files;
+    struct metadata_logfile *metalogfile, *compactionfile, **tmp_metalogfiles;
+    char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+    for (i = 0 ; i < *matched_files ; ++i) {
+        metalogfile = metalogfiles[i];
+        if (0 == metalogfile->starting_fileno)
+            continue; /* skip standard metadata log files */
+        break; /* this is a compaction temporary file */
+    }
+    if (i == *matched_files) /* no recovery needed */
+        return 0;
+    info("Starting metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+
+    if (*matched_files - i > 1) { /* Can't have more than 1 temporary compaction files */
+        error("Metadata log files are in an invalid state. Cannot proceed.");
+        return 1;
+    }
+    compactionfile = metalogfile;
+    starting_fileno = compactionfile->starting_fileno;
+    fileno = compactionfile->fileno;
+    /* scratchpad space to move file pointers around */
+    tmp_metalogfiles = callocz(*matched_files, sizeof(*tmp_metalogfiles));
+
+    for (j = 0, recovered_files = 0 ; j < i ; ++j) {
+        metalogfile = metalogfiles[j];
+        assert(0 == metalogfile->starting_fileno);
+        if (metalogfile->fileno < starting_fileno) {
+            tmp_metalogfiles[recovered_files++] = metalogfile;
+            continue;
+        }
+        break; /* reached compaction file serial number */
+    }
+
+    if ((j == i) /* Shouldn't be possible, invalid compaction temporary file */ ||
+        (metalogfile->fileno == starting_fileno && metalogfile->fileno == fileno)) {
+        error("Deleting invalid compaction temporary file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
+              METALOG_EXTENSION"\"", dbfiles_path, starting_fileno, fileno);
+        unlink_metadata_logfile(compactionfile);
+        freez(compactionfile);
+        freez(tmp_metalogfiles);
+        --*matched_files; /* delete the last one */
+
+        info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+        return 0;
+    }
+
+    for ( ; j < i ; ++j) { /* continue iterating through normal metadata log files */
+        metalogfile = metalogfiles[j];
+        assert(0 == metalogfile->starting_fileno);
+        if (metalogfile->fileno < fileno) { /* It has already been compacted */
+            error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
+                      METALOG_EXTENSION"\"", dbfiles_path, 0U, metalogfile->fileno);
+            unlink_metadata_logfile(metalogfile);
+            freez(metalogfile);
+            continue;
+        }
+        tmp_metalogfiles[recovered_files++] = metalogfile;
+    }
+
+    /* compaction temporary file is valid */
+    tmp_metalogfiles[recovered_files++] = compactionfile;
+    ret = rename_metadata_logfile(compactionfile, 0, starting_fileno);
+    if (ret < 0) {
+        error("Cannot rename temporary compaction files. Cannot proceed.");
+        freez(tmp_metalogfiles);
+        return 1;
+    }
+
+    memcpy(metalogfiles, tmp_metalogfiles, recovered_files * sizeof(*metalogfiles));
+    *matched_files = recovered_files;
+    freez(tmp_metalogfiles);
+
+    info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
+    return 0;
+}

database/engine/metadata_log/compaction.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_COMPACTION_H
+#define NETDATA_COMPACTION_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include "../rrdengine.h"
+
+struct logfile_compaction_state {
+    unsigned fileno; /* Starts at 1 */
+    unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
+
+    struct metadata_record_commit_log records_log;
+    struct metadata_logfile_list new_metadata_logfiles;
+    struct metadata_logfile *last_original_logfile; /* Marks the end of compaction */
+    uint8_t throttle; /* set non-zero to throttle compaction */
+};
+
+extern int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
+                                       unsigned *matched_files);
+extern void metalog_do_compaction(struct metalog_worker_config *wc);
+extern void after_compact_old_records(struct metalog_worker_config* wc);
+
+#endif /* NETDATA_COMPACTION_H */

database/engine/metadata_log/logfile.c

@@ -0,0 +1,790 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "metadatalog.h"
+#include "metalogpluginsd.h"
+
+static void mlf_record_block_insert(struct metadata_logfile *metalogfile, struct metalog_record_block *record_block)
+{
+
+    if (likely(NULL != metalogfile->records.last)) {
+        metalogfile->records.last->next = record_block;
+    }
+    if (unlikely(NULL == metalogfile->records.first)) {
+        metalogfile->records.first = record_block;
+    }
+    metalogfile->records.last = record_block;
+}
+
+void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record)
+{
+    struct metalog_record_block *record_block;
+    struct metalog_instance *ctx = metalogfile->ctx;
+
+    record_block = metalogfile->records.last;
+    if (likely(NULL != record_block && record_block->records_nr < MAX_METALOG_RECORDS_PER_BLOCK)) {
+        record_block->record_array[record_block->records_nr++] = *record;
+    } else { /* Create new record block, the last one filled up */
+        record_block = mallocz(sizeof(*record_block));
+        record_block->records_nr = 1;
+        record_block->record_array[0] = *record;
+        record_block->next = NULL;
+
+        mlf_record_block_insert(metalogfile, record_block);
+    }
+    rrd_atomic_fetch_add(&ctx->records_nr, 1);
+}
+
+struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile)
+{
+    struct metalog_records *records = &metalogfile->records;
+    struct metalog_record_block *record_block = metalogfile->records.first;
+
+    records->iterator.current = record_block;
+    records->iterator.record_i = 0;
+
+    if (unlikely(NULL == record_block || !record_block->records_nr)) {
+        error("Cannot iterate empty metadata log file %u-%u.", metalogfile->starting_fileno, metalogfile->fileno);
+        return NULL;
+    }
+
+    return &record_block->record_array[0];
+}
+
+/* Must have called mlf_record_get_first before calling this function. */
+struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile)
+{
+    struct metalog_records *records = &metalogfile->records;
+    struct metalog_record_block *record_block = records->iterator.current;
+
+    if (unlikely(NULL == record_block)) {
+        return NULL;
+    }
+    if (++records->iterator.record_i >= record_block->records_nr) {
+        record_block = record_block->next;
+        if (unlikely(NULL == record_block || !record_block->records_nr)) {
+            return NULL;
+        }
+        records->iterator.current = record_block;
+        records->iterator.record_i = 0;
+        return &record_block->record_array[0];
+    }
+    return &record_block->record_array[records->iterator.record_i];
+}
+
+static void flush_records_buffer_cb(uv_fs_t* req)
+{
+    struct generic_io_descriptor *io_descr = req->data;
+    struct metalog_worker_config *wc = req->loop->data;
+    struct metalog_instance *ctx = wc->ctx;
+
+    debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
+    if (req->result < 0) {
+        ++ctx->stats.io_errors;
+        rrd_stat_atomic_add(&global_io_errors, 1);
+        error("%s: uv_fs_write: %s", __func__, uv_strerror((int)req->result));
+    } else {
+        debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
+    }
+
+    uv_fs_req_cleanup(req);
+    free(io_descr->buf);
+    freez(io_descr);
+}
+
+/* Careful to always call this before creating a new metadata log file to finish writing the old one */
+void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+                              struct metadata_logfile_list *metadata_logfiles)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    int ret;
+    struct generic_io_descriptor *io_descr;
+    unsigned pos, size;
+    struct metadata_logfile *metalogfile;
+
+    if (unlikely(NULL == records_log->buf || 0 == records_log->buf_pos)) {
+        return;
+    }
+    /* care with outstanding records when switching metadata log files */
+    metalogfile = metadata_logfiles->last;
+
+    io_descr = mallocz(sizeof(*io_descr));
+    pos = records_log->buf_pos;
+    size = pos; /* no need to align the I/O when doing buffered writes */
+    io_descr->buf = records_log->buf;
+    io_descr->bytes = size;
+    io_descr->pos = metalogfile->pos;
+    io_descr->req.data = io_descr;
+    io_descr->completion = NULL;
+
+    io_descr->iov = uv_buf_init((void *)io_descr->buf, size);
+    ret = uv_fs_write(wc->loop, &io_descr->req, metalogfile->file, &io_descr->iov, 1,
+                      metalogfile->pos, flush_records_buffer_cb);
+    assert (-1 != ret);
+    metalogfile->pos += size;
+    rrd_atomic_fetch_add(&ctx->disk_space, size);
+    records_log->buf = NULL;
+    ctx->stats.io_write_bytes += size;
+    ++ctx->stats.io_write_requests;
+}
+
+void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+                             struct metadata_logfile_list *metadata_logfiles, unsigned size)
+{
+    int ret;
+    unsigned buf_pos, buf_size;
+
+    assert(size);
+    if (records_log->buf) {
+        unsigned remaining;
+
+        buf_pos = records_log->buf_pos;
+        buf_size = records_log->buf_size;
+        remaining = buf_size - buf_pos;
+        if (size > remaining) {
+            /* we need a new buffer */
+            mlf_flush_records_buffer(wc, records_log, metadata_logfiles);
+        }
+    }
+    if (NULL == records_log->buf) {
+        buf_size = ALIGN_BYTES_CEILING(size);
+        ret = posix_memalign((void *)&records_log->buf, RRDFILE_ALIGNMENT, buf_size);
+        if (unlikely(ret)) {
+            fatal("posix_memalign:%s", strerror(ret));
+        }
+        buf_pos = records_log->buf_pos = 0;
+        records_log->buf_size =  buf_size;
+    }
+    records_log->buf_pos += size;
+
+    return records_log->buf + buf_pos;
+}
+
+
+void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
+{
+    if (likely(NULL != metadata_logfiles->last)) {
+        metadata_logfiles->last->next = metalogfile;
+    }
+    if (unlikely(NULL == metadata_logfiles->first)) {
+        metadata_logfiles->first = metalogfile;
+    }
+    metadata_logfiles->last = metalogfile;
+}
+
+void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
+{
+    struct metadata_logfile *next;
+
+    next = metalogfile->next;
+    assert((NULL != next) && (metadata_logfiles->first == metalogfile) &&
+           (metadata_logfiles->last != metalogfile));
+    metadata_logfiles->first = next;
+}
+
+void generate_metadata_logfile_path(struct metadata_logfile *metalogfile, char *str, size_t maxlen)
+{
+    (void) snprintf(str, maxlen, "%s/" METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION,
+                    metalogfile->ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
+}
+
+void metadata_logfile_init(struct metadata_logfile *metalogfile, struct metalog_instance *ctx, unsigned starting_fileno,
+                           unsigned fileno)
+{
+    metalogfile->starting_fileno = starting_fileno;
+    metalogfile->fileno = fileno;
+    metalogfile->file = (uv_file)0;
+    metalogfile->pos = 0;
+    metalogfile->records.first = metalogfile->records.last = NULL;
+    metalogfile->next = NULL;
+    metalogfile->ctx = ctx;
+}
+
+int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno, unsigned new_fileno)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    uv_fs_t req;
+    int ret;
+    char oldpath[RRDENG_PATH_MAX], newpath[RRDENG_PATH_MAX];
+    unsigned backup_starting_fileno, backup_fileno;
+
+    backup_starting_fileno = metalogfile->starting_fileno;
+    backup_fileno = metalogfile->fileno;
+    generate_metadata_logfile_path(metalogfile, oldpath, sizeof(oldpath));
+    metalogfile->starting_fileno = new_starting_fileno;
+    metalogfile->fileno = new_fileno;
+    generate_metadata_logfile_path(metalogfile, newpath, sizeof(newpath));
+
+    info("Renaming metadata log file \"%s\" to \"%s\".", oldpath, newpath);
+    ret = uv_fs_rename(NULL, &req, oldpath, newpath, NULL);
+    if (ret < 0) {
+        error("uv_fs_rename(%s): %s", oldpath, uv_strerror(ret));
+        ++ctx->stats.fs_errors; /* this is racy, may miss some errors */
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+        /* restore previous values */
+        metalogfile->starting_fileno = backup_starting_fileno;
+        metalogfile->fileno = backup_fileno;
+    }
+    uv_fs_req_cleanup(&req);
+
+    return ret;
+}
+
+int close_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    uv_fs_t req;
+    int ret;
+    char path[RRDENG_PATH_MAX];
+
+    generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+    ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
+    if (ret < 0) {
+        error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+
+    return ret;
+}
+
+int unlink_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    uv_fs_t req;
+    int ret;
+    char path[RRDENG_PATH_MAX];
+
+    generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+    ret = uv_fs_unlink(NULL, &req, path, NULL);
+    if (ret < 0) {
+        error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+
+    return ret;
+}
+
+int destroy_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    uv_fs_t req;
+    int ret;
+    char path[RRDENG_PATH_MAX];
+
+    generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+
+    ret = uv_fs_ftruncate(NULL, &req, metalogfile->file, 0, NULL);
+    if (ret < 0) {
+        error("uv_fs_ftruncate(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+
+    ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
+    if (ret < 0) {
+        error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+
+    ret = uv_fs_unlink(NULL, &req, path, NULL);
+    if (ret < 0) {
+        error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+
+//    ++ctx->stats.metadata_logfile_deletions;
+
+    return ret;
+}
+
+int create_metadata_logfile(struct metadata_logfile *metalogfile)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    uv_fs_t req;
+    uv_file file;
+    int ret, fd;
+    struct rrdeng_metalog_sb *superblock;
+    uv_buf_t iov;
+    char path[RRDENG_PATH_MAX];
+
+    generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+    fd = open_file_buffered_io(path, O_CREAT | O_RDWR | O_TRUNC, &file);
+    if (fd < 0) {
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+        return fd;
+    }
+    metalogfile->file = file;
+//    ++ctx->stats.metadata_logfile_creations;
+
+    ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
+    if (unlikely(ret)) {
+        fatal("posix_memalign:%s", strerror(ret));
+    }
+    (void) strncpy(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ);
+    superblock->version = RRDENG_METALOG_VER;
+
+    iov = uv_buf_init((void *)superblock, sizeof(*superblock));
+
+    ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);
+    if (ret < 0) {
+        assert(req.result < 0);
+        error("uv_fs_write: %s", uv_strerror(ret));
+        ++ctx->stats.io_errors;
+        rrd_stat_atomic_add(&global_io_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+    free(superblock);
+    if (ret < 0) {
+        destroy_metadata_logfile(metalogfile);
+        return ret;
+    }
+
+    metalogfile->pos = sizeof(*superblock);
+    ctx->stats.io_write_bytes += sizeof(*superblock);
+    ++ctx->stats.io_write_requests;
+
+    return 0;
+}
+
+static int check_metadata_logfile_superblock(uv_file file)
+{
+    int ret;
+    struct rrdeng_metalog_sb *superblock;
+    uv_buf_t iov;
+    uv_fs_t req;
+
+    ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
+    if (unlikely(ret)) {
+        fatal("posix_memalign:%s", strerror(ret));
+    }
+    iov = uv_buf_init((void *)superblock, sizeof(*superblock));
+
+    ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);
+    if (ret < 0) {
+        error("uv_fs_read: %s", uv_strerror(ret));
+        uv_fs_req_cleanup(&req);
+        goto error;
+    }
+    assert(req.result >= 0);
+    uv_fs_req_cleanup(&req);
+
+    if (strncmp(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ)) {
+        error("File has invalid superblock.");
+        ret = UV_EINVAL;
+    } else {
+        ret = 0;
+    }
+    if (superblock->version > RRDENG_METALOG_VER) {
+        error("File has unknown version %"PRIu16". Compatibility is not guaranteed.", superblock->version);
+    }
+error:
+    free(superblock);
+    return ret;
+}
+
+void replay_record(struct metadata_logfile *metalogfile, struct rrdeng_metalog_record_header *header, void *payload)
+{
+    struct metalog_instance *ctx = metalogfile->ctx;
+    char *line, *nextline, *record_end;
+    int ret;
+
+    debug(D_METADATALOG, "RECORD contents: %.*s", (int)header->payload_length, (char *)payload);
+    record_end = (char *)payload + header->payload_length - 1;
+    *record_end = '\0';
+
+    for (line = payload ; line ; line = nextline) {
+        nextline = strchr(line, '\n');
+        if (nextline) {
+            *nextline++ = '\0';
+        }
+        ret = parser_action(ctx->metalog_parser_object->parser, line);
+        debug(D_METADATALOG, "parser_action ret:%d", ret);
+        if (ret)
+            return; /* skip record due to error */
+    };
+}
+
+/* This function only works with buffered I/O */
+static inline int metalogfile_read(struct metadata_logfile *metalogfile, void *buf, size_t len, uint64_t offset)
+{
+    struct metalog_instance *ctx;
+    uv_file file;
+    uv_buf_t iov;
+    uv_fs_t req;
+    int ret;
+
+    ctx = metalogfile->ctx;
+    file = metalogfile->file;
+    iov = uv_buf_init(buf, len);
+    ret = uv_fs_read(NULL, &req, file, &iov, 1, offset, NULL);
+    if (unlikely(ret < 0 && ret != req.result)) {
+        fatal("uv_fs_read: %s", uv_strerror(ret));
+    }
+    if (req.result < 0) {
+        ++ctx->stats.io_errors;
+        rrd_stat_atomic_add(&global_io_errors, 1);
+        error("%s: uv_fs_read - %s - record at offset %"PRIu64"(%u) in metadata logfile %u-%u.", __func__,
+              uv_strerror((int)req.result), offset, (unsigned)len, metalogfile->starting_fileno, metalogfile->fileno);
+    }
+    uv_fs_req_cleanup(&req);
+    ctx->stats.io_read_bytes += len;
+    ++ctx->stats.io_read_requests;
+
+    return ret;
+}
+
+/* Return 0 on success */
+static int metadata_record_integrity_check(void *record)
+{
+    int ret;
+    uint32_t data_size;
+    struct rrdeng_metalog_record_header *header;
+    struct rrdeng_metalog_record_trailer *trailer;
+    uLong crc;
+
+    header = record;
+    data_size = header->header_length + header->payload_length;
+    trailer = record + data_size;
+
+    crc = crc32(0L, Z_NULL, 0);
+    crc = crc32(crc, record, data_size);
+    ret = crc32cmp(trailer->checksum, crc);
+
+    return ret;
+}
+
+#define MAX_READ_BYTES (RRDENG_BLOCK_SIZE * 32) /* no record should be over 128KiB in this version */
+
+/*
+ * Iterates metadata log file records and creates database objects (host/chart/dimension)
+ */
+static void iterate_records(struct metadata_logfile *metalogfile)
+{
+    uint32_t file_size, pos, bytes_remaining, record_size;
+    void *buf;
+    struct rrdeng_metalog_record_header *header;
+    struct metalog_instance *ctx = metalogfile->ctx;
+    struct metalog_pluginsd_state *state = ctx->metalog_parser_object->private;
+    const size_t min_header_size = offsetof(struct rrdeng_metalog_record_header, header_length) +
+                                   sizeof(header->header_length);
+
+    file_size = metalogfile->pos;
+    state->metalogfile = metalogfile;
+
+    buf = mallocz(MAX_READ_BYTES);
+
+    for (pos = sizeof(struct rrdeng_metalog_sb) ; pos < file_size ; pos += record_size) {
+        bytes_remaining = file_size - pos;
+        if (bytes_remaining < min_header_size) {
+            error("%s: unexpected end of file in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+                  metalogfile->fileno);
+            break;
+        }
+        if (metalogfile_read(metalogfile, buf, min_header_size, pos) < 0)
+            break;
+        header = (struct rrdeng_metalog_record_header *)buf;
+        if (METALOG_STORE_PADDING == header->type) {
+            info("%s: Skipping padding in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+                 metalogfile->fileno);
+            record_size = ALIGN_BYTES_FLOOR(pos + RRDENG_BLOCK_SIZE) - pos;
+            continue;
+        }
+        if (metalogfile_read(metalogfile, buf + min_header_size, sizeof(*header) - min_header_size,
+                             pos + min_header_size) < 0)
+            break;
+        record_size = header->header_length + header->payload_length + sizeof(struct rrdeng_metalog_record_trailer);
+        if (header->header_length < min_header_size || record_size > bytes_remaining) {
+            error("%s: Corrupted record in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
+                  metalogfile->fileno);
+            break;
+        }
+        if (record_size > MAX_READ_BYTES) {
+            error("%s: Record is too long (%u bytes) in metadata logfile %u-%u.", __func__, record_size,
+                  metalogfile->starting_fileno, metalogfile->fileno);
+            continue;
+        }
+        if (metalogfile_read(metalogfile, buf + sizeof(*header), record_size - sizeof(*header),
+                             pos + sizeof(*header)) < 0)
+            break;
+        if (metadata_record_integrity_check(buf)) {
+            error("%s: Record at offset %"PRIu32" was read from disk. CRC32 check: FAILED", __func__, pos);
+            continue;
+        }
+        debug(D_METADATALOG, "%s: Record at offset %"PRIu32" was read from disk. CRC32 check: SUCCEEDED", __func__,
+              pos);
+
+        replay_record(metalogfile, header, buf + header->header_length);
+        if (!uuid_is_null(state->uuid)) { /* It's a valid object */
+            struct metalog_record record;
+
+            uuid_copy(record.uuid, state->uuid);
+            mlf_record_insert(metalogfile, &record);
+            uuid_clear(state->uuid); /* Clear state for parsing of next record */
+        }
+    }
+
+    freez(buf);
+}
+
+int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
+{
+    uv_fs_t req;
+    uv_file file;
+    int ret, fd, error;
+    uint64_t file_size;
+    char path[RRDENG_PATH_MAX];
+
+    generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+    fd = open_file_buffered_io(path, O_RDWR, &file);
+    if (fd < 0) {
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+        return fd;
+    }
+    info("Loading metadata log \"%s\".", path);
+
+    ret = check_file_properties(file, &file_size, sizeof(struct rrdeng_metalog_sb));
+    if (ret)
+        goto error;
+
+    ret = check_metadata_logfile_superblock(file);
+    if (ret)
+        goto error;
+    ctx->stats.io_read_bytes += sizeof(struct rrdeng_jf_sb);
+    ++ctx->stats.io_read_requests;
+
+    metalogfile->file = file;
+    metalogfile->pos = file_size;
+
+    iterate_records(metalogfile);
+
+    info("Metadata log \"%s\" loaded (size:%"PRIu64").", path, file_size);
+    return 0;
+
+error:
+    error = ret;
+    ret = uv_fs_close(NULL, &req, file, NULL);
+    if (ret < 0) {
+        error("uv_fs_close(%s): %s", path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+    }
+    uv_fs_req_cleanup(&req);
+    return error;
+}
+
+void init_metadata_record_log(struct metadata_record_commit_log *records_log)
+{
+    records_log->buf = NULL;
+    records_log->buf_pos = 0;
+    records_log->record_id = 1;
+}
+
+static int scan_metalog_files_cmp(const void *a, const void *b)
+{
+    struct metadata_logfile *file1, *file2;
+    char path1[RRDENG_PATH_MAX], path2[RRDENG_PATH_MAX];
+
+    file1 = *(struct metadata_logfile **)a;
+    file2 = *(struct metadata_logfile **)b;
+    generate_metadata_logfile_path(file1, path1, sizeof(path1));
+    generate_metadata_logfile_path(file2, path2, sizeof(path2));
+    return strcmp(path1, path2);
+}
+
+/* Returns number of metadata logfiles that were loaded or < 0 on error */
+static int scan_metalog_files(struct metalog_instance *ctx)
+{
+    int ret;
+    unsigned starting_no, no, matched_files, i, failed_to_load;
+    static uv_fs_t req;
+    uv_dirent_t dent;
+    struct metadata_logfile **metalogfiles, *metalogfile;
+    char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+    ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
+    if (ret < 0) {
+        assert(req.result < 0);
+        uv_fs_req_cleanup(&req);
+        error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
+        ++ctx->stats.fs_errors;
+        rrd_stat_atomic_add(&global_fs_errors, 1);
+        return ret;
+    }
+    info("Found %d files in path %s", ret, dbfiles_path);
+
+    metalogfiles = callocz(MIN(ret, MAX_DATAFILES), sizeof(*metalogfiles));
+    for (matched_files = 0 ; UV_EOF != uv_fs_scandir_next(&req, &dent) && matched_files < MAX_DATAFILES ; ) {
+        info("Scanning file \"%s/%s\"", dbfiles_path, dent.name);
+        ret = sscanf(dent.name, METALOG_PREFIX METALOG_FILE_NUMBER_SCAN_TMPL METALOG_EXTENSION, &starting_no, &no);
+        if (2 == ret) {
+            info("Matched file \"%s/%s\"", dbfiles_path, dent.name);
+            metalogfile = mallocz(sizeof(*metalogfile));
+            metadata_logfile_init(metalogfile, ctx, starting_no, no);
+            metalogfiles[matched_files++] = metalogfile;
+        }
+    }
+    uv_fs_req_cleanup(&req);
+
+    if (0 == matched_files) {
+        freez(metalogfiles);
+        return 0;
+    }
+    if (matched_files == MAX_DATAFILES) {
+        error("Warning: hit maximum database engine file limit of %d files", MAX_DATAFILES);
+    }
+    qsort(metalogfiles, matched_files, sizeof(*metalogfiles), scan_metalog_files_cmp);
+    ret = compaction_failure_recovery(ctx, metalogfiles, &matched_files);
+    if (ret) { /* If the files are corrupted fail */
+        for (i = 0 ; i < matched_files ; ++i) {
+            freez(metalogfiles[i]);
+        }
+        freez(metalogfiles);
+        return UV_EINVAL;
+    }
+    ctx->last_fileno = metalogfiles[matched_files - 1]->fileno;
+
+    struct plugind cd = {
+        .enabled = 1,
+        .update_every = 0,
+        .pid = 0,
+        .serial_failures = 0,
+        .successful_collections = 0,
+        .obsolete = 0,
+        .started_t = INVALID_TIME,
+        .next = NULL,
+        .version = 0,
+    };
+
+    struct metalog_pluginsd_state metalog_parser_state;
+    metalog_pluginsd_state_init(&metalog_parser_state, ctx);
+
+    PARSER_USER_OBJECT metalog_parser_object;
+    metalog_parser_object.enabled = cd.enabled;
+    metalog_parser_object.host = ctx->rrdeng_ctx->host;
+    metalog_parser_object.cd = &cd;
+    metalog_parser_object.trust_durations = 0;
+    metalog_parser_object.private = &metalog_parser_state;
+
+    PARSER *parser = parser_init(metalog_parser_object.host, &metalog_parser_object, NULL, PARSER_INPUT_SPLIT);
+    if (unlikely(!parser)) {
+        error("Failed to initialize metadata log parser.");
+        failed_to_load = matched_files;
+        goto after_failed_to_parse;
+    }
+    parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
+    parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
+    parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
+    parser->plugins_action->dimension_action = &metalog_pluginsd_dimension_action;
+    parser->plugins_action->chart_action     = &metalog_pluginsd_chart_action;
+    parser->plugins_action->guid_action      = &metalog_pluginsd_guid_action;
+    parser->plugins_action->context_action   = &metalog_pluginsd_context_action;
+    parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
+
+    metalog_parser_object.parser = parser;
+    ctx->metalog_parser_object = &metalog_parser_object;
+
+    for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
+        metalogfile = metalogfiles[i];
+        ret = load_metadata_logfile(ctx, metalogfile);
+        if (0 != ret) {
+            freez(metalogfile);
+            ++failed_to_load;
+            break;
+        }
+        metadata_logfile_list_insert(&ctx->metadata_logfiles, metalogfile);
+        rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
+    }
+    debug(D_METADATALOG, "PARSER ended");
+
+    parser_destroy(parser);
+
+    size_t count = metalog_parser_object.count;
+
+    debug(D_METADATALOG, "Parsing count=%u", (unsigned)count);
+after_failed_to_parse:
+
+    freez(metalogfiles);
+    if (failed_to_load) {
+        error("%u metadata log files failed to load.", failed_to_load);
+        finalize_metalog_files(ctx);
+        return UV_EIO;
+    }
+
+    return matched_files;
+}
+
+/* Creates a metadata log file */
+int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
+                             unsigned starting_fileno, unsigned fileno)
+{
+    struct metadata_logfile *metalogfile;
+    int ret;
+    char path[RRDENG_PATH_MAX];
+
+    info("Creating new metadata log file in path %s", ctx->rrdeng_ctx->dbfiles_path);
+    metalogfile = mallocz(sizeof(*metalogfile));
+    metadata_logfile_init(metalogfile, ctx, starting_fileno, fileno);
+    ret = create_metadata_logfile(metalogfile);
+    if (!ret) {
+        generate_metadata_logfile_path(metalogfile, path, sizeof(path));
+        info("Created metadata log file \"%s\".", path);
+    } else {
+        freez(metalogfile);
+        return ret;
+    }
+    metadata_logfile_list_insert(logfile_list, metalogfile);
+    rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
+
+    return 0;
+}
+
+/* Return 0 on success. */
+int init_metalog_files(struct metalog_instance *ctx)
+{
+    int ret;
+    char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
+
+    ret = scan_metalog_files(ctx);
+    if (ret < 0) {
+        error("Failed to scan path \"%s\".", dbfiles_path);
+        return ret;
+    } else if (0 == ret) {
+        info("Metadata log files not found, creating in path \"%s\".", dbfiles_path);
+        ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, 1);
+        if (ret) {
+            error("Failed to create metadata log file in path \"%s\".", dbfiles_path);
+            return ret;
+        }
+        ctx->last_fileno = 1;
+    }
+
+    return 0;
+}
+
+void finalize_metalog_files(struct metalog_instance *ctx)
+{
+    struct metadata_logfile *metalogfile, *next_metalogfile;
+    struct metalog_record_block *record_block, *next_record_block;
+
+    for (metalogfile = ctx->metadata_logfiles.first ; metalogfile != NULL ; metalogfile = next_metalogfile) {
+        next_metalogfile = metalogfile->next;
+
+        for (record_block = metalogfile->records.first ; record_block != NULL ; record_block = next_record_block) {
+            next_record_block = record_block->next;
+            freez(record_block);
+        }
+        close_metadata_logfile(metalogfile);
+        freez(metalogfile);
+    }
+}

database/engine/metadata_log/logfile.h

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOGFILE_H
+#define NETDATA_LOGFILE_H
+
+#include "metadatalogprotocol.h"
+#include "../rrdengine.h"
+
+/* Forward declarations */
+struct metadata_logfile;
+struct metalog_worker_config;
+
+#define METALOG_PREFIX "metadatalog-"
+#define METALOG_EXTENSION ".mlf"
+
+#define MAX_METALOGFILE_SIZE   (524288LU)
+
+/* Deletions are ignored during compaction, so only creation UUIDs are stored */
+struct metalog_record {
+    uuid_t uuid;
+};
+
+#define MAX_METALOG_RECORDS_PER_BLOCK   (1024LU)
+struct metalog_record_block {
+    uint64_t file_offset;
+    uint32_t io_size;
+
+    struct metalog_record record_array[MAX_METALOG_RECORDS_PER_BLOCK];
+    uint16_t records_nr;
+
+    struct metalog_record_block *next;
+};
+
+struct metalog_records {
+    /* the record block list is sorted based on disk offset */
+    struct metalog_record_block *first;
+    struct metalog_record_block *last;
+    struct {
+        struct metalog_record_block *current;
+        uint16_t record_i;
+    } iterator;
+};
+
+/* only one event loop is supported for now */
+struct metadata_logfile {
+    unsigned fileno; /* Starts at 1 */
+    unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
+    uv_file file;
+    uint64_t pos;
+    struct metalog_instance *ctx;
+    struct metalog_records records;
+    struct metadata_logfile *next;
+};
+
+struct metadata_logfile_list {
+    struct metadata_logfile *first; /* oldest */
+    struct metadata_logfile *last; /* newest */
+};
+
+struct metadata_record_commit_log {
+    uint64_t record_id;
+
+    /* outstanding record buffer */
+    void *buf;
+    unsigned buf_pos;
+    unsigned buf_size;
+};
+
+extern void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record);
+extern struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile);
+extern struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile);
+extern void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+                                     struct metadata_logfile_list *metadata_logfiles);
+extern void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
+                                    struct metadata_logfile_list *metadata_logfiles, unsigned size);
+extern void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles,
+                                         struct metadata_logfile *metalogfile);
+extern void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles,
+                                         struct metadata_logfile *metalogfile);
+extern void generate_metadata_logfile_path(struct metadata_logfile *metadatalog, char *str, size_t maxlen);
+extern void metadata_logfile_init(struct metadata_logfile *metadatalog, struct metalog_instance *ctx,
+                              unsigned tier, unsigned fileno);
+extern int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno,
+                                   unsigned new_fileno);
+extern int close_metadata_logfile(struct metadata_logfile *metadatalog);
+extern int unlink_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int destroy_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int create_metadata_logfile(struct metadata_logfile *metalogfile);
+extern int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *logfile);
+extern void init_metadata_record_log(struct metadata_record_commit_log *records_log);
+extern int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
+                                    unsigned tier, unsigned fileno);
+extern int init_metalog_files(struct metalog_instance *ctx);
+extern void finalize_metalog_files(struct metalog_instance *ctx);
+
+
+#endif /* NETDATA_LOGFILE_H */

database/engine/metadata_log/metadatalog.c

@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+static void sanity_check(void)
+{
+    /* Magic numbers must fit in the super-blocks */
+    BUILD_BUG_ON(strlen(RRDENG_METALOG_MAGIC) > RRDENG_MAGIC_SZ);
+
+    /* Metadata log file super-block cannot be larger than RRDENG_BLOCK_SIZE */
+    BUILD_BUG_ON(RRDENG_METALOG_SB_PADDING_SZ < 0);
+
+    /* Object duplication factor cannot be less than 1, or too close to 1 */
+    BUILD_BUG_ON(MAX_DUPLICATION_PERCENTAGE < 110);
+}
+
+char *get_metalog_statistics(struct metalog_instance *ctx, char *str, size_t size)
+{
+    snprintfz(str, size,
+              "io_write_bytes: %ld\n"
+              "io_write_requests: %ld\n"
+              "io_read_bytes: %ld\n"
+              "io_read_requests: %ld\n"
+              "io_write_record_bytes: %ld\n"
+              "io_write_records: %ld\n"
+              "io_read_record_bytes: %ld\n"
+              "io_read_records: %ld\n"
+              "metadata_logfile_creations: %ld\n"
+              "metadata_logfile_deletions: %ld\n"
+              "io_errors: %ld\n"
+              "fs_errors: %ld\n",
+              (long)ctx->stats.io_write_bytes,
+              (long)ctx->stats.io_write_requests,
+              (long)ctx->stats.io_read_bytes,
+              (long)ctx->stats.io_read_requests,
+              (long)ctx->stats.io_write_record_bytes,
+              (long)ctx->stats.io_write_records,
+              (long)ctx->stats.io_read_record_bytes,
+              (long)ctx->stats.io_read_records,
+              (long)ctx->stats.metadata_logfile_creations,
+              (long)ctx->stats.metadata_logfile_deletions,
+              (long)ctx->stats.io_errors,
+              (long)ctx->stats.fs_errors
+    );
+    return str;
+}
+
+/* The buffer must not be empty */
+void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode, uuid_t *uuid,
+                           int compacting)
+{
+    struct metalog_cmd cmd;
+
+    assert(buffer_strlen(buffer));
+    assert(opcode == METALOG_COMMIT_CREATION_RECORD || opcode == METALOG_COMMIT_DELETION_RECORD);
+
+    cmd.opcode = opcode;
+    cmd.record_io_descr.buffer = buffer;
+    cmd.record_io_descr.compacting = compacting;
+    if (!uuid)
+        uuid_clear(cmd.record_io_descr.uuid);
+    else
+        uuid_copy(cmd.record_io_descr.uuid, *uuid);
+    metalog_enq_cmd(&ctx->worker_config, &cmd);
+}
+
+static void commit_record(struct metalog_worker_config* wc, struct metalog_record_io_descr *io_descr, uint8_t type)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    unsigned payload_length, size_bytes;
+    void *buf, *mlf_payload;
+    /* persistent structures */
+    struct rrdeng_metalog_record_header *mlf_header;
+    struct rrdeng_metalog_record_trailer *mlf_trailer;
+    uLong crc;
+
+    payload_length = buffer_strlen(io_descr->buffer);
+    size_bytes = sizeof(*mlf_header) + payload_length + sizeof(*mlf_trailer);
+
+    if (io_descr->compacting)
+        buf = mlf_get_records_buffer(wc, &ctx->compaction_state.records_log,
+                                     &ctx->compaction_state.new_metadata_logfiles, size_bytes);
+    else
+        buf = mlf_get_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles, size_bytes);
+
+    mlf_header = buf;
+    mlf_header->type = type;
+    mlf_header->header_length = sizeof(*mlf_header);
+    mlf_header->payload_length = payload_length;
+
+    mlf_payload = buf + sizeof(*mlf_header);
+    memcpy(mlf_payload, buffer_tostring(io_descr->buffer), payload_length);
+
+    mlf_trailer = buf + sizeof(*mlf_header) + payload_length;
+    crc = crc32(0L, Z_NULL, 0);
+    crc = crc32(crc, buf, sizeof(*mlf_header) + payload_length);
+    crc32set(mlf_trailer->checksum, crc);
+
+    buffer_free(io_descr->buffer);
+}
+
+static void do_commit_record(struct metalog_worker_config* wc, uint8_t type, void *data)
+{
+    struct metalog_record_io_descr *io_descr = (struct metalog_record_io_descr *)data;
+    switch (type) {
+    case METALOG_CREATE_OBJECT:
+        if (!uuid_is_null(io_descr->uuid)) { /* It's a valid object */
+            struct metalog_record record;
+
+            uuid_copy(record.uuid, io_descr->uuid);
+            if (io_descr->compacting)
+                mlf_record_insert(wc->ctx->compaction_state.new_metadata_logfiles.last, &record);
+            else
+                mlf_record_insert(wc->ctx->metadata_logfiles.last, &record);
+        } /* fall through */
+        case METALOG_DELETE_OBJECT:
+        commit_record(wc, (struct metalog_record_io_descr *)data, type);
+        break;
+    default:
+        fatal("Unknown metadata log file record type, possible memory corruption.");
+        break;
+    }
+}
+
+/* Only creates a new metadata file and links it to the metadata log if the last one is non empty. */
+void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    struct metadata_logfile *metalogfile;
+    int ret;
+
+    metalogfile = ctx->metadata_logfiles.last;
+    if (metalogfile->records.first) { /* it has records */
+        /* Finalize metadata log file and create a new one */
+        mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+        ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, ctx->last_fileno + 1);
+        if (likely(!ret)) {
+            ++ctx->last_fileno;
+        }
+    }
+}
+
+void metalog_test_quota(struct metalog_worker_config *wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+    struct metadata_logfile *metalogfile;
+    unsigned current_size;
+    uint8_t only_one_metalogfile;
+
+    metalogfile = ctx->metadata_logfiles.last;
+    current_size = metalogfile->pos;
+    if (unlikely(current_size >= MAX_METALOGFILE_SIZE)) {
+        metalog_try_link_new_metadata_logfile(wc);
+    }
+
+    metalogfile = ctx->metadata_logfiles.last;
+    only_one_metalogfile = (metalogfile == ctx->metadata_logfiles.first) ? 1 : 0;
+    debug(D_METADATALOG, "records=%lu objects=%lu", (long unsigned)ctx->records_nr,
+          (long unsigned)ctx->rrdeng_ctx->host->objects_nr);
+    if (unlikely(!only_one_metalogfile &&
+                 ctx->records_nr > (ctx->rrdeng_ctx->host->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&
+                NO_QUIESCE == ctx->quiesce) {
+        metalog_do_compaction(wc);
+    }
+}
+
+static inline int metalog_threads_alive(struct metalog_worker_config* wc)
+{
+    if (wc->cleanup_thread_compacting_files) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static void metalog_cleanup_finished_threads(struct metalog_worker_config *wc)
+{
+    struct metalog_instance *ctx = wc->ctx;
+
+    if (unlikely(wc->cleanup_thread_compacting_files)) {
+        after_compact_old_records(wc);
+    }
+    if (unlikely(SET_QUIESCE == ctx->quiesce && !metalog_threads_alive(wc))) {
+        ctx->quiesce = QUIESCED;
+        complete(&ctx->metalog_completion);
+    }
+}
+
+static void metalog_init_cmd_queue(struct metalog_worker_config *wc)
+{
+    wc->cmd_queue.head = wc->cmd_queue.tail = 0;
+    wc->queue_size = 0;
+    assert(0 == uv_cond_init(&wc->cmd_cond));
+    assert(0 == uv_mutex_init(&wc->cmd_mutex));
+}
+
+void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd)
+{
+    unsigned queue_size;
+
+    /* wait for free space in queue */
+    uv_mutex_lock(&wc->cmd_mutex);
+    while ((queue_size = wc->queue_size) == METALOG_CMD_Q_MAX_SIZE) {
+        uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
+    }
+    assert(queue_size < METALOG_CMD_Q_MAX_SIZE);
+    /* enqueue command */
+    wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
+    wc->cmd_queue.tail = wc->cmd_queue.tail != METALOG_CMD_Q_MAX_SIZE - 1 ?
+                         wc->cmd_queue.tail + 1 : 0;
+    wc->queue_size = queue_size + 1;
+    uv_mutex_unlock(&wc->cmd_mutex);
+
+    /* wake up event loop */
+    assert(0 == uv_async_send(&wc->async));
+}
+
+struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc)
+{
+    struct metalog_cmd ret;
+    unsigned queue_size;
+
+    uv_mutex_lock(&wc->cmd_mutex);
+    queue_size = wc->queue_size;
+    if (queue_size == 0) {
+        ret.opcode = METALOG_NOOP;
+    } else {
+        /* dequeue command */
+        ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
+        if (queue_size == 1) {
+            wc->cmd_queue.head = wc->cmd_queue.tail = 0;
+        } else {
+            wc->cmd_queue.head = wc->cmd_queue.head != RRDENG_CMD_Q_MAX_SIZE - 1 ?
+                                 wc->cmd_queue.head + 1 : 0;
+        }
+        wc->queue_size = queue_size - 1;
+
+        /* wake up producers */
+        uv_cond_signal(&wc->cmd_cond);
+    }
+    uv_mutex_unlock(&wc->cmd_mutex);
+
+    return ret;
+}
+
+static void async_cb(uv_async_t *handle)
+{
+    uv_stop(handle->loop);
+    uv_update_time(handle->loop);
+    debug(D_METADATALOG, "%s called, active=%d.", __func__, uv_is_active((uv_handle_t *)handle));
+}
+
+/* Flushes metadata log when timer expires */
+#define TIMER_PERIOD_MS (5000)
+
+static void timer_cb(uv_timer_t* handle)
+{
+    struct metalog_worker_config* wc = handle->data;
+    struct metalog_instance *ctx = wc->ctx;
+
+    uv_stop(handle->loop);
+    uv_update_time(handle->loop);
+    metalog_test_quota(wc);
+    debug(D_METADATALOG, "%s: timeout reached.", __func__);
+#ifdef NETDATA_INTERNAL_CHECKS
+    {
+        char buf[4096];
+        debug(D_METADATALOG, "%s", get_metalog_statistics(wc->ctx, buf, sizeof(buf)));
+    }
+#endif
+    mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+}
+
+#define MAX_CMD_BATCH_SIZE (256)
+
+void metalog_worker(void* arg)
+{
+    struct metalog_worker_config *wc = arg;
+    struct metalog_instance *ctx = wc->ctx;
+    uv_loop_t* loop;
+    int shutdown, ret;
+    enum metalog_opcode opcode;
+    uv_timer_t timer_req;
+    struct metalog_cmd cmd;
+    unsigned cmd_batch_size;
+
+    metalog_init_cmd_queue(wc);
+
+    loop = wc->loop = mallocz(sizeof(uv_loop_t));
+    ret = uv_loop_init(loop);
+    if (ret) {
+        error("uv_loop_init(): %s", uv_strerror(ret));
+        goto error_after_loop_init;
+    }
+    loop->data = wc;
+
+    ret = uv_async_init(wc->loop, &wc->async, async_cb);
+    if (ret) {
+        error("uv_async_init(): %s", uv_strerror(ret));
+        goto error_after_async_init;
+    }
+    wc->async.data = wc;
+
+    wc->now_compacting_files = NULL;
+    wc->cleanup_thread_compacting_files = 0;
+
+    /* quota check timer */
+    ret = uv_timer_init(loop, &timer_req);
+    if (ret) {
+        error("uv_timer_init(): %s", uv_strerror(ret));
+        goto error_after_timer_init;
+    }
+    timer_req.data = wc;
+
+    wc->error = 0;
+    /* wake up initialization thread */
+    complete(&ctx->metalog_completion);
+
+    assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
+    shutdown = 0;
+    while (likely(shutdown == 0 || metalog_threads_alive(wc))) {
+        uv_run(loop, UV_RUN_DEFAULT);
+        metalog_cleanup_finished_threads(wc);
+
+        /* wait for commands */
+        cmd_batch_size = 0;
+        do {
+            /*
+             * Avoid starving the loop when there are too many commands coming in.
+             * timer_cb will interrupt the loop again to allow serving more commands.
+             */
+            if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))
+                break;
+
+            cmd = metalog_deq_cmd(wc);
+            opcode = cmd.opcode;
+            ++cmd_batch_size;
+
+            switch (opcode) {
+            case METALOG_NOOP:
+                /* the command queue was empty, do nothing */
+                break;
+            case METALOG_SHUTDOWN:
+                shutdown = 1;
+                break;
+            case METALOG_QUIESCE:
+                ctx->quiesce = SET_QUIESCE;
+                assert(0 == uv_timer_stop(&timer_req));
+                uv_close((uv_handle_t *)&timer_req, NULL);
+                mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+                if (!metalog_threads_alive(wc)) {
+                    ctx->quiesce = QUIESCED;
+                    complete(&ctx->metalog_completion);
+                }
+                break;
+            case METALOG_COMMIT_CREATION_RECORD:
+                do_commit_record(wc, METALOG_CREATE_OBJECT, &cmd.record_io_descr);
+                break;
+            case METALOG_COMMIT_DELETION_RECORD:
+                do_commit_record(wc, METALOG_DELETE_OBJECT, &cmd.record_io_descr);
+                break;
+            case METALOG_COMPACTION_FLUSH:
+                mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log,
+                                         &ctx->compaction_state.new_metadata_logfiles);
+                complete(cmd.record_io_descr.completion);
+                break;
+                default:
+                debug(D_METADATALOG, "%s: default.", __func__);
+                break;
+            }
+        } while (opcode != METALOG_NOOP);
+    }
+
+    /* cleanup operations of the event loop */
+    info("Shutting down RRD metadata log event loop.");
+
+    /*
+     * uv_async_send after uv_close does not seem to crash in linux at the moment,
+     * it is however undocumented behaviour and we need to be aware if this becomes
+     * an issue in the future.
+     */
+    uv_close((uv_handle_t *)&wc->async, NULL);
+
+    mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
+    uv_run(loop, UV_RUN_DEFAULT);
+
+    info("Shutting down RRD metadata log loop complete.");
+    /* TODO: don't let the API block by waiting to enqueue commands */
+    uv_cond_destroy(&wc->cmd_cond);
+/*  uv_mutex_destroy(&wc->cmd_mutex); */
+    assert(0 == uv_loop_close(loop));
+    freez(loop);
+
+    return;
+
+error_after_timer_init:
+    uv_close((uv_handle_t *)&wc->async, NULL);
+error_after_async_init:
+    assert(0 == uv_loop_close(loop));
+error_after_loop_init:
+    freez(loop);
+
+    wc->error = UV_EAGAIN;
+    /* wake up initialization thread */
+    complete(&ctx->metalog_completion);
+}

database/engine/metadata_log/metadatalog.h

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOG_H
+#define NETDATA_METADATALOG_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include "../rrdengine.h"
+#include "metadatalogprotocol.h"
+#include "logfile.h"
+#include "metadatalogapi.h"
+#include "compaction.h"
+
+/* Forward declerations */
+struct metalog_instance;
+struct parser_user_object;
+
+#define MAX_PAGES_PER_EXTENT (64) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
+
+#define METALOG_FILE_NUMBER_SCAN_TMPL "%5u-%5u"
+#define METALOG_FILE_NUMBER_PRINT_TMPL "%5.5u-%5.5u"
+
+#define MAX_DUPLICATION_PERCENTAGE 150 /* the maximum duplication factor of objects in metadata log records */
+
+typedef enum {
+    METALOG_STATUS_UNINITIALIZED = 0,
+    METALOG_STATUS_INITIALIZING,
+    METALOG_STATUS_INITIALIZED
+} metalog_state_t;
+
+struct metalog_record_io_descr {
+    BUFFER *buffer;
+    struct completion *completion;
+    int compacting; /* When 0 append at the end of the metadata log file list.
+                       When 1 append to the temporary compaction metadata log file list. */
+    uuid_t uuid;
+};
+
+enum metalog_opcode {
+    /* can be used to return empty status or flush the command queue */
+    METALOG_NOOP = 0,
+
+    METALOG_SHUTDOWN,
+    METALOG_COMMIT_CREATION_RECORD,
+    METALOG_COMMIT_DELETION_RECORD,
+    METALOG_COMPACTION_FLUSH,
+    METALOG_QUIESCE,
+
+    METALOG_MAX_OPCODE
+};
+
+struct metalog_cmd {
+    enum metalog_opcode opcode;
+    struct metalog_record_io_descr record_io_descr;
+};
+
+#define METALOG_CMD_Q_MAX_SIZE (2048)
+
+struct metalog_cmdqueue {
+    unsigned head, tail;
+    struct metalog_cmd cmd_array[METALOG_CMD_Q_MAX_SIZE];
+};
+
+struct metalog_worker_config {
+    struct metalog_instance *ctx;
+
+    uv_thread_t thread;
+    uv_loop_t *loop;
+    uv_async_t async;
+
+    /* metadata log file comapaction thread */
+    uv_thread_t *now_compacting_files;
+    unsigned long cleanup_thread_compacting_files; /* set to 0 when now_compacting_files is still running */
+
+    /* FIFO command queue */
+    uv_mutex_t cmd_mutex;
+    uv_cond_t cmd_cond;
+    volatile unsigned queue_size;
+    struct metalog_cmdqueue cmd_queue;
+
+    int error;
+};
+
+/*
+ * Debug statistics not used by code logic.
+ * They only describe operations since DB engine instance load time.
+ */
+struct metalog_statistics {
+    rrdeng_stats_t io_write_bytes;
+    rrdeng_stats_t io_write_requests;
+    rrdeng_stats_t io_read_bytes;
+    rrdeng_stats_t io_read_requests;
+    rrdeng_stats_t io_write_record_bytes;
+    rrdeng_stats_t io_write_records;
+    rrdeng_stats_t io_read_record_bytes;
+    rrdeng_stats_t io_read_records;
+    rrdeng_stats_t metadata_logfile_creations;
+    rrdeng_stats_t metadata_logfile_deletions;
+    rrdeng_stats_t io_errors;
+    rrdeng_stats_t fs_errors;
+};
+
+struct metalog_instance {
+    struct rrdengine_instance *rrdeng_ctx;
+    struct metalog_worker_config worker_config;
+    struct completion metalog_completion;
+    struct metadata_record_commit_log records_log;
+    struct metadata_logfile_list metadata_logfiles;
+    struct parser_user_object *metalog_parser_object;
+    struct logfile_compaction_state compaction_state;
+    uint32_t current_compaction_id; /* Every compaction run increments this by 1 */
+    unsigned long disk_space;
+    unsigned long records_nr;
+    unsigned last_fileno; /* newest index of metadata log file */
+
+    uint8_t quiesce; /*
+                      * 0 initial state when all operations function normally
+                      * 1 set it before shutting down the instance, quiesce long running operations
+                      * 2 is set after all threads have finished running
+                      */
+
+    struct metalog_statistics stats;
+};
+
+extern void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode,
+                                  uuid_t *uuid, int compacting);
+    extern int init_metadata_logfiles(struct metalog_instance *ctx);
+extern void finalize_metadata_logfiles(struct metalog_instance *ctx);
+extern void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc);
+extern void metalog_test_quota(struct metalog_worker_config *wc);
+extern void metalog_worker(void* arg);
+extern void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd);
+extern struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc);
+
+#endif /* NETDATA_METADATALOG_H */

database/engine/metadata_log/metadatalogapi.c

@@ -0,0 +1,444 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+
+static inline int metalog_is_initialized(struct metalog_instance *ctx)
+{
+    return ctx->rrdeng_ctx->metalog_ctx != NULL;
+}
+
+static inline void metalog_commit_creation_record(struct metalog_instance *ctx, BUFFER *buffer, uuid_t *uuid)
+{
+    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 0);
+}
+
+static inline void metalog_commit_deletion_record(struct metalog_instance *ctx, BUFFER *buffer)
+{
+    metalog_commit_record(ctx, buffer, METALOG_COMMIT_DELETION_RECORD, NULL, 0);
+}
+
+BUFFER *metalog_update_host_buffer(RRDHOST *host)
+{
+    BUFFER *buffer;
+    buffer = buffer_create(4096); /* This will be freed after it has been committed to the metadata log buffer */
+
+    rrdhost_rdlock(host);
+
+    buffer_sprintf(buffer,
+                   "HOST \"%s\" \"%s\" \"%s\" %d \"%s\" \"%s\" \"%s\"\n",
+//                 "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\""  /* system */
+//                 "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"", /* info   */
+                   host->machine_guid,
+                   host->hostname,
+                   host->registry_hostname,
+                   default_rrd_update_every,
+                   host->os,
+                   host->timezone,
+                   (host->tags) ? host->tags : "");
+
+    netdata_rwlock_rdlock(&host->labels_rwlock);
+    struct label *labels = host->labels;
+    while (labels) {
+        buffer_sprintf(buffer
+            , "LABEL \"%s\" = %d %s\n"
+            , labels->key
+            , (int)labels->label_source
+            , labels->value);
+
+        labels = labels->next;
+    }
+    netdata_rwlock_unlock(&host->labels_rwlock);
+
+    buffer_strcat(buffer, "OVERWRITE labels\n");
+
+    rrdhost_unlock(host);
+    return buffer;
+}
+
+void metalog_commit_update_host(RRDHOST *host)
+{
+    struct metalog_instance *ctx;
+    BUFFER *buffer;
+
+    /* Metadata are only available with dbengine */
+    if (!host->rrdeng_ctx)
+        return;
+
+    ctx = host->rrdeng_ctx->metalog_ctx;
+    if (!ctx) /* metadata log has not been initialized yet */
+        return;
+
+    buffer = metalog_update_host_buffer(host);
+
+    metalog_commit_creation_record(ctx, buffer, &host->host_uuid);
+}
+
+/* compaction_id 0 means it was not called by compaction logic */
+BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id)
+{
+    BUFFER *buffer;
+    RRDHOST *host = st->rrdhost;
+
+    buffer = buffer_create(1024); /* This will be freed after it has been committed to the metadata log buffer */
+
+    rrdset_rdlock(st);
+
+    buffer_sprintf(buffer, "CONTEXT %s\n", host->machine_guid);
+
+    char uuid_str[37];
+    uuid_unparse_lower(*st->chart_uuid, uuid_str);
+    buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+    // properly set the name for the remote end to parse it
+    char *name = "";
+    if(likely(st->name)) {
+        if(unlikely(strcmp(st->id, st->name))) {
+            // they differ
+            name = strchr(st->name, '.');
+            if(name)
+                name++;
+            else
+                name = "";
+        }
+    }
+
+    // send the chart
+    buffer_sprintf(
+        buffer
+        , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
+        , st->id
+        , name
+        , st->title
+        , st->units
+        , st->family
+        , st->context
+        , rrdset_type_name(st->chart_type)
+        , st->priority
+        , st->update_every
+        , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
+        , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
+        , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
+        , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
+        , (st->plugin_name)?st->plugin_name:""
+        , (st->module_name)?st->module_name:""
+    );
+
+    // send the dimensions
+    RRDDIM *rd;
+    rrddim_foreach_read(rd, st) {
+        char uuid_str[37];
+
+        uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+        buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+        buffer_sprintf(
+            buffer
+            , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
+            , rd->id
+            , rd->name
+            , rrd_algorithm_name(rd->algorithm)
+            , rd->multiplier
+            , rd->divisor
+            , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
+            , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
+            , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
+        );
+        if (compaction_id && compaction_id > rd->state->compaction_id) {
+            /* No need to use this dimension again during this compaction cycle */
+            rd->state->compaction_id = compaction_id;
+        }
+    }
+    rrdset_unlock(st);
+    return buffer;
+}
+
+void metalog_commit_update_chart(RRDSET *st)
+{
+    struct metalog_instance *ctx;
+    BUFFER *buffer;
+    RRDHOST *host = st->rrdhost;
+
+    /* Metadata are only available with dbengine */
+    if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+        return;
+
+    ctx = host->rrdeng_ctx->metalog_ctx;
+    if (!ctx) /* metadata log has not been initialized yet */
+        return;
+
+    buffer = metalog_update_chart_buffer(st, 0);
+
+    metalog_commit_creation_record(ctx, buffer, st->chart_uuid);
+}
+
+void metalog_commit_delete_chart(RRDSET *st)
+{
+    struct metalog_instance *ctx;
+    BUFFER *buffer;
+    RRDHOST *host = st->rrdhost;
+    char uuid_str[37];
+
+    /* Metadata are only available with dbengine */
+    if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+        return;
+
+    ctx = host->rrdeng_ctx->metalog_ctx;
+    if (!ctx) /* metadata log has not been initialized yet */
+        return;
+    buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
+
+    uuid_unparse_lower(*st->chart_uuid, uuid_str);
+    buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
+
+    metalog_commit_deletion_record(ctx, buffer);
+}
+
+BUFFER *metalog_update_dimension_buffer(RRDDIM *rd)
+{
+    BUFFER *buffer;
+    RRDSET *st = rd->rrdset;
+    char uuid_str[37];
+
+    buffer = buffer_create(128); /* This will be freed after it has been committed to the metadata log buffer */
+
+    uuid_unparse_lower(*st->chart_uuid, uuid_str);
+    buffer_sprintf(buffer, "CONTEXT %s\n", uuid_str);
+    // Activate random GUID
+    uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+    buffer_sprintf(buffer, "GUID %s\n", uuid_str);
+
+    buffer_sprintf(
+        buffer
+        , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
+        , rd->id
+        , rd->name
+        , rrd_algorithm_name(rd->algorithm)
+        , rd->multiplier
+        , rd->divisor
+        , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
+        , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
+        , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
+    );
+    return buffer;
+}
+
+void metalog_commit_update_dimension(RRDDIM *rd)
+{
+    struct metalog_instance *ctx;
+    BUFFER *buffer;
+    RRDSET *st = rd->rrdset;
+    RRDHOST *host = st->rrdhost;
+
+    /* Metadata are only available with dbengine */
+    if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+        return;
+
+    ctx = host->rrdeng_ctx->metalog_ctx;
+    if (!ctx) /* metadata log has not been initialized yet */
+        return;
+
+    buffer = metalog_update_dimension_buffer(rd);
+
+    metalog_commit_creation_record(ctx, buffer, rd->state->metric_uuid);
+}
+
+void metalog_commit_delete_dimension(RRDDIM *rd)
+{
+    struct metalog_instance *ctx;
+    BUFFER *buffer;
+    RRDSET *st = rd->rrdset;
+    RRDHOST *host = st->rrdhost;
+    char uuid_str[37];
+
+    /* Metadata are only available with dbengine */
+    if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
+        return;
+
+    ctx = host->rrdeng_ctx->metalog_ctx;
+    if (!ctx) /* metadata log has not been initialized yet */
+        return;
+    buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
+
+    uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
+    buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
+
+    metalog_commit_deletion_record(ctx, buffer);
+}
+
+RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
+{
+    GUID_TYPE ret;
+    char chart_object[33], chart_fullid[RRD_ID_LENGTH_MAX + 1];
+    uuid_t *machine_guid, *chart_char_guid;
+
+    ret = find_object_by_guid(chart_uuid, chart_object, 33);
+    assert(GUID_TYPE_CHART == ret);
+
+    machine_guid = (uuid_t *)chart_object;
+    RRDHOST *host = ctx->rrdeng_ctx->host;
+    assert(!uuid_compare(host->host_uuid, *machine_guid));
+
+    chart_char_guid = (uuid_t *)(chart_object + 16);
+
+    ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
+    assert(GUID_TYPE_CHAR == ret);
+    RRDSET *st = rrdset_find(host, chart_fullid);
+
+    return st;
+}
+
+RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
+{
+    GUID_TYPE ret;
+    char dim_object[49], chart_object[33], id_str[PLUGINSD_LINE_MAX], chart_fullid[RRD_ID_LENGTH_MAX + 1];
+    uuid_t *machine_guid, *chart_guid, *chart_char_guid, *dim_char_guid;
+
+    ret = find_object_by_guid(metric_uuid, dim_object, sizeof(dim_object));
+    if (GUID_TYPE_DIMENSION != ret) /* not found */
+        return NULL;
+
+    machine_guid = (uuid_t *)dim_object;
+    RRDHOST *host = ctx->rrdeng_ctx->host;
+    assert(!uuid_compare(host->host_uuid, *machine_guid));
+
+    chart_guid = (uuid_t *)(dim_object + 16);
+    dim_char_guid = (uuid_t *)(dim_object + 16 + 16);
+
+    ret = find_object_by_guid(dim_char_guid, id_str, sizeof(id_str));
+    assert(GUID_TYPE_CHAR == ret);
+
+    ret = find_object_by_guid(chart_guid, chart_object, sizeof(chart_object));
+    assert(GUID_TYPE_CHART == ret);
+    chart_char_guid = (uuid_t *)(chart_object + 16);
+
+    ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
+    assert(GUID_TYPE_CHAR == ret);
+    RRDSET *st = rrdset_find(host, chart_fullid);
+    assert(st);
+
+    RRDDIM *rd = rrddim_find(st, id_str);
+
+    return rd;
+}
+
+/* This function is called by dbengine rotation logic when the metric has no writers */
+void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
+{
+    RRDDIM *rd;
+    RRDSET *st;
+    RRDHOST *host;
+    uint8_t empty_chart;
+
+    rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
+    if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
+        uuid_t multihost_uuid;
+
+        rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->host->machine_guid, metric_uuid, &multihost_uuid);
+        rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);
+    }
+    if(!rd) {
+        info("Rotated unknown archived metric.");
+        return;
+    }
+    st = rd->rrdset;
+    host = st->rrdhost;
+
+    /* Since the metric has no writer it will not be commited to the metadata log by rrddim_free_custom().
+     * It must be commited explicitly before calling rrddim_free_custom(). */
+    metalog_commit_delete_dimension(rd);
+
+    rrdset_wrlock(st);
+    rrddim_free_custom(st, rd, 1);
+    empty_chart = (NULL == st->dimensions);
+    rrdset_unlock(st);
+
+    if (empty_chart) {
+        rrdhost_wrlock(host);
+        rrdset_rdlock(st);
+        rrdset_delete_custom(st, 1);
+        rrdset_unlock(st);
+        rrdset_free(st);
+        rrdhost_unlock(host);
+    }
+}
+
+/*
+ * Returns 0 on success, negative on error
+ */
+int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
+{
+    struct metalog_instance *ctx;
+    int error;
+
+    ctx = callocz(1, sizeof(*ctx));
+    ctx->records_nr = 0;
+    ctx->current_compaction_id = 0;
+    ctx->quiesce = NO_QUIESCE;
+
+    memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
+    ctx->rrdeng_ctx = rrdeng_parent_ctx;
+    ctx->worker_config.ctx = ctx;
+    init_metadata_record_log(&ctx->records_log);
+    error = init_metalog_files(ctx);
+    if (error) {
+        goto error_after_init_rrd_files;
+    }
+
+    init_completion(&ctx->metalog_completion);
+    assert(0 == uv_thread_create(&ctx->worker_config.thread, metalog_worker, &ctx->worker_config));
+    /* wait for worker thread to initialize */
+    wait_for_completion(&ctx->metalog_completion);
+    destroy_completion(&ctx->metalog_completion);
+    uv_thread_set_name_np(ctx->worker_config.thread, "METALOG");
+    if (ctx->worker_config.error) {
+        goto error_after_rrdeng_worker;
+    }
+    rrdeng_parent_ctx->metalog_ctx = ctx; /* notify dbengine that the metadata log has finished initializing */
+    return 0;
+
+error_after_rrdeng_worker:
+    finalize_metalog_files(ctx);
+error_after_init_rrd_files:
+    freez(ctx);
+    return UV_EIO;
+}
+
+/*
+ * Returns 0 on success, 1 on error
+ */
+int metalog_exit(struct metalog_instance *ctx)
+{
+    struct metalog_cmd cmd;
+
+    if (NULL == ctx) {
+        return 1;
+    }
+
+    cmd.opcode = METALOG_SHUTDOWN;
+    metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+    assert(0 == uv_thread_join(&ctx->worker_config.thread));
+
+    finalize_metalog_files(ctx);
+    freez(ctx);
+
+    return 0;
+}
+
+void metalog_prepare_exit(struct metalog_instance *ctx)
+{
+    struct metalog_cmd cmd;
+
+    if (NULL == ctx) {
+        return;
+    }
+
+    init_completion(&ctx->metalog_completion);
+    cmd.opcode = METALOG_QUIESCE;
+    metalog_enq_cmd(&ctx->worker_config, &cmd);
+
+    /* wait for metadata log to quiesce */
+    wait_for_completion(&ctx->metalog_completion);
+    destroy_completion(&ctx->metalog_completion);
+}

database/engine/metadata_log/metadatalogapi.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOGAPI_H
+#define NETDATA_METADATALOGAPI_H
+
+#include "metadatalog.h"
+
+extern BUFFER *metalog_update_host_buffer(RRDHOST *host);
+extern void metalog_commit_update_host(RRDHOST *host);
+extern BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id);
+extern void metalog_commit_update_chart(RRDSET *st);
+extern void metalog_commit_delete_chart(RRDSET *st);
+extern BUFFER *metalog_update_dimension_buffer(RRDDIM *rd);
+extern void metalog_commit_update_dimension(RRDDIM *rd);
+extern void metalog_commit_delete_dimension(RRDDIM *rd);
+
+extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
+extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
+extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
+
+/* must call once before using anything */
+extern int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx);
+extern int metalog_exit(struct metalog_instance *ctx);
+extern void metalog_prepare_exit(struct metalog_instance *ctx);
+
+#endif /* NETDATA_METADATALOGAPI_H */

database/engine/metadata_log/metadatalogprotocol.h

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METADATALOGPROTOCOL_H
+#define NETDATA_METADATALOGPROTOCOL_H
+
+#include "../rrddiskprotocol.h"
+
+#define RRDENG_METALOG_MAGIC "netdata-metadata-log"
+
+#define RRDENG_METALOG_VER (1)
+
+#define RRDENG_METALOG_SB_PADDING_SZ (RRDENG_BLOCK_SIZE - (RRDENG_MAGIC_SZ + sizeof(uint16_t)))
+/*
+ * Metadata log persistent super-block
+ */
+struct rrdeng_metalog_sb {
+    char magic_number[RRDENG_MAGIC_SZ];
+    uint16_t version;
+    uint8_t padding[RRDENG_METALOG_SB_PADDING_SZ];
+} __attribute__ ((packed));
+
+/*
+ * Metadata log record types
+ */
+#define METALOG_STORE_PADDING       (0)
+#define METALOG_CREATE_OBJECT       (1)
+#define METALOG_DELETE_OBJECT       (2)
+#define METALOG_OTHER               (3) /* reserved */
+
+/*
+ * Metadata log record header
+ */
+struct rrdeng_metalog_record_header {
+    /* when set to METALOG_STORE_PADDING jump to start of next block */
+    uint8_t type;
+
+    uint16_t header_length;
+    uint32_t payload_length;
+    /******************************************************
+     * No fields above this point can ever change.        *
+     ******************************************************
+     * All fields below this point are subject to change. *
+     ******************************************************/
+} __attribute__ ((packed));
+
+/*
+ * Metadata log record trailer
+ */
+struct rrdeng_metalog_record_trailer {
+    uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */
+} __attribute__ ((packed));
+
+#endif /* NETDATA_METADATALOGPROTOCOL_H */

database/engine/metadata_log/metalogpluginsd.c

@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#define NETDATA_RRD_INTERNALS
+
+#include "metadatalog.h"
+#include "metalogpluginsd.h"
+
+PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
+                                        char *title, char *units, char *plugin, char *module, int priority,
+                                        int update_every, RRDSET_TYPE chart_type, char *options)
+{
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+    RRDSET *st = NULL;
+    RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
+    uuid_t *chart_uuid;
+
+    chart_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
+    st = rrdset_create_custom(
+        host, type, id, name, family, context, title, units,
+        plugin, module, priority, update_every,
+        chart_type, RRD_MEMORY_MODE_DBENGINE, (host)->rrd_history_entries, 1, chart_uuid);
+
+    if (options && *options) {
+        if (strstr(options, "obsolete"))
+            rrdset_is_obsolete(st);
+        else
+            rrdset_isnot_obsolete(st);
+
+        if (strstr(options, "detail"))
+            rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+        else
+            rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+
+        if (strstr(options, "hidden"))
+            rrdset_flag_set(st, RRDSET_FLAG_HIDDEN);
+        else
+            rrdset_flag_clear(st, RRDSET_FLAG_HIDDEN);
+
+        if (strstr(options, "store_first"))
+            rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+        else
+            rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+    } else {
+        rrdset_isnot_obsolete(st);
+        rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+        rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+    }
+    ((PARSER_USER_OBJECT *)user)->st = st;
+
+    if (chart_uuid) { /* It's a valid object */
+        struct metalog_record record;
+        struct metadata_logfile *metalogfile = state->metalogfile;
+
+        uuid_copy(record.uuid, state->uuid);
+        mlf_record_insert(metalogfile, &record);
+        uuid_clear(state->uuid); /* Consume UUID */
+    }
+    return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+                                            long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type)
+{
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+    UNUSED(user);
+    UNUSED(algorithm);
+    uuid_t *dim_uuid;
+
+    dim_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
+
+    RRDDIM *rd = rrddim_add_custom(st, id, name, multiplier, divisor, algorithm_type, RRD_MEMORY_MODE_DBENGINE, 1,
+                                   dim_uuid);
+    rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
+    rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+    if (options && *options) {
+        if (strstr(options, "obsolete") != NULL)
+            rrddim_is_obsolete(st, rd);
+        else
+            rrddim_isnot_obsolete(st, rd);
+        if (strstr(options, "hidden") != NULL)
+            rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
+        if (strstr(options, "noreset") != NULL)
+            rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+        if (strstr(options, "nooverflow") != NULL)
+            rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+    } else {
+        rrddim_isnot_obsolete(st, rd);
+    }
+    if (dim_uuid) { /* It's a valid object */
+        struct metalog_record record;
+        struct metadata_logfile *metalogfile = state->metalogfile;
+
+        uuid_copy(record.uuid, state->uuid);
+        mlf_record_insert(metalogfile, &record);
+        uuid_clear(state->uuid); /* Consume UUID */
+    }
+    return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)
+{
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+
+    uuid_copy(state->uuid, *uuid);
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
+{
+    GUID_TYPE ret;
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+    struct metalog_instance *ctx = state->ctx;
+    char object[49], chart_object[33], id_str[1024];
+    uuid_t *chart_guid, *chart_char_guid;
+    RRDHOST *host;
+
+    ret = find_object_by_guid(uuid, object, 49);
+    switch (ret) {
+    case GUID_TYPE_CHAR:
+        assert(0);
+        break;
+    case GUID_TYPE_CHART:
+    case GUID_TYPE_DIMENSION:
+        host = ctx->rrdeng_ctx->host;
+        switch (ret) {
+        case GUID_TYPE_CHART:
+            chart_char_guid = (uuid_t *)(object + 16);
+
+            ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
+            assert(GUID_TYPE_CHAR == ret);
+            ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
+            break;
+        case GUID_TYPE_DIMENSION:
+            chart_guid = (uuid_t *)(object + 16);
+
+            ret = find_object_by_guid(chart_guid, chart_object, 33);
+            assert(GUID_TYPE_CHART == ret);
+            chart_char_guid = (uuid_t *)(chart_object + 16);
+
+            ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
+            assert(GUID_TYPE_CHAR == ret);
+            ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
+            break;
+        default:
+            assert(0);
+            break;
+        }
+        break;
+    case GUID_TYPE_HOST:
+        /* Ignore for now */
+        break;
+    default:
+        break;
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
+{
+    GUID_TYPE ret;
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+    struct metalog_instance *ctx = state->ctx;
+    RRDHOST *host = ctx->rrdeng_ctx->host;
+    RRDSET *st;
+    RRDDIM *rd;
+
+    ret = find_object_by_guid(uuid, NULL, 0);
+    switch (ret) {
+        case GUID_TYPE_CHAR:
+            assert(0);
+            break;
+        case GUID_TYPE_CHART:
+            st = metalog_get_chart_from_uuid(ctx, uuid);
+            if (st) {
+                rrdhost_wrlock(host);
+                rrdset_free(st);
+                rrdhost_unlock(host);
+            }
+            break;
+        case GUID_TYPE_DIMENSION:
+            rd = metalog_get_dimension_from_uuid(ctx, uuid);
+            if (rd) {
+                st = rd->rrdset;
+                rrdset_wrlock(st);
+                rrddim_free_custom(st, rd, 0);
+                rrdset_unlock(st);
+            }
+            break;
+        case GUID_TYPE_HOST:
+            /* Ignore for now */
+            break;
+        default:
+            break;
+    }
+
+    return PARSER_RC_OK;
+}
+
+void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx)
+{
+    state->ctx = ctx;
+    state->skip_record = 0;
+    uuid_clear(state->uuid);
+    state->metalogfile = NULL;
+}

database/engine/metadata_log/metalogpluginsd.h

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_METALOGPLUGINSD_H
+#define NETDATA_METALOGPLUGINSD_H
+
+#include "../../../collectors/plugins.d/pluginsd_parser.h"
+#include "../../../collectors/plugins.d/plugins_d.h"
+#include "../../../parser/parser.h"
+
+struct metalog_pluginsd_state {
+    struct metalog_instance *ctx;
+    uuid_t uuid;
+    uint8_t skip_record; /* skip this record due to errors in parsing */
+    struct metadata_logfile *metalogfile; /* current metadata log file being replayed */
+};
+
+extern void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx);
+
+extern PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family,
+                                               char *context, char *title, char *units, char *plugin, char *module,
+                                               int priority, int update_every, RRDSET_TYPE chart_type, char *options);
+extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
+                                                   long multiplier, long divisor, char *options,
+                                                   RRD_ALGORITHM algorithm_type);
+extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
+extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
+extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
+
+#endif /* NETDATA_METALOGPLUGINSD_H */

database/engine/pagecache.c

@@ -383,17 +383,26 @@ static int pg_cache_try_evict_one_page_unsafe(struct rrdengine_instance *ctx)
     return 0;
 }
 
-/*
- * Callers of this function need to make sure they're not deleting the same descriptor concurrently
+/**
+ * Deletes a page from the database.
+ * Callers of this function need to make sure they're not deleting the same descriptor concurrently.
+ * @param ctx is the database instance.
+ * @param descr is the page descriptor.
+ * @param remove_dirty must be non-zero if the page to be deleted is dirty.
+ * @param is_exclusive_holder must be non-zero if the caller holds an exclusive page reference.
+ * @param metric_id is set to the metric the page belongs to, if it's safe to delete the metric and metric_id is not
+ *        NULL. Otherwise, metric_id is not set.
+ * @return 1 if it's safe to delete the metric, 0 otherwise.
  */
-void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
-                         uint8_t is_exclusive_holder)
+uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
+                         uint8_t is_exclusive_holder, uuid_t *metric_id)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct page_cache_descr *pg_cache_descr = NULL;
     Pvoid_t *PValue;
     struct pg_cache_page_index *page_index;
     int ret;
+    uint8_t can_delete_metric = 0;
 
     uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
     PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, descr->id, sizeof(uuid_t));
@@ -403,14 +412,22 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
 
     uv_rwlock_wrlock(&page_index->lock);
     ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
-    uv_rwlock_wrunlock(&page_index->lock);
     if (unlikely(0 == ret)) {
+        uv_rwlock_wrunlock(&page_index->lock);
         error("Page under deletion was not in index.");
         if (unlikely(debug_flags & D_RRDENGINE)) {
             print_page_descr(descr);
         }
         goto destroy;
     }
+    --page_index->page_count;
+    if (!page_index->writers && !page_index->page_count) {
+        can_delete_metric = 1;
+        if (metric_id) {
+            memcpy(metric_id, page_index->id, sizeof(uuid_t));
+        }
+    }
+    uv_rwlock_wrunlock(&page_index->lock);
     assert(1 == ret);
 
     uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);
@@ -459,6 +476,8 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
 destroy:
     freez(descr);
     pg_cache_update_metric_times(page_index);
+
+    return can_delete_metric;
 }
 
 static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t start_time, usec_t end_time)
@@ -588,6 +607,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
     uv_rwlock_wrlock(&page_index->lock);
     PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
     *PValue = descr;
+    ++page_index->page_count;
     pg_cache_add_new_metric_time(page_index, descr);
     uv_rwlock_wrunlock(&page_index->lock);
 
@@ -1032,6 +1052,8 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
     page_index->oldest_time = INVALID_TIME;
     page_index->latest_time = INVALID_TIME;
     page_index->prev = NULL;
+    page_index->page_count = 0;
+    page_index->writers = 0;
 
     return page_index;
 }

database/engine/pagecache.h

@@ -85,6 +85,8 @@ struct pg_cache_page_index {
      * TODO: examine if we want to support better granularity than seconds
      */
     Pvoid_t JudyL_array;
+    Word_t page_count;
+    unsigned short writers;
     uv_rwlock_t lock;
 
     /*
@@ -163,8 +165,8 @@ extern void pg_cache_put_unsafe(struct rrdeng_page_descr *descr);
 extern void pg_cache_put(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr);
 extern void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index *index,
                             struct rrdeng_page_descr *descr);
-extern void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
-                                uint8_t is_exclusive_holder);
+extern uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
+                                   uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
 extern usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
                                             usec_t start_time, usec_t end_time);
 extern void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,

database/engine/rrdengine.c

@@ -304,7 +304,7 @@ static void invalidate_oldest_committed(void *arg)
 
             goto out;
         }
-        pg_cache_punch_hole(ctx, descr, 1, 1);
+        pg_cache_punch_hole(ctx, descr, 1, 1, NULL);
 
         uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
         nr_committed_pages = --pg_cache->committed_page_index.nr_committed_pages;
@@ -326,6 +326,9 @@ void rrdeng_invalidate_oldest_committed(struct rrdengine_worker_config* wc)
     unsigned nr_committed_pages;
     int error;
 
+    if (unlikely(ctx->quiesce != NO_QUIESCE)) /* Shutting down */
+        return;
+
     uv_rwlock_rdlock(&pg_cache->committed_page_index.lock);
     nr_committed_pages = pg_cache->committed_page_index.nr_committed_pages;
     uv_rwlock_rdunlock(&pg_cache->committed_page_index.lock);
@@ -630,6 +633,8 @@ static void delete_old_data(void *arg)
     struct extent_info *extent, *next;
     struct rrdeng_page_descr *descr;
     unsigned count, i;
+    uint8_t can_delete_metric;
+    uuid_t metric_id;
 
     /* Safe to use since it will be deleted after we are done */
     datafile = ctx->datafiles.first;
@@ -638,7 +643,14 @@ static void delete_old_data(void *arg)
         count = extent->number_of_pages;
         for (i = 0 ; i < count ; ++i) {
             descr = extent->pages[i];
-            pg_cache_punch_hole(ctx, descr, 0, 0);
+            can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);
+            if (unlikely(can_delete_metric && ctx->metalog_ctx)) {
+                /*
+                 * If the metric is empty, has no active writers and if the metadata log has been initialized then
+                 * attempt to delete the corresponding netdata dimension.
+                 */
+                metalog_delete_dimension_by_uuid(ctx->metalog_ctx, &metric_id);
+            }
         }
         next = extent->next;
         freez(extent);
@@ -674,7 +686,7 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
             ++ctx->last_fileno;
         }
     }
-    if (unlikely(out_of_space)) {
+    if (unlikely(out_of_space && NO_QUIESCE == ctx->quiesce)) {
         /* delete old data */
         if (wc->now_deleting_files) {
             /* already deleting data */
@@ -710,12 +722,18 @@ static inline int rrdeng_threads_alive(struct rrdengine_worker_config* wc)
 
 static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
 {
+    struct rrdengine_instance *ctx = wc->ctx;
+
     if (unlikely(wc->cleanup_thread_invalidating_dirty_pages)) {
         after_invalidate_oldest_committed(wc);
     }
     if (unlikely(wc->cleanup_thread_deleting_files)) {
         after_delete_old_data(wc);
     }
+    if (unlikely(SET_QUIESCE == ctx->quiesce && !rrdeng_threads_alive(wc))) {
+        ctx->quiesce = QUIESCED;
+        complete(&ctx->rrdengine_completion);
+    }
 }
 
 /* return 0 on success */
@@ -799,14 +817,16 @@ void async_cb(uv_async_t *handle)
 void timer_cb(uv_timer_t* handle)
 {
     struct rrdengine_worker_config* wc = handle->data;
+    struct rrdengine_instance *ctx = wc->ctx;
 
     uv_stop(handle->loop);
     uv_update_time(handle->loop);
+    if (unlikely(!ctx->metalog_ctx))
+        return; /* Wait for the metadata log to initialize */
     rrdeng_test_quota(wc);
     debug(D_RRDENGINE, "%s: timeout reached.", __func__);
     if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
         /* There is free space so we can write to disk and we are not actively deleting dirty buffers */
-        struct rrdengine_instance *ctx = wc->ctx;
         struct page_cache *pg_cache = &ctx->pg_cache;
         unsigned long total_bytes, bytes_written, nr_committed_pages, bytes_to_write = 0, producers, low_watermark,
                       high_watermark;
@@ -920,7 +940,20 @@ void rrdeng_worker(void* arg)
                 break;
             case RRDENG_SHUTDOWN:
                 shutdown = 1;
+                break;
+            case RRDENG_QUIESCE:
                 ctx->drop_metrics_under_page_cache_pressure = 0;
+                ctx->quiesce = SET_QUIESCE;
+                assert(0 == uv_timer_stop(&timer_req));
+                uv_close((uv_handle_t *)&timer_req, NULL);
+                while (do_flush_pages(wc, 1, NULL)) {
+                    ; /* Force flushing of all committed pages. */
+                }
+                wal_flush_transaction_buffer(wc);
+                if (!rrdeng_threads_alive(wc)) {
+                    ctx->quiesce = QUIESCED;
+                    complete(&ctx->rrdengine_completion);
+                }
                 break;
             case RRDENG_READ_PAGE:
                 do_read_extent(wc, &cmd.read_page.page_cache_descr, 1, 0);
@@ -959,8 +992,6 @@ void rrdeng_worker(void* arg)
      * an issue in the future.
      */
     uv_close((uv_handle_t *)&wc->async, NULL);
-    assert(0 == uv_timer_stop(&timer_req));
-    uv_close((uv_handle_t *)&timer_req, NULL);
 
     while (do_flush_pages(wc, 1, NULL)) {
         ; /* Force flushing of all committed pages. */
@@ -998,7 +1029,7 @@ void rrdengine_main(void)
     struct rrdengine_instance *ctx;
 
     sanity_check();
-    ret = rrdeng_init(&ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
+    ret = rrdeng_init(NULL, &ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
     if (ret) {
         exit(ret);
     }

database/engine/rrdengine.h

@@ -17,6 +17,7 @@
 #include "rrdenginelib.h"
 #include "datafile.h"
 #include "journalfile.h"
+#include "metadata_log/metadatalog.h"
 #include "rrdengineapi.h"
 #include "pagecache.h"
 #include "rrdenglocking.h"
@@ -50,6 +51,7 @@ enum rrdeng_opcode {
     RRDENG_FLUSH_PAGES,
     RRDENG_SHUTDOWN,
     RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE,
+    RRDENG_QUIESCE,
 
     RRDENG_MAX_OPCODE
 };
@@ -169,7 +171,13 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
 extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;
 extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */
 
+#define NO_QUIESCE  (0) /* initial state when all operations function normally */
+#define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
+#define QUIESCED    (2) /* is set after all threads have finished running */
+
 struct rrdengine_instance {
+    RRDHOST *host;
+    struct metalog_instance *metalog_ctx;
     struct rrdengine_worker_config worker_config;
     struct completion rrdengine_completion;
     struct page_cache pg_cache;
@@ -185,6 +193,8 @@ struct rrdengine_instance {
     unsigned long cache_pages_low_watermark;
     unsigned long metric_API_max_producers;
 
+    uint8_t quiesce; /* set to SET_QUIESCE before shutdown of the engine */
+
     struct rrdengine_statistics stats;
 };
 

database/engine/rrdengineapi.c

@@ -9,6 +9,114 @@ int default_rrdeng_disk_quota_mb = 256;
 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
 
+/* This UUID is not unique across hosts */
+void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
+{
+    EVP_MD_CTX *evpctx;
+    unsigned char hash_value[EVP_MAX_MD_SIZE];
+    unsigned int hash_len;
+
+    evpctx = EVP_MD_CTX_create();
+    EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
+    EVP_DigestUpdate(evpctx, dim_id, strlen(dim_id));
+    EVP_DigestUpdate(evpctx, chart_id, strlen(chart_id));
+    EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
+    EVP_MD_CTX_destroy(evpctx);
+    assert(hash_len > sizeof(uuid_t));
+    memcpy(ret_uuid, hash_value, sizeof(uuid_t));
+}
+
+/* Transform legacy UUID to be unique across hosts deterministacally */
+void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid, uuid_t *ret_uuid)
+{
+    EVP_MD_CTX *evpctx;
+    unsigned char hash_value[EVP_MAX_MD_SIZE];
+    unsigned int hash_len;
+
+    evpctx = EVP_MD_CTX_create();
+    EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
+    EVP_DigestUpdate(evpctx, machine_guid, GUID_LEN);
+    EVP_DigestUpdate(evpctx, *legacy_uuid, sizeof(uuid_t));
+    EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
+    EVP_MD_CTX_destroy(evpctx);
+    assert(hash_len > sizeof(uuid_t));
+    memcpy(ret_uuid, hash_value, sizeof(uuid_t));
+}
+
+void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)
+{
+    struct page_cache *pg_cache;
+    struct rrdengine_instance *ctx;
+    uuid_t legacy_uuid;
+    Pvoid_t *PValue;
+    struct pg_cache_page_index *page_index;
+    int replace_instead_of_generate = 0;
+
+    ctx = rd->rrdset->rrdhost->rrdeng_ctx;
+    pg_cache = &ctx->pg_cache;
+
+    rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
+    rd->state->metric_uuid = callocz(1, sizeof(uuid_t));
+
+    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+    PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &legacy_uuid, sizeof(uuid_t));
+    if (likely(NULL != PValue)) {
+        page_index = *PValue;
+    }
+    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+    if (NULL == PValue) {
+        /* First time we see the legacy UUID, drop legacy support, normal path */
+
+        if (NULL != dim_uuid) {
+            replace_instead_of_generate = 1;
+            uuid_copy(*rd->state->metric_uuid, *dim_uuid);
+        }
+        if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION,
+                                           replace_instead_of_generate))) {
+            errno = 0;
+            error("FAILED to generate GUID for %s", rd->id);
+            freez(rd->state->metric_uuid);
+            rd->state->metric_uuid = NULL;
+            assert(0);
+        }
+
+        uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+        PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t));
+        if (likely(NULL != PValue)) {
+            page_index = *PValue;
+        }
+        uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+        if (NULL == PValue) {
+            uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
+            PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t), PJE0);
+            assert(NULL == *PValue); /* TODO: figure out concurrency model */
+            *PValue = page_index = create_page_index(rd->state->metric_uuid);
+            page_index->prev = pg_cache->metrics_index.last_page_index;
+            pg_cache->metrics_index.last_page_index = page_index;
+            uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
+        }
+    } else {
+        /* There are legacy UUIDs in the database, implement backward compatibility */
+
+
+        rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
+                                                rd->state->metric_uuid);
+        if (dim_uuid && uuid_compare(*rd->state->metric_uuid, *dim_uuid)) {
+            error("Mismatch of metadata log DIMENSION GUID with dbengine metric GUID.");
+        }
+        if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION, 1))) {
+            errno = 0;
+            error("FAILED to generate GUID for %s", rd->id);
+            freez(rd->state->metric_uuid);
+            rd->state->metric_uuid = NULL;
+            assert(0);
+        }
+    }
+    rd->state->rrdeng_uuid = &page_index->id;
+    rd->state->page_index = page_index;
+    rd->state->compaction_id = 0;
+}
+
 /*
  * Gets a handle for storing metrics to the database.
  * The handle must be released with rrdeng_store_metric_final().
@@ -16,53 +124,21 @@ uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
 void rrdeng_store_metric_init(RRDDIM *rd)
 {
     struct rrdeng_collect_handle *handle;
-    struct page_cache *pg_cache;
     struct rrdengine_instance *ctx;
-    uuid_t temp_id;
-    Pvoid_t *PValue;
     struct pg_cache_page_index *page_index;
-    EVP_MD_CTX *evpctx;
-    unsigned char hash_value[EVP_MAX_MD_SIZE];
-    unsigned int hash_len;
-
-    //&default_global_ctx; TODO: test this use case or remove it?
 
     ctx = rd->rrdset->rrdhost->rrdeng_ctx;
-    pg_cache = &ctx->pg_cache;
     handle = &rd->state->handle.rrdeng;
     handle->ctx = ctx;
 
-    evpctx = EVP_MD_CTX_create();
-    EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
-    EVP_DigestUpdate(evpctx, rd->id, strlen(rd->id));
-    EVP_DigestUpdate(evpctx, rd->rrdset->id, strlen(rd->rrdset->id));
-    EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
-    EVP_MD_CTX_destroy(evpctx);
-    assert(hash_len > sizeof(temp_id));
-    memcpy(&temp_id, hash_value, sizeof(temp_id));
-
     handle->descr = NULL;
     handle->prev_descr = NULL;
     handle->unaligned_page = 0;
 
-    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
-    PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t));
-    if (likely(NULL != PValue)) {
-        page_index = *PValue;
-    }
-    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
-    if (NULL == PValue) {
-        /* First time we see the UUID */
-        uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
-        PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t), PJE0);
-        assert(NULL == *PValue); /* TODO: figure out concurrency model */
-        *PValue = page_index = create_page_index(&temp_id);
-        page_index->prev = pg_cache->metrics_index.last_page_index;
-        pg_cache->metrics_index.last_page_index = page_index;
-        uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
-    }
-    rd->state->rrdeng_uuid = &page_index->id;
-    handle->page_index = page_index;
+    page_index = rd->state->page_index;
+    uv_rwlock_wrlock(&page_index->lock);
+    ++page_index->writers;
+    uv_rwlock_wrunlock(&page_index->lock);
 }
 
 /* The page must be populated and referenced */
@@ -109,7 +185,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
             if (unlikely(debug_flags & D_RRDENGINE))
                 print_page_cache_descr(descr);
             pg_cache_put(ctx, descr);
-            pg_cache_punch_hole(ctx, descr, 1, 0);
+            pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
             handle->prev_descr = NULL;
         } else {
             /* added 1 extra reference to keep 2 dirty pages pinned per metric, expected refcnt = 2 */
@@ -170,7 +246,7 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
                  must_flush_unaligned_page)) {
         rrdeng_store_metric_flush_current_page(rd);
 
-        page = rrdeng_create_page(ctx, &handle->page_index->id, &descr);
+        page = rrdeng_create_page(ctx, &rd->state->page_index->id, &descr);
         assert(page);
 
         handle->descr = descr;
@@ -204,27 +280,38 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
             }
         }
 
-        pg_cache_insert(ctx, handle->page_index, descr);
+        pg_cache_insert(ctx, rd->state->page_index, descr);
     } else {
-        pg_cache_add_new_metric_time(handle->page_index, descr);
+        pg_cache_add_new_metric_time(rd->state->page_index, descr);
     }
 }
 
 /*
  * Releases the database reference from the handle for storing metrics.
+ * Returns 1 if it's safe to delete the dimension.
  */
-void rrdeng_store_metric_finalize(RRDDIM *rd)
+int rrdeng_store_metric_finalize(RRDDIM *rd)
 {
     struct rrdeng_collect_handle *handle;
     struct rrdengine_instance *ctx;
+    struct pg_cache_page_index *page_index;
+    uint8_t can_delete_metric = 0;
 
     handle = &rd->state->handle.rrdeng;
     ctx = handle->ctx;
+    page_index = rd->state->page_index;
     rrdeng_store_metric_flush_current_page(rd);
     if (handle->prev_descr) {
         /* unpin old second page */
         pg_cache_put(ctx, handle->prev_descr);
     }
+    uv_rwlock_wrlock(&page_index->lock);
+    if (!--page_index->writers && !page_index->page_count) {
+        can_delete_metric = 1;
+    }
+    uv_rwlock_wrunlock(&page_index->lock);
+
+   return can_delete_metric;
 }
 
 /* Returns 1 if the data collection interval is well defined, 0 otherwise */
@@ -577,21 +664,17 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
 
 time_t rrdeng_metric_latest_time(RRDDIM *rd)
 {
-    struct rrdeng_collect_handle *handle;
     struct pg_cache_page_index *page_index;
 
-    handle = &rd->state->handle.rrdeng;
-    page_index = handle->page_index;
+    page_index = rd->state->page_index;
 
     return page_index->latest_time / USEC_PER_SEC;
 }
 time_t rrdeng_metric_oldest_time(RRDDIM *rd)
 {
-    struct rrdeng_collect_handle *handle;
     struct pg_cache_page_index *page_index;
 
-    handle = &rd->state->handle.rrdeng;
-    page_index = handle->page_index;
+    page_index = rd->state->page_index;
 
     return page_index->oldest_time / USEC_PER_SEC;
 }
@@ -765,7 +848,8 @@ void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle)
 /*
  * Returns 0 on success, negative on error
  */
-int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb, unsigned disk_space_mb)
+int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
+                unsigned disk_space_mb)
 {
     struct rrdengine_instance *ctx;
     int error;
@@ -776,8 +860,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
     /* reserve RRDENG_FD_BUDGET_PER_INSTANCE file descriptors for this instance */
     rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, RRDENG_FD_BUDGET_PER_INSTANCE);
     if (rrdeng_reserved_file_descriptors > max_open_files) {
-        error("Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
-              (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
+        error(
+            "Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
+            (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
 
         rrd_stat_atomic_add(&global_fs_errors, 1);
         rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
@@ -804,6 +889,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
     ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';
     ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
     ctx->metric_API_max_producers = 0;
+    ctx->quiesce = NO_QUIESCE;
+    ctx->metalog_ctx = NULL; /* only set this after the metadata log has finished initializing */
+    ctx->host = host;
 
     memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
     ctx->worker_config.ctx = ctx;
@@ -823,6 +911,11 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
     if (ctx->worker_config.error) {
         goto error_after_rrdeng_worker;
     }
+    error = metalog_init(ctx);
+    if(error) {
+        error("Failed to initialize metadata log file event loop.");
+        goto error_after_rrdeng_worker;
+    }
     return 0;
 
 error_after_rrdeng_worker:
@@ -855,6 +948,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
     assert(0 == uv_thread_join(&ctx->worker_config.thread));
 
     finalize_rrd_files(ctx);
+    metalog_exit(ctx->metalog_ctx);
     free_page_cache(ctx);
 
     if (ctx != &default_global_ctx) {
@@ -863,3 +957,23 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
     rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
     return 0;
 }
+
+void rrdeng_prepare_exit(struct rrdengine_instance *ctx)
+{
+    struct rrdeng_cmd cmd;
+
+    if (NULL == ctx) {
+        return;
+    }
+
+    init_completion(&ctx->rrdengine_completion);
+    cmd.opcode = RRDENG_QUIESCE;
+    rrdeng_enq_cmd(&ctx->worker_config, &cmd);
+
+    /* wait for dbengine to quiesce */
+    wait_for_completion(&ctx->rrdengine_completion);
+    destroy_completion(&ctx->rrdengine_completion);
+
+    metalog_prepare_exit(ctx->metalog_ctx);
+}
+

database/engine/rrdengineapi.h

@@ -28,10 +28,17 @@ extern void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_pag
 extern void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **handle);
 extern void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle);
 extern void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle);
+
+extern void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid);
+extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
+                                                    uuid_t *ret_uuid);
+
+
+extern void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid);
 extern void rrdeng_store_metric_init(RRDDIM *rd);
 extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
 extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number number);
-extern void rrdeng_store_metric_finalize(RRDDIM *rd);
+extern int rrdeng_store_metric_finalize(RRDDIM *rd);
 extern unsigned
     rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
                                     struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp);
@@ -45,9 +52,10 @@ extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
 extern void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
 
 /* must call once before using anything */
-extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
+extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
                        unsigned disk_space_mb);
 
 extern int rrdeng_exit(struct rrdengine_instance *ctx);
+extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
 
 #endif /* NETDATA_RRDENGINEAPI_H */

database/engine/rrdenginelib.c

@@ -78,17 +78,22 @@ int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size)
     return 0;
 }
 
-/*
- * Tries to open a file in direct I/O mode, falls back to buffered mode if not possible.
- * Returns UV error number that is < 0 on failure.
- * On success sets (*file) to be the uv_file that was opened.
+/**
+ * Open file for I/O.
+ *
+ * @param path The full path of the file.
+ * @param flags Same flags as the open() system call uses.
+ * @param file On success sets (*file) to be the uv_file that was opened.
+ * @param direct Tries to open a file in direct I/O mode when direct=1, falls back to buffered mode if not possible.
+ * @return Returns UV error number that is < 0 on failure. 0 on success.
  */
-int open_file_direct_io(char *path, int flags, uv_file *file)
+int open_file_for_io(char *path, int flags, uv_file *file, int direct)
 {
     uv_fs_t req;
-    int fd, current_flags, direct;
+    int fd, current_flags;
 
-    for (direct = 1 ; direct >= 0 ; --direct) {
+    assert(0 == direct || 1 == direct);
+    for ( ; direct >= 0 ; --direct) {
 #ifdef __APPLE__
         /* Apple OS does not support O_DIRECT */
         direct = 0;

database/engine/rrdenginelib.h

@@ -100,7 +100,15 @@ static inline void crc32set(void *crcp, uLong crc)
 extern void print_page_cache_descr(struct rrdeng_page_descr *page_cache_descr);
 extern void print_page_descr(struct rrdeng_page_descr *descr);
 extern int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size);
-extern int open_file_direct_io(char *path, int flags, uv_file *file);
+extern int open_file_for_io(char *path, int flags, uv_file *file, int direct);
+static inline int open_file_direct_io(char *path, int flags, uv_file *file)
+{
+    return open_file_for_io(path, flags, file, 1);
+}
+static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
+{
+    return open_file_for_io(path, flags, file, 0);
+}
 extern char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t size);
 
 #endif /* NETDATA_RRDENGINELIB_H */

database/rrd.h

@@ -16,6 +16,7 @@ typedef struct alarm_entry ALARM_ENTRY;
 
 // forward declarations
 struct rrddim_volatile;
+struct rrdset_volatile;
 #ifdef ENABLE_DBENGINE
 struct rrdeng_page_descr;
 struct rrdengine_instance;
@@ -31,6 +32,11 @@ struct pg_cache_page_index;
 #include "rrdcalctemplate.h"
 #include "../streaming/rrdpush.h"
 
+#define META_CHART_UPDATED 1
+#define META_PLUGIN_UPDATED 2
+#define META_MODULE_UPDATED 4
+#define META_CHART_ACTIVATED 8
+
 #define UPDATE_EVERY 1
 #define UPDATE_EVERY_MAX 3600
 
@@ -136,7 +142,10 @@ typedef enum rrddim_flags {
     RRDDIM_FLAG_NONE                            = 0,
     RRDDIM_FLAG_HIDDEN                          = (1 << 0),  // this dimension will not be offered to callers
     RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1),  // do not offer RESET or OVERFLOW info to callers
-    RRDDIM_FLAG_OBSOLETE                        = (1 << 2)   // this is marked by the collector/module as obsolete
+    RRDDIM_FLAG_OBSOLETE                        = (1 << 2),  // this is marked by the collector/module as obsolete
+    // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at
+    // least rrdset_free_obsolete_time seconds ago.
+    RRDDIM_FLAG_ARCHIVED                        = (1 << 3)
 } RRDDIM_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
@@ -274,7 +283,6 @@ union rrddim_collect_handle {
         struct rrdeng_page_descr *descr, *prev_descr;
         unsigned long page_correlation_id;
         struct rrdengine_instance *ctx;
-        struct pg_cache_page_index *page_index;
         // set to 1 when this dimension is not page aligned with the other dimensions in the chart
         uint8_t unaligned_page;
     } rrdeng; // state the database engine uses
@@ -312,6 +320,9 @@ struct rrddim_query_handle {
 struct rrddim_volatile {
 #ifdef ENABLE_DBENGINE
     uuid_t *rrdeng_uuid;                 // database engine metric UUID
+    uuid_t *metric_uuid;                 // global UUID for this metric (unique_across hosts)
+    struct pg_cache_page_index *page_index;
+    uint32_t compaction_id;              // The last metadata log compaction procedure that has processed this object.
 #endif
     union rrddim_collect_handle handle;
     // ------------------------------------------------------------------------
@@ -324,7 +335,8 @@ struct rrddim_volatile {
         void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);
 
         // an finalization function to run after collection is over
-        void (*finalize)(RRDDIM *rd);
+        // returns 1 if it's safe to delete the dimension
+        int (*finalize)(RRDDIM *rd);
     } collect_ops;
 
     // function pointers that handle database queries
@@ -349,6 +361,14 @@ struct rrddim_volatile {
     } query_ops;
 };
 
+// ----------------------------------------------------------------------------
+// volatile state per chart
+struct rrdset_volatile {
+    char *old_title;
+    char *old_family;
+    char *old_context;
+};
+
 // ----------------------------------------------------------------------------
 // these loop macros make sure the linked list is accessed with the right lock
 
@@ -382,7 +402,10 @@ typedef enum rrdset_flags {
     RRDSET_FLAG_HOMOGENEOUS_CHECK   = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous
     RRDSET_FLAG_HIDDEN              = 1 << 12, // if set, do not show this chart on the dashboard, but use it for backends
     RRDSET_FLAG_SYNC_CLOCK          = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
-    RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14  // this is marked by the collector/module when a chart has obsolete dimensions
+    RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions
+    // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
+    // least rrdset_free_obsolete_time seconds ago.
+    RRDSET_FLAG_ARCHIVED            = 1 << 15
 } RRDSET_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
@@ -459,8 +482,11 @@ struct rrdset {
 
     char *plugin_name;                              // the name of the plugin that generated this
     char *module_name;                              // the name of the plugin module that generated this
-
-    size_t unused[5];
+    uuid_t *chart_uuid;                             // Store the global GUID for this chart
+    size_t compaction_id;                           // The last metadata log compaction procedure that has processed
+                                                    // this object.
+    struct rrdset_volatile *state;                  // volatile state that is not persistently stored
+    size_t unused[2];
 
     size_t rrddim_page_alignment;                   // keeps metric pages in alignment when using dbengine
 
@@ -783,6 +809,10 @@ struct rrdhost {
 
 #ifdef ENABLE_DBENGINE
     struct rrdengine_instance *rrdeng_ctx;          // DB engine instance for this host
+    uuid_t  host_uuid;                              // Global GUID for this host
+    unsigned long objects_nr;                       // Number of charts and dimensions in this host
+    uint32_t compaction_id;                         // The last metadata log compaction procedure that has processed
+                                                    // this object.
 #endif
 
 #ifdef ENABLE_HTTPS
@@ -847,6 +877,26 @@ extern RRDHOST *rrdhost_find_or_create(
         , struct rrdhost_system_info *system_info
 );
 
+extern void rrdhost_update(RRDHOST *host
+    , const char *hostname
+    , const char *registry_hostname
+    , const char *guid
+    , const char *os
+    , const char *timezone
+    , const char *tags
+    , const char *program_name
+    , const char *program_version
+    , int update_every
+    , long history
+    , RRD_MEMORY_MODE mode
+    , unsigned int health_enabled
+    , unsigned int rrdpush_enabled
+    , char *rrdpush_destination
+    , char *rrdpush_api_key
+    , char *rrdpush_send_charts_matching
+    , struct rrdhost_system_info *system_info
+);
+
 extern int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);
 
 #if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)
@@ -892,10 +942,12 @@ extern RRDSET *rrdset_create_custom(RRDHOST *host
                              , int update_every
                              , RRDSET_TYPE chart_type
                              , RRD_MEMORY_MODE memory_mode
-                             , long history_entries);
+                             , long history_entries
+                             , int is_archived
+                             , uuid_t *chart_uuid);
 
 #define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
-    rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
+    rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries, 0, NULL)
 
 #define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
     rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
@@ -919,6 +971,14 @@ extern RRDSET *rrdset_find(RRDHOST *host, const char *id);
 
 extern RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
 #define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
+/* This will not return charts that are archived */
+static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
+{
+    RRDSET *st = rrdset_find_bytype_localhost(type, id);
+    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
+        return NULL;
+    return st;
+}
 
 extern RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
 #define rrdset_find_byname_localhost(name)  rrdset_find_byname(localhost, name)
@@ -933,8 +993,9 @@ extern void rrdset_is_obsolete(RRDSET *st);
 extern void rrdset_isnot_obsolete(RRDSET *st);
 
 // checks if the RRDSET should be offered to viewers
-#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
-#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions)
+#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
+#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
+#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
 
 // get the total duration in seconds of the round robin database
 #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
@@ -1062,8 +1123,11 @@ static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
 // RRD DIMENSION functions
 
 extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host);
-extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);
-#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
+extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
+                                 collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
+                                 int is_archived, uuid_t *dim_uuid);
+#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \
+                                                                                   algorithm, (st)->rrd_memory_mode, 0, NULL)
 
 extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
 extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
@@ -1071,6 +1135,15 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
 extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
 
 extern RRDDIM *rrddim_find(RRDSET *st, const char *id);
+/* This will not return dimensions that are archived */
+static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id)
+{
+    RRDDIM *rd = rrddim_find(st, id);
+    if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
+        return NULL;
+    return rd;
+}
+
 
 extern int rrddim_hide(RRDSET *st, const char *id);
 extern int rrddim_unhide(RRDSET *st, const char *id);
@@ -1099,7 +1172,8 @@ extern avl_tree_lock rrdhost_root_index;
 extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
 extern char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section);
 
-extern void rrddim_free(RRDSET *st, RRDDIM *rd);
+#define rrddim_free(st, rd) rrddim_free_custom(st, rd, 0)
+extern void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated);
 
 extern int rrddim_compare(void* a, void* b);
 extern int rrdset_compare(void* a, void* b);
@@ -1116,7 +1190,8 @@ extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
 extern void rrdset_free(RRDSET *st);
 extern void rrdset_reset(RRDSET *st);
 extern void rrdset_save(RRDSET *st);
-extern void rrdset_delete(RRDSET *st);
+#define rrdset_delete(st) rrdset_delete_custom(st, 0)
+extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
 extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
 
 extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);

database/rrddim.c

@@ -100,10 +100,10 @@ static void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storag
 
     rd->values[rd->rrdset->current_entry] = number;
 }
-static void rrddim_collect_finalize(RRDDIM *rd) {
+static int rrddim_collect_finalize(RRDDIM *rd) {
     (void)rd;
 
-    return;
+    return 0;
 }
 
 // ----------------------------------------------------------------------------
@@ -189,7 +189,9 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
 #endif
 }
 
-RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) {
+RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
+                          collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
+                          int is_archived, uuid_t *dim_uuid) {
     RRDHOST *host = st->rrdhost;
     rrdset_wrlock(st);
 
@@ -200,11 +202,21 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     if(unlikely(rd)) {
         debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
 
-        rrddim_set_name(st, rd, name);
-        rrddim_set_algorithm(st, rd, algorithm);
-        rrddim_set_multiplier(st, rd, multiplier);
-        rrddim_set_divisor(st, rd, divisor);
-
+        int rc = rrddim_set_name(st, rd, name);
+        rc += rrddim_set_algorithm(st, rd, algorithm);
+        rc += rrddim_set_multiplier(st, rd, multiplier);
+        rc += rrddim_set_divisor(st, rd, divisor);
+        if (!is_archived && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
+            rd->state->collect_ops.init(rd);
+            rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
+        }
+        // DBENGINE available and activated?
+#ifdef ENABLE_DBENGINE
+        if (likely(!is_archived && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) {
+            debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
+            metalog_commit_update_dimension(rd);
+        }
+#endif
         rrdset_unlock(st);
         return rd;
     }
@@ -301,7 +313,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         else
             rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
     }
-
     rd->memsize = size;
 
     strcpy(rd->magic, RRDDIMENSION_MAGIC);
@@ -350,15 +361,16 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     rd->state = mallocz(sizeof(*rd->state));
     if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
 #ifdef ENABLE_DBENGINE
-        rd->state->collect_ops.init         = rrdeng_store_metric_init;
+        rrdeng_metric_init(rd, dim_uuid);
+        rd->state->collect_ops.init = rrdeng_store_metric_init;
         rd->state->collect_ops.store_metric = rrdeng_store_metric_next;
-        rd->state->collect_ops.finalize     = rrdeng_store_metric_finalize;
-        rd->state->query_ops.init           = rrdeng_load_metric_init;
-        rd->state->query_ops.next_metric    = rrdeng_load_metric_next;
-        rd->state->query_ops.is_finished    = rrdeng_load_metric_is_finished;
-        rd->state->query_ops.finalize       = rrdeng_load_metric_finalize;
-        rd->state->query_ops.latest_time    = rrdeng_metric_latest_time;
-        rd->state->query_ops.oldest_time    = rrdeng_metric_oldest_time;
+        rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
+        rd->state->query_ops.init = rrdeng_load_metric_init;
+        rd->state->query_ops.next_metric = rrdeng_load_metric_next;
+        rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
+        rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
+        rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
+        rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
 #endif
     } else {
         rd->state->collect_ops.init         = rrddim_collect_init;
@@ -371,7 +383,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         rd->state->query_ops.latest_time    = rrddim_query_latest_time;
         rd->state->query_ops.oldest_time    = rrddim_query_oldest_time;
     }
-    rd->state->collect_ops.init(rd);
+    if (is_archived)
+        rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
+    else
+        rd->state->collect_ops.init(rd); // only initialize if a collector created this dimension
     // append this dimension
     if(!st->dimensions)
         st->dimensions = rd;
@@ -432,17 +447,30 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     if (netdata_cloud_setting)
         aclk_update_chart(host, st->id, ACLK_CMD_CHART);
 #endif
+#ifdef ENABLE_DBENGINE
+    rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
+    metalog_commit_update_dimension(rd);
+#endif
+
     return(rd);
 }
 
 // ----------------------------------------------------------------------------
 // RRDDIM remove / free a dimension
 
-void rrddim_free(RRDSET *st, RRDDIM *rd)
+void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
 {
     debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
 
-    rd->state->collect_ops.finalize(rd);
+    if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
+        uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
+        if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+#ifdef ENABLE_DBENGINE
+            /* This metric has no data and no references */
+            metalog_commit_delete_dimension(rd);
+#endif
+        }
+    }
     freez(rd->state);
 
     if(rd == st->dimensions)
@@ -486,9 +514,12 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
             break;
     }
 #ifdef ENABLE_ACLK
-    if (netdata_cloud_setting)
+    if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != rd->rrd_memory_mode))
         aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
 #endif
+#ifdef ENABLE_DBENGINE
+    rrd_atomic_fetch_add(&st->rrdhost->objects_nr, -1);
+#endif
 }
 
 
@@ -541,6 +572,10 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
     if (netdata_cloud_setting)
         aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
 #endif
+#ifdef ENABLE_DBENGINE
+    metalog_commit_update_dimension(rd);
+#endif
+
 }
 
 inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
@@ -551,6 +586,9 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
     if (netdata_cloud_setting)
         aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
 #endif
+#ifdef ENABLE_DBENGINE
+    metalog_commit_update_dimension(rd);
+#endif
 }
 
 // ----------------------------------------------------------------------------

database/rrdhost.c

@@ -245,6 +245,9 @@ RRDHOST *rrdhost_create(const char *hostname,
     }
     if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
 #ifdef ENABLE_DBENGINE
+        uuid_parse(host->machine_guid, host->host_uuid);
+        host->objects_nr = 1;
+        host->compaction_id = 0;
         char dbenginepath[FILENAME_MAX + 1];
         int ret;
 
@@ -253,7 +256,7 @@ RRDHOST *rrdhost_create(const char *hostname,
         if(ret != 0 && errno != EEXIST)
             error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
         else
-            ret = rrdeng_init(&host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
+            ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
         if(ret) {
             error("Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
                   host->hostname, host->machine_guid, host->cache_dir);
@@ -361,9 +364,91 @@ RRDHOST *rrdhost_create(const char *hostname,
 
     rrd_hosts_available++;
 
+#ifdef ENABLE_DBENGINE
+    if (likely(!is_localhost && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
+            metalog_commit_update_host(host);
+#endif
     return host;
 }
 
+void rrdhost_update(RRDHOST *host
+                    , const char *hostname
+                    , const char *registry_hostname
+                    , const char *guid
+                    , const char *os
+                    , const char *timezone
+                    , const char *tags
+                    , const char *program_name
+                    , const char *program_version
+                    , int update_every
+                    , long history
+                    , RRD_MEMORY_MODE mode
+                    , unsigned int health_enabled
+                    , unsigned int rrdpush_enabled
+                    , char *rrdpush_destination
+                    , char *rrdpush_api_key
+                    , char *rrdpush_send_charts_matching
+                    , struct rrdhost_system_info *system_info
+)
+{
+    UNUSED(guid);
+    UNUSED(rrdpush_enabled);
+    UNUSED(rrdpush_destination);
+    UNUSED(rrdpush_api_key);
+    UNUSED(rrdpush_send_charts_matching);
+
+    host->health_enabled = health_enabled;
+    //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION;        Unused?
+
+    rrdhost_system_info_free(host->system_info);
+    host->system_info = system_info;
+
+    rrdhost_init_os(host, os);
+    rrdhost_init_timezone(host, timezone);
+
+    freez(host->registry_hostname);
+    host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
+
+    if(strcmp(host->hostname, hostname) != 0) {
+        info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
+        char *t = host->hostname;
+        host->hostname = strdupz(hostname);
+        host->hash_hostname = simple_hash(host->hostname);
+        freez(t);
+    }
+
+    if(strcmp(host->program_name, program_name) != 0) {
+        info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
+        char *t = host->program_name;
+        host->program_name = strdupz(program_name);
+        freez(t);
+    }
+
+    if(strcmp(host->program_version, program_version) != 0) {
+        info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
+        char *t = host->program_version;
+        host->program_version = strdupz(program_version);
+        freez(t);
+    }
+
+    if(host->rrd_update_every != update_every)
+        error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
+
+    if(host->rrd_history_entries < history)
+        error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
+
+    if(host->rrd_memory_mode != mode)
+        error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
+
+    // update host tags
+    rrdhost_init_tags(host, tags);
+#ifdef ENABLE_DBENGINE
+    if (likely(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
+        metalog_commit_update_host(host);
+#endif
+    return;
+}
+
 RRDHOST *rrdhost_find_or_create(
           const char *hostname
         , const char *registry_hostname
@@ -410,42 +495,24 @@ RRDHOST *rrdhost_find_or_create(
         );
     }
     else {
-        host->health_enabled = health_enabled;
-        //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION;        Unused?
-
-        if(strcmp(host->hostname, hostname) != 0) {
-            info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
-            char *t = host->hostname;
-            host->hostname = strdupz(hostname);
-            host->hash_hostname = simple_hash(host->hostname);
-            freez(t);
-        }
-
-        if(strcmp(host->program_name, program_name) != 0) {
-            info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
-            char *t = host->program_name;
-            host->program_name = strdupz(program_name);
-            freez(t);
-        }
-
-        if(strcmp(host->program_version, program_version) != 0) {
-            info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
-            char *t = host->program_version;
-            host->program_version = strdupz(program_version);
-            freez(t);
-        }
-
-        if(host->rrd_update_every != update_every)
-            error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
-
-        if(host->rrd_history_entries < history)
-            error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
-
-        if(host->rrd_memory_mode != mode)
-            error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
-
-        // update host tags
-        rrdhost_init_tags(host, tags);
+        rrdhost_update(host
+           , hostname
+           , registry_hostname
+           , guid
+           , os
+           , timezone
+           , tags
+           , program_name
+           , program_version
+           , update_every
+           , history
+           , mode
+           , health_enabled
+           , rrdpush_enabled
+           , rrdpush_destination
+           , rrdpush_api_key
+           , rrdpush_send_charts_matching
+           , system_info);
     }
 
     rrdhost_cleanup_orphan_hosts_nolock(host);
@@ -454,7 +521,6 @@ RRDHOST *rrdhost_find_or_create(
 
     return host;
 }
-
 inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) {
     if(host != protected
        && host != localhost
@@ -618,6 +684,9 @@ void rrdhost_free(RRDHOST *host) {
     // ------------------------------------------------------------------------
     // release its children resources
 
+#ifdef ENABLE_DBENGINE
+    rrdeng_prepare_exit(host->rrdeng_ctx);
+#endif
     while(host->rrdset_root)
         rrdset_free(host->rrdset_root);
 
@@ -1322,7 +1391,17 @@ restart_after_removal:
                     && st->last_updated.tv_sec + rrdset_free_obsolete_time < now
                     && st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now
         )) {
-
+#ifdef ENABLE_DBENGINE
+            if(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+                rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
+                rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+                if (st->dimensions) {
+                    /* If the chart still has dimensions don't delete it from the metadata log */
+                    continue;
+                }
+                metalog_commit_delete_chart(st);
+            }
+#endif
             rrdset_rdlock(st);
 
             if(rrdhost_delete_obsolete_charts)

database/rrdset.c

@@ -2,6 +2,7 @@
 
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
+#include <sched.h>
 
 void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
     debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id);
@@ -260,7 +261,7 @@ void rrdset_reset(RRDSET *st) {
         rd->collections_counter = 0;
         // memset(rd->values, 0, rd->entries * sizeof(storage_number));
 #ifdef ENABLE_DBENGINE
-        if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode) {
+        if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode && !rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
             rrdeng_store_metric_flush_current_page(rd);
         }
 #endif
@@ -366,6 +367,10 @@ void rrdset_free(RRDSET *st) {
     freez(st->config_section);
     freez(st->plugin_name);
     freez(st->module_name);
+    freez(st->state->old_title);
+    freez(st->state->old_family);
+    freez(st->state->old_context);
+    freez(st->state);
 
     switch(st->rrd_memory_mode) {
         case RRD_MEMORY_MODE_SAVE:
@@ -381,6 +386,10 @@ void rrdset_free(RRDSET *st) {
             freez(st);
             break;
     }
+#ifdef ENABLE_DBENGINE
+    rrd_atomic_fetch_add(&host->objects_nr, -1);
+#endif
+
 }
 
 void rrdset_save(RRDSET *st) {
@@ -402,7 +411,7 @@ void rrdset_save(RRDSET *st) {
     }
 }
 
-void rrdset_delete(RRDSET *st) {
+void rrdset_delete_custom(RRDSET *st, int db_rotated) {
     RRDDIM *rd;
 
     rrdset_check_rdlock(st);
@@ -425,11 +434,12 @@ void rrdset_delete(RRDSET *st) {
 
     recursively_delete_dir(st->cache_dir, "left-over chart");
 #ifdef ENABLE_ACLK
-    if (netdata_cloud_setting) {
+    if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)) {
         aclk_del_collector(st->rrdhost->hostname, st->plugin_name, st->module_name);
         aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHARTDEL);
     }
 #endif
+
 }
 
 void rrdset_delete_obsolete_dimensions(RRDSET *st) {
@@ -480,6 +490,8 @@ RRDSET *rrdset_create_custom(
         , RRDSET_TYPE chart_type
         , RRD_MEMORY_MODE memory_mode
         , long history_entries
+        , int is_archived
+        , uuid_t *chart_uuid
 ) {
     if(!type || !type[0]) {
         fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
@@ -516,15 +528,139 @@ RRDSET *rrdset_create_custom(
     snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
 
     RRDSET *st = rrdset_find_on_create(host, fullid);
-    if(st) {
+    if (st) {
+        int mark_rebuild = 0;
         rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
         rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+        if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
+            rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
+            mark_rebuild |= META_CHART_ACTIVATED;
+        }
+        char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
+             *old_title_v = NULL, *old_family_v = NULL, *old_context_v = NULL;
+        const char *new_name = name ? name : id;
 
-        if(unlikely(name))
-            rrdset_set_name(st, name);
-        else
-            rrdset_set_name(st, id);
+        if (unlikely((st->name && !strcmp(st->name, new_name)) || !st->name)) {
+            mark_rebuild |= META_CHART_UPDATED;
+            rrdset_set_name(st, new_name);
+        }
 
+        if (unlikely(st->priority != priority)) {
+            st->priority = priority;
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+        if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != update_every)) {
+            st->update_every = update_every;
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+
+        if (plugin && st->plugin_name) {
+            if (unlikely(strcmp(plugin, st->plugin_name))) {
+                old_plugin = st->plugin_name;
+                st->plugin_name = strdupz(plugin);
+                mark_rebuild |= META_PLUGIN_UPDATED;
+            }
+        } else {
+            if (plugin != st->plugin_name) { // one is NULL?
+                old_plugin = st->plugin_name;
+                st->plugin_name = plugin ? strdupz(plugin) : NULL;
+                mark_rebuild |= META_PLUGIN_UPDATED;
+            }
+        }
+
+        if (module && st->module_name) {
+            if (unlikely(strcmp(module, st->module_name))) {
+                old_module = st->module_name;
+                st->module_name = strdupz(module);
+                mark_rebuild |= META_MODULE_UPDATED;
+            }
+        } else {
+            if (module != st->module_name) {
+                if (st->module_name && *st->module_name) {
+                    old_module = st->module_name;
+                    st->module_name = module ? strdupz(module) : NULL;
+                    mark_rebuild |= META_MODULE_UPDATED;
+                }
+            }
+        }
+
+        if (unlikely(title && st->state->old_title && strcmp(st->state->old_title, title))) {
+            char *new_title = strdupz(title);
+            old_title_v = st->state->old_title;
+            st->state->old_title = strdupz(title);
+            json_fix_string(new_title);
+            old_title = st->title;
+            st->title = new_title;
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+
+        RRDSET_TYPE new_chart_type =
+            rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
+        if (st->chart_type != new_chart_type) {
+            st->chart_type = new_chart_type;
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+
+        if (unlikely(family && st->state->old_family && strcmp(st->state->old_family, family))) {
+            char *new_family = strdupz(family);
+            old_family_v = st->state->old_family;
+            st->state->old_family = strdupz(family);
+            json_fix_string(new_family);
+            old_family = st->family;
+            rrdfamily_free(host, st->rrdfamily);
+            st->family = new_family;
+            st->rrdfamily = rrdfamily_create(host, st->family);
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+
+        if (unlikely(context && st->state->old_context && strcmp(st->state->old_context, context))) {
+            char *new_context = strdupz(context);
+            old_context_v = st->state->old_context;
+            st->state->old_context = strdupz(context);
+            json_fix_string(new_context);
+            old_context = st->context;
+            st->context = new_context;
+            st->hash_context = simple_hash(st->context);
+            mark_rebuild |= META_CHART_UPDATED;
+        }
+
+        if (mark_rebuild) {
+#ifdef ENABLE_ACLK
+            if (netdata_cloud_setting) {
+                if (mark_rebuild & META_CHART_ACTIVATED) {
+                    aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
+                }
+                else {
+                    if (mark_rebuild & (META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
+                        aclk_del_collector(
+                            host->hostname, mark_rebuild & META_PLUGIN_UPDATED ? old_plugin : st->plugin_name,
+                            mark_rebuild & META_MODULE_UPDATED ? old_module : st->module_name);
+                        aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
+                    }
+                }
+                aclk_update_chart(host, st->id, ACLK_CMD_CHART);
+            }
+#endif
+            freez(old_plugin);
+            freez(old_module);
+            freez(old_title);
+            freez(old_family);
+            freez(old_context);
+            freez(old_title_v);
+            freez(old_family_v);
+            freez(old_context_v);
+            if (mark_rebuild != META_CHART_ACTIVATED) {
+                info("Collector updated metadata for chart %s", st->id);
+                sched_yield();
+            }
+        }
+#ifdef ENABLE_DBENGINE
+        if (is_archived == 0 && st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
+            (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) {
+            debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
+            metalog_commit_update_chart(st);
+        }
+#endif
         return st;
     }
 
@@ -535,6 +671,9 @@ RRDSET *rrdset_create_custom(
         rrdhost_unlock(host);
         rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
         rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+        if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
+            rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
+        }
         return st;
     }
 
@@ -664,6 +803,8 @@ RRDSET *rrdset_create_custom(
         else
             st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
     }
+    if (is_archived)
+        rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
 
     st->plugin_name = plugin?strdupz(plugin):NULL;
     st->module_name = module?strdupz(module):NULL;
@@ -687,13 +828,16 @@ RRDSET *rrdset_create_custom(
     st->chart_type = rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
     st->type       = config_get(st->config_section, "type", type);
 
+    st->state = mallocz(sizeof(*st->state));
     st->family     = config_get(st->config_section, "family", family?family:st->type);
+    st->state->old_family = strdupz(st->family);
     json_fix_string(st->family);
 
     st->units      = config_get(st->config_section, "units", units?units:"");
     json_fix_string(st->units);
 
     st->context    = config_get(st->config_section, "context", context?context:st->id);
+    st->state->old_context = strdupz(st->context);
     json_fix_string(st->context);
     st->hash_context = simple_hash(st->context);
 
@@ -745,6 +889,7 @@ RRDSET *rrdset_create_custom(
         rrdset_set_name(st, id);
 
     st->title = config_get(st->config_section, "title", title);
+    st->state->old_title = strdupz(st->title);
     json_fix_string(st->title);
 
     st->rrdfamily = rrdfamily_create(host, st->family);
@@ -765,6 +910,26 @@ RRDSET *rrdset_create_custom(
 
     rrdsetcalc_link_matching(st);
     rrdcalctemplate_link_matching(st);
+#ifdef ENABLE_DBENGINE
+    if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+        int replace_instead_of_generate = 0;
+
+        st->chart_uuid = callocz(1, sizeof(uuid_t));
+        if (NULL != chart_uuid) {
+            replace_instead_of_generate = 1;
+            uuid_copy(*st->chart_uuid, *chart_uuid);
+        }
+        if (unlikely(
+                find_or_generate_guid((void *) st, st->chart_uuid, GUID_TYPE_CHART, replace_instead_of_generate))) {
+            errno = 0;
+            error("FAILED to generate GUID for %s", st->id);
+            freez(st->chart_uuid);
+            st->chart_uuid = NULL;
+            assert(0);
+        }
+        st->compaction_id = 0;
+    }
+#endif
 
     rrdhost_cleanup_obsolete_charts(host);
 
@@ -775,6 +940,11 @@ RRDSET *rrdset_create_custom(
         aclk_update_chart(host, st->id, ACLK_CMD_CHART);
     }
 #endif
+#ifdef ENABLE_DBENGINE
+    rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
+    metalog_commit_update_chart(st);
+#endif
+
     return(st);
 }
 
@@ -1006,6 +1176,9 @@ static inline size_t rrdset_done_interpolate(
         last_ut = next_store_ut;
 
         rrddim_foreach_read(rd, st) {
+            if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+                continue;
+
             calculated_number new_value;
 
             switch(rd->algorithm) {
@@ -1374,6 +1547,8 @@ void rrdset_done(RRDSET *st) {
     int dimensions = 0;
     st->collected_total = 0;
     rrddim_foreach_read(rd, st) {
+        if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+            continue;
         dimensions++;
         if(likely(rd->updated))
             st->collected_total += rd->collected_value;
@@ -1385,6 +1560,8 @@ void rrdset_done(RRDSET *st) {
     // based on the collected figures only
     // at this stage we do not interpolate anything
     rrddim_foreach_read(rd, st) {
+        if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+            continue;
 
         if(unlikely(!rd->updated)) {
             rd->calculated_value = 0;
@@ -1630,6 +1807,8 @@ void rrdset_done(RRDSET *st) {
     st->last_collected_total  = st->collected_total;
 
     rrddim_foreach_read(rd, st) {
+        if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
+            continue;
         if(unlikely(!rd->updated))
             continue;
 
@@ -1692,7 +1871,7 @@ void rrdset_done(RRDSET *st) {
     // find if there are any obsolete dimensions
     time_t now = now_realtime_sec();
 
-    if(unlikely(rrddim_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
+    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
         rrddim_foreach_read(rd, st)
             if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)))
                 break;
@@ -1714,6 +1893,21 @@ void rrdset_done(RRDSET *st) {
                             error("Cannot delete dimension file '%s'", rd->cache_filename);
                     }
 
+#ifdef ENABLE_DBENGINE
+                    if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+                        rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
+                        rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
+                        /* only a collector can mark a chart as obsolete, so we must remove the reference */
+                        uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
+                        if (can_delete_metric) {
+                            /* This metric has no data and no references */
+                            metalog_commit_delete_dimension(rd);
+                        } else {
+                            /* Do not delete this dimension */
+                            continue;
+                        }
+                    }
+#endif
                     if(unlikely(!last)) {
                         rrddim_free(st, rd);
                         rd = st->dimensions;

exporting/tests/netdata_doubles.c

@@ -100,7 +100,9 @@ RRDSET *rrdset_create_custom(
     int update_every,
     RRDSET_TYPE chart_type,
     RRD_MEMORY_MODE memory_mode,
-    long history_entries)
+    long history_entries,
+    int is_archived,
+    uuid_t *chart_uuid)
 {
     check_expected_ptr(host);
     check_expected_ptr(type);
@@ -117,6 +119,8 @@ RRDSET *rrdset_create_custom(
     check_expected(chart_type);
     UNUSED(memory_mode);
     UNUSED(history_entries);
+    UNUSED(is_archived);
+    UNUSED(chart_uuid);
 
     function_called();
 

health/health.c

@@ -488,6 +488,11 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
         return 0;
     }
 
+    if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) {
+        debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rc->chart?rc->chart:"NOCHART", rc->name);
+        return 0;
+    }
+
     if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
         debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
         return 0;

libnetdata/log/log.h

@@ -38,6 +38,8 @@
 #define D_STREAM            0x0000000040000000
 #define D_RRDENGINE         0x0000000100000000
 #define D_ACLK              0x0000000200000000
+#define D_METADATALOG       0x0000000400000000
+#define D_GUIDLOG           0x0000000800000000
 #define D_SYSTEM            0x8000000000000000
 
 //#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)

parser/parser.h

@@ -31,6 +31,10 @@ typedef struct pluginsd_action {
     PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
     PARSER_RC (*label_action)(void *user, char *key, char *value, LABEL_SOURCE source);
     PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, struct label *new_labels);
+
+    PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
+    PARSER_RC (*context_action)(void *user, uuid_t *uuid);
+    PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid);
 } PLUGINSD_ACTION;
 
 typedef enum parser_input_type {
@@ -101,5 +105,8 @@ extern PARSER_RC pluginsd_flush(char **words, void *user, PLUGINSD_ACTION  *plug
 extern PARSER_RC pluginsd_disable(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
 extern PARSER_RC pluginsd_label(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
 extern PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
+extern PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
+extern PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
+extern PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
 
 #endif

streaming/receiver.c

@@ -279,6 +279,25 @@ static int rrdpush_receive(struct receiver_state *rpt)
         }
         netdata_mutex_unlock(&rpt->host->receiver_lock);
     }
+    else rrdhost_update(rpt->host
+            , rpt->hostname
+            , rpt->registry_hostname
+            , rpt->machine_guid
+            , rpt->os
+            , rpt->timezone
+            , rpt->tags
+            , program_name
+            , program_version
+            , rpt->update_every
+            , history
+            , mode
+            , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
+            , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
+            , rrdpush_destination
+            , rrdpush_api_key
+            , rrdpush_send_charts_matching
+            , rpt->system_info);
+
 
     int ssl = 0;
 #ifdef ENABLE_HTTPS

web/api/formatters/charts2json.c

@@ -36,7 +36,7 @@ static inline const char* get_release_channel() {
     return (use_stable)?"stable":"nightly";
 }
 
-void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
+void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived) {
     static char *custom_dashboard_info_js_filename = NULL;
     size_t c, dimensions = 0, memory = 0, alarms = 0;
     RRDSET *st;
@@ -71,7 +71,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
     c = 0;
     rrdhost_rdlock(host);
     rrdset_foreach_read(st, host) {
-        if(rrdset_is_available_for_viewers(st)) {
+        if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
             if(c) buffer_strcat(wb, ",");
             buffer_strcat(wb, "\n\t\t\"");
             buffer_strcat(wb, st->id);

web/api/formatters/charts2json.h

@@ -5,7 +5,7 @@
 
 #include "rrd2json.h"
 
-extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile);
+extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
 extern void chartcollectors2json(RRDHOST *host, BUFFER *wb);
 
 #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H

web/api/web_api_v1.c

@@ -349,7 +349,16 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
 
     buffer_flush(w->response.data);
     w->response.data->contenttype = CT_APPLICATION_JSON;
-    charts2json(host, w->response.data, 0);
+    charts2json(host, w->response.data, 0, 0);
+    return HTTP_RESP_OK;
+}
+
+inline int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url) {
+    (void)url;
+
+    buffer_flush(w->response.data);
+    w->response.data->contenttype = CT_APPLICATION_JSON;
+    charts2json(host, w->response.data, 0, 1);
     return HTTP_RESP_OK;
 }
 
@@ -915,6 +924,7 @@ static struct api_command {
         { "data",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_data            },
         { "chart",           0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_chart           },
         { "charts",          0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_charts          },
+        { "archivedcharts",  0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_archivedcharts  },
 
         // registry checks the ACL by itself, so we allow everything
         { "registry",        0, WEB_CLIENT_ACL_NOCHECK,   web_client_api_request_v1_registry        },

web/api/web_api_v1.h

@@ -19,6 +19,7 @@ extern int web_client_api_request_single_chart(RRDHOST *host, struct web_client
 extern int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
 extern int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
 extern int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
 extern int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
 extern int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
 extern int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);