diff --git a/CMakeLists.txt b/CMakeLists.txt
index d0e397b5f5..86ffc710f5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1170,6 +1170,8 @@ set(DAEMON_FILES
src/daemon/pulse/pulse-daemon-memory-system.c
src/daemon/pulse/pulse-network.c
src/daemon/pulse/pulse-network.h
+ src/daemon/pulse/pulse-db-dbengine-retention.c
+ src/daemon/pulse/pulse-db-dbengine-retention.h
)
set(H2O_FILES
diff --git a/src/daemon/config/netdata-conf-db.c b/src/daemon/config/netdata-conf-db.c
index 98e20c92a0..2f239757a3 100644
--- a/src/daemon/config/netdata-conf-db.c
+++ b/src/daemon/config/netdata-conf-db.c
@@ -320,7 +320,7 @@ void netdata_conf_dbengine_init(const char *hostname) {
for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++)
rrdeng_readiness_wait(multidb_ctx[tier]);
- calculate_tier_disk_space_percentage();
+ rrdeng_calculate_tier_disk_space_percentage();
dbengine_enabled = true;
#else
diff --git a/src/daemon/pulse/pulse-db-dbengine-retention.c b/src/daemon/pulse/pulse-db-dbengine-retention.c
new file mode 100644
index 0000000000..31e72b5e70
--- /dev/null
+++ b/src/daemon/pulse/pulse-db-dbengine-retention.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "pulse-db-dbengine-retention.h"
+#ifdef ENABLE_DBENGINE
+#include "database/engine/rrdengineapi.h"
+
+void dbengine_retention_statistics(bool extended __maybe_unused) {
+
+ static bool init = false;
+ static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
+
+ if (!localhost)
+ return;
+
+ rrdeng_calculate_tier_disk_space_percentage();
+
+ for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
+ STORAGE_ENGINE *eng = localhost->db[tier].eng;
+ if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
+ continue;
+
+ if (init == false) {
+ char id[200];
+ snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
+ stats[tier].st = rrdset_create_localhost(
+ "netdata",
+ id,
+ NULL,
+ "dbengine retention",
+ "netdata.dbengine_tier_retention",
+ "dbengine space and time retention",
+ "%",
+ "netdata",
+ "stats",
+ 134900, // before "dbengine memory" (dbengine2_statistics_charts)
+ 10,
+ RRDSET_TYPE_LINE);
+
+ stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ char tier_str[5];
+ snprintfz(tier_str, 4, "%zu", tier);
+ rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
+
+ rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
+ rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
+ rrdset_metadata_updated(stats[tier].st);
+ }
+
+ time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
+ time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
+
+ //
+ // Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
+ // get_used_disk_space is used to determine if database cleanup (file rotation should happen)
+ // and adds to the disk space used the desired file size of the active
+ // datafile
+ uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier]);
+ //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
+
+ uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
+ if (!config_disk_space) {
+ config_disk_space = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
+ config_disk_space += disk_space;
+ }
+
+ collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
+
+ collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
+ 100 * retention / multidb_ctx[tier]->config.max_retention_s :
+ 0;
+
+ if (retention_percentage > 100)
+ retention_percentage = 100;
+
+ rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
+ rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
+
+ rrdset_done(stats[tier].st);
+ }
+ init = true;
+}
+#endif
diff --git a/src/daemon/pulse/pulse-db-dbengine-retention.h b/src/daemon/pulse/pulse-db-dbengine-retention.h
new file mode 100644
index 0000000000..c3f6465d1f
--- /dev/null
+++ b/src/daemon/pulse/pulse-db-dbengine-retention.h
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PULSE_DB_DBENGINE_RETENTION_H
+#define NETDATA_PULSE_DB_DBENGINE_RETENTION_H
+
+#include "libnetdata/libnetdata.h"
+
+#ifdef ENABLE_DBENGINE
+void dbengine_retention_statistics(bool extended __maybe_unused);
+#endif
+
+#endif //NETDATA_PULSE_DB_DBENGINE_RETENTION_H
diff --git a/src/daemon/pulse/pulse-db-dbengine.c b/src/daemon/pulse/pulse-db-dbengine.c
index 961b6b6a32..a050abf0a4 100644
--- a/src/daemon/pulse/pulse-db-dbengine.c
+++ b/src/daemon/pulse/pulse-db-dbengine.c
@@ -686,7 +686,6 @@ void pulse_dbengine_do(bool extended) {
mrg_get_statistics(main_mrg, &mrg_stats);
size_t priority = 135000;
-
{
static RRDSET *st_pgc_memory = NULL;
static RRDDIM *rd_pgc_memory_main = NULL;
diff --git a/src/daemon/pulse/pulse-network.c b/src/daemon/pulse/pulse-network.c
index 352f5ec451..cb57f65fcb 100644
--- a/src/daemon/pulse/pulse-network.c
+++ b/src/daemon/pulse/pulse-network.c
@@ -124,7 +124,7 @@ static void pulse_aclk_time_heatmap(void) {
rds[0] = rrddim_add(st, "instant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
for(size_t i = 1; i < _countof(rds) - 1 ;i++) {
- char buf[128];
+ char buf[350];
snprintf(buf, sizeof(buf), "%.2fs", (double)aclk_time_heatmap.array[i].upto / (double)USEC_PER_SEC);
// duration_snprintf(buf, sizeof(buf), aclk_time_heatmap.array[i].upto, "us", false);
rds[i] = rrddim_add(st, buf, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
diff --git a/src/daemon/pulse/pulse.c b/src/daemon/pulse/pulse.c
index 8bdca13e6c..c5f8c5e226 100644
--- a/src/daemon/pulse/pulse.c
+++ b/src/daemon/pulse/pulse.c
@@ -118,6 +118,7 @@ void *pulse_thread_main(void *ptr) {
if(dbengine_enabled) {
worker_is_busy(WORKER_JOB_DBENGINE);
pulse_dbengine_do(pulse_extended_enabled);
+ dbengine_retention_statistics(pulse_extended_enabled);
}
#endif
diff --git a/src/daemon/pulse/pulse.h b/src/daemon/pulse/pulse.h
index da4b4b9172..b5245af7e5 100644
--- a/src/daemon/pulse/pulse.h
+++ b/src/daemon/pulse/pulse.h
@@ -17,6 +17,7 @@ extern bool pulse_extended_enabled;
#include "pulse-daemon-memory.h"
#include "pulse-sqlite3.h"
#include "pulse-db-dbengine.h"
+#include "pulse-db-dbengine-retention.h"
#include "pulse-db-rrd.h"
#include "pulse-string.h"
#include "pulse-heartbeat.h"
diff --git a/src/daemon/service.c b/src/daemon/service.c
index e10a0cfc9b..d5314c5760 100644
--- a/src/daemon/service.c
+++ b/src/daemon/service.c
@@ -312,10 +312,6 @@ void *service_main(void *ptr)
}
real_step = USEC_PER_SEC;
-#ifdef ENABLE_DBENGINE
- dbengine_retention_statistics();
-#endif
-
svc_rrd_cleanup_obsolete_charts_from_all_hosts();
if (service_running(SERVICE_MAINTENANCE))
diff --git a/src/database/contexts/api_v2_contexts_agents.c b/src/database/contexts/api_v2_contexts_agents.c
index 99805a133c..48ef753eb7 100644
--- a/src/database/contexts/api_v2_contexts_agents.c
+++ b/src/database/contexts/api_v2_contexts_agents.c
@@ -113,7 +113,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
#ifdef ENABLE_DBENGINE
if (!max && eng->seb == STORAGE_ENGINE_BACKEND_DBENGINE) {
- max = get_directory_free_bytes_space(multidb_ctx[tier]);
+ max = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
max += used;
}
#endif
diff --git a/src/database/engine/rrdengine.c b/src/database/engine/rrdengine.c
index 44ffb2a6bd..adeccda09f 100644
--- a/src/database/engine/rrdengine.c
+++ b/src/database/engine/rrdengine.c
@@ -1639,7 +1639,7 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo
return data;
}
-uint64_t get_used_disk_space(struct rrdengine_instance *ctx)
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx)
{
uint64_t active_space = 0;
@@ -1680,7 +1680,7 @@ bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)
// only 1 datafile available
return false;
- uint64_t estimated_disk_space = get_used_disk_space(ctx);
+ uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx);
time_t retention = get_tier_retention(ctx);
if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)
@@ -1819,7 +1819,7 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w
work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);
}
-uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
+uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx)
{
uint64_t free_bytes = 0;
struct statvfs buff_statvfs;
@@ -1829,7 +1829,7 @@ uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
return (free_bytes - (free_bytes * 5 / 100));
}
-void calculate_tier_disk_space_percentage(void)
+void rrdeng_calculate_tier_disk_space_percentage(void)
{
uint64_t tier_space[RRD_STORAGE_TIERS];
@@ -1845,7 +1845,7 @@ void calculate_tier_disk_space_percentage(void)
}
uint64_t tier_disk_space = multidb_ctx[tier]->config.max_disk_space ?
multidb_ctx[tier]->config.max_disk_space :
- get_directory_free_bytes_space(multidb_ctx[tier]);
+ rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
total_diskspace += tier_disk_space;
tier_space[tier] = tier_disk_space;
}
@@ -1857,84 +1857,6 @@ void calculate_tier_disk_space_percentage(void)
}
}
-void dbengine_retention_statistics(void)
-{
- static bool init = false;
- static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
-
- if (!localhost)
- return;
-
- calculate_tier_disk_space_percentage();
-
- for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
- STORAGE_ENGINE *eng = localhost->db[tier].eng;
- if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
- continue;
-
- if (init == false) {
- char id[200];
- snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
- stats[tier].st = rrdset_create_localhost(
- "netdata",
- id,
- NULL,
- "dbengine retention",
- "netdata.dbengine_tier_retention",
- "dbengine space and time retention",
- "%",
- "netdata",
- "stats",
- 134900, // before "dbengine memory" (dbengine2_statistics_charts)
- 10,
- RRDSET_TYPE_LINE);
-
- stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
- stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
-
- char tier_str[5];
- snprintfz(tier_str, 4, "%zu", tier);
- rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
-
- rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
- rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
- rrdset_metadata_updated(stats[tier].st);
- }
-
- time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
- time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
-
- //
- // Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
- // get_used_disk_space is used to determine if database cleanup (file rotation should happen)
- // and adds to the disk space used the desired file size of the active
- // datafile
- uint64_t disk_space = get_used_disk_space(multidb_ctx[tier]);
- //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
-
- uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
- if (!config_disk_space) {
- config_disk_space = get_directory_free_bytes_space(multidb_ctx[tier]);
- config_disk_space += disk_space;
- }
-
- collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
-
- collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
- 100 * retention / multidb_ctx[tier]->config.max_retention_s :
- 0;
-
- if (retention_percentage > 100)
- retention_percentage = 100;
-
- rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
- rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
-
- rrdset_done(stats[tier].st);
- }
- init = true;
-}
-
void dbengine_event_loop(void* arg) {
sanity_check();
uv_thread_set_name_np("DBENGINE");
diff --git a/src/database/engine/rrdengine.h b/src/database/engine/rrdengine.h
index 379dbb8518..b47f9a5131 100644
--- a/src/database/engine/rrdengine.h
+++ b/src/database/engine/rrdengine.h
@@ -563,9 +563,8 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
}
// --------------------------------------------------------------------------------------------------------------------
-uint64_t get_used_disk_space(struct rrdengine_instance *ctx);
-void calculate_tier_disk_space_percentage(void);
-void dbengine_retention_statistics(void);
-uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx);
+void rrdeng_calculate_tier_disk_space_percentage(void);
+uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
#endif /* NETDATA_RRDENGINE_H */