archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-28 06:32:30 +00:00
Code Issues Projects Releases Wiki Activity

move dbengine-retention chart to pulse ()

Browse source
This commit is contained in:
Costa Tsaousis 2025-01-19 19:37:43 +00:00 • committed by GitHub
parent 26fa11fa53
commit c7a58fe374
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 111 additions and 95 deletions

2
CMakeLists.txt
View file

@ -1170,6 +1170,8 @@ set(DAEMON_FILES
src/daemon/pulse/pulse-daemon-memory-system.c
src/daemon/pulse/pulse-network.c
src/daemon/pulse/pulse-network.h
src/daemon/pulse/pulse-db-dbengine-retention.c
src/daemon/pulse/pulse-db-dbengine-retention.h
)
set(H2O_FILES

2
src/daemon/config/netdata-conf-db.c
View file

@ -320,7 +320,7 @@ void netdata_conf_dbengine_init(const char *hostname) {
for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++)
rrdeng_readiness_wait(multidb_ctx[tier]);
calculate_tier_disk_space_percentage();
rrdeng_calculate_tier_disk_space_percentage();
dbengine_enabled = true;
#else

84
src/daemon/pulse/pulse-db-dbengine-retention.c Normal file
View file

@ -0,0 +1,84 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "pulse-db-dbengine-retention.h"
#ifdef ENABLE_DBENGINE
#include "database/engine/rrdengineapi.h"
void dbengine_retention_statistics(bool extended __maybe_unused) {
static bool init = false;
static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
if (!localhost)
return;
rrdeng_calculate_tier_disk_space_percentage();
for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
STORAGE_ENGINE *eng = localhost->db[tier].eng;
if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
continue;
if (init == false) {
char id[200];
snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
stats[tier].st = rrdset_create_localhost(
"netdata",
id,
NULL,
"dbengine retention",
"netdata.dbengine_tier_retention",
"dbengine space and time retention",
"%",
"netdata",
"stats",
134900, // before "dbengine memory" (dbengine2_statistics_charts)
10,
RRDSET_TYPE_LINE);
stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
char tier_str[5];
snprintfz(tier_str, 4, "%zu", tier);
rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
rrdset_metadata_updated(stats[tier].st);
}
time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
//
// Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
// get_used_disk_space is used to determine if database cleanup (file rotation should happen)
// and adds to the disk space used the desired file size of the active
// datafile
uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier]);
//uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
if (!config_disk_space) {
config_disk_space = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
config_disk_space += disk_space;
}
collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
100 * retention / multidb_ctx[tier]->config.max_retention_s :
0;
if (retention_percentage > 100)
retention_percentage = 100;
rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
rrdset_done(stats[tier].st);
}
init = true;
}
#endif

12
src/daemon/pulse/pulse-db-dbengine-retention.h Normal file
View file

@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_PULSE_DB_DBENGINE_RETENTION_H
#define NETDATA_PULSE_DB_DBENGINE_RETENTION_H
#include "libnetdata/libnetdata.h"
#ifdef ENABLE_DBENGINE
void dbengine_retention_statistics(bool extended __maybe_unused);
#endif
#endif //NETDATA_PULSE_DB_DBENGINE_RETENTION_H

1
src/daemon/pulse/pulse-db-dbengine.c
View file

@ -686,7 +686,6 @@ void pulse_dbengine_do(bool extended) {
mrg_get_statistics(main_mrg, &mrg_stats);
size_t priority = 135000;
{
static RRDSET *st_pgc_memory = NULL;
static RRDDIM *rd_pgc_memory_main = NULL;

2
src/daemon/pulse/pulse-network.c
View file

@ -124,7 +124,7 @@ static void pulse_aclk_time_heatmap(void) {
rds[0] = rrddim_add(st, "instant", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
for(size_t i = 1; i < _countof(rds) - 1 ;i++) {
char buf[128];
char buf[350];
snprintf(buf, sizeof(buf), "%.2fs", (double)aclk_time_heatmap.array[i].upto / (double)USEC_PER_SEC);
// duration_snprintf(buf, sizeof(buf), aclk_time_heatmap.array[i].upto, "us", false);
rds[i] = rrddim_add(st, buf, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

1
src/daemon/pulse/pulse.c
View file

@ -118,6 +118,7 @@ void *pulse_thread_main(void *ptr) {
if(dbengine_enabled) {
worker_is_busy(WORKER_JOB_DBENGINE);
pulse_dbengine_do(pulse_extended_enabled);
dbengine_retention_statistics(pulse_extended_enabled);
}
#endif

1
src/daemon/pulse/pulse.h
View file

@ -17,6 +17,7 @@ extern bool pulse_extended_enabled;
#include "pulse-daemon-memory.h"
#include "pulse-sqlite3.h"
#include "pulse-db-dbengine.h"
#include "pulse-db-dbengine-retention.h"
#include "pulse-db-rrd.h"
#include "pulse-string.h"
#include "pulse-heartbeat.h"

4
src/daemon/service.c
View file

@ -312,10 +312,6 @@ void *service_main(void *ptr)
}
real_step = USEC_PER_SEC;
#ifdef ENABLE_DBENGINE
dbengine_retention_statistics();
#endif
svc_rrd_cleanup_obsolete_charts_from_all_hosts();
if (service_running(SERVICE_MAINTENANCE))

2
src/database/contexts/api_v2_contexts_agents.c
View file

@ -113,7 +113,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
#ifdef ENABLE_DBENGINE
if (!max && eng->seb == STORAGE_ENGINE_BACKEND_DBENGINE) {
max = get_directory_free_bytes_space(multidb_ctx[tier]);
max = rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
max += used;
}
#endif

88
src/database/engine/rrdengine.c
View file

@ -1639,7 +1639,7 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo
return data;
}
uint64_t get_used_disk_space(struct rrdengine_instance *ctx)
uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx)
{
uint64_t active_space = 0;
@ -1680,7 +1680,7 @@ bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)
// only 1 datafile available
return false;
uint64_t estimated_disk_space = get_used_disk_space(ctx);
uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx);
time_t retention = get_tier_retention(ctx);
if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)
@ -1819,7 +1819,7 @@ static inline void worker_dispatch_query_prep(struct rrdeng_cmd cmd, bool from_w
work_dispatch(ctx, pdc, NULL, cmd.opcode, query_prep_tp_worker, NULL);
}
uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx)
{
uint64_t free_bytes = 0;
struct statvfs buff_statvfs;
@ -1829,7 +1829,7 @@ uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx)
return (free_bytes - (free_bytes * 5 / 100));
}
void calculate_tier_disk_space_percentage(void)
void rrdeng_calculate_tier_disk_space_percentage(void)
{
uint64_t tier_space[RRD_STORAGE_TIERS];
@ -1845,7 +1845,7 @@ void calculate_tier_disk_space_percentage(void)
}
uint64_t tier_disk_space = multidb_ctx[tier]->config.max_disk_space ?
multidb_ctx[tier]->config.max_disk_space :
get_directory_free_bytes_space(multidb_ctx[tier]);
rrdeng_get_directory_free_bytes_space(multidb_ctx[tier]);
total_diskspace += tier_disk_space;
tier_space[tier] = tier_disk_space;
}
@ -1857,84 +1857,6 @@ void calculate_tier_disk_space_percentage(void)
}
}
void dbengine_retention_statistics(void)
{
static bool init = false;
static DBENGINE_TIER_STATS stats[RRD_STORAGE_TIERS];
if (!localhost)
return;
calculate_tier_disk_space_percentage();
for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {
STORAGE_ENGINE *eng = localhost->db[tier].eng;
if (!eng || eng->seb != STORAGE_ENGINE_BACKEND_DBENGINE)
continue;
if (init == false) {
char id[200];
snprintfz(id, sizeof(id) - 1, "dbengine_retention_tier%zu", tier);
stats[tier].st = rrdset_create_localhost(
"netdata",
id,
NULL,
"dbengine retention",
"netdata.dbengine_tier_retention",
"dbengine space and time retention",
"%",
"netdata",
"stats",
134900, // before "dbengine memory" (dbengine2_statistics_charts)
10,
RRDSET_TYPE_LINE);
stats[tier].rd_space = rrddim_add(stats[tier].st, "space", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
stats[tier].rd_time = rrddim_add(stats[tier].st, "time", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
char tier_str[5];
snprintfz(tier_str, 4, "%zu", tier);
rrdlabels_add(stats[tier].st->rrdlabels, "tier", tier_str, RRDLABEL_SRC_AUTO);
rrdset_flag_set(stats[tier].st, RRDSET_FLAG_METADATA_UPDATE);
rrdhost_flag_set(stats[tier].st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
rrdset_metadata_updated(stats[tier].st);
}
time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
time_t retention = first_time_s ? now_realtime_sec() - first_time_s : 0;
//
// Note: storage_engine_disk_space_used is the exact diskspace (as reported by api/v2/node_instances
// get_used_disk_space is used to determine if database cleanup (file rotation should happen)
// and adds to the disk space used the desired file size of the active
// datafile
uint64_t disk_space = get_used_disk_space(multidb_ctx[tier]);
//uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);
if (!config_disk_space) {
config_disk_space = get_directory_free_bytes_space(multidb_ctx[tier]);
config_disk_space += disk_space;
}
collected_number disk_percentage = (collected_number) (config_disk_space ? 100 * disk_space / config_disk_space : 0);
collected_number retention_percentage = (collected_number)multidb_ctx[tier]->config.max_retention_s ?
100 * retention / multidb_ctx[tier]->config.max_retention_s :
0;
if (retention_percentage > 100)
retention_percentage = 100;
rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_space, (collected_number) disk_percentage);
rrddim_set_by_pointer(stats[tier].st, stats[tier].rd_time, (collected_number) retention_percentage);
rrdset_done(stats[tier].st);
}
init = true;
}
void dbengine_event_loop(void* arg) {
sanity_check();
uv_thread_set_name_np("DBENGINE");

7
src/database/engine/rrdengine.h
View file

@ -563,9 +563,8 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
}
// --------------------------------------------------------------------------------------------------------------------
uint64_t get_used_disk_space(struct rrdengine_instance *ctx);
void calculate_tier_disk_space_percentage(void);
void dbengine_retention_statistics(void);
uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);
uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx);
void rrdeng_calculate_tier_disk_space_percentage(void);
uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
#endif /* NETDATA_RRDENGINE_H */