archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-16 10:31:07 +00:00
Code Issues Projects Releases Wiki Activity

Adjust cloud dimension update frequency ()

Browse source 
* Queue a chart immediately to the cloud

* Do not inform the cloud immediately if a dimension stopped collecting use MAX(obsoletion time, 1.5 * update_every)

* Notify cloud immediately on dimension deletion

* Add debug messages

* Do not schedule an update if we are shutting down
This commit is contained in:
Stelios Fragkakis 2022-03-08 20:06:30 +02:00 • committed by GitHub
parent 3b07293ede
commit 6872df9e6a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 62 additions and 10 deletions

1
database/rrd.h
View file

@ -53,7 +53,6 @@ struct context_param {
uint8_t flags;
};
#define RRDSET_MINIMUM_LIVE_COUNT 3
#define META_CHART_UPDATED 1
#define META_PLUGIN_UPDATED 2
#define META_MODULE_UPDATED 4

4
database/rrddim.c
View file

@ -504,6 +504,10 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
// free(rd->annotations);
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
if (!netdata_exit)
aclk_send_dimension_update(rd);
#endif
RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
switch(rrd_memory_mode) {

5
database/rrdhost.c
View file

@ -1516,6 +1516,11 @@ restart_after_removal:
}
continue;
}
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
else {
aclk_send_dimension_update(rd);
}
#endif
}
last = rd;
rd = rd->next;

12
database/rrdset.c
View file

@ -1400,10 +1400,8 @@ void rrdset_done(RRDSET *st) {
#ifdef ENABLE_ACLK
if (likely(!st->state->is_ar_chart)) {
if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
if (st->counter_done >= RRDSET_MINIMUM_LIVE_COUNT && st->dimensions) {
if (likely(!queue_chart_to_aclk(st)))
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
}
if (likely(st->dimensions && !queue_chart_to_aclk(st)))
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
}
}
#endif
@ -1832,7 +1830,8 @@ after_second_database_work:
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
if (likely(!st->state->is_ar_chart)) {
if (!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) {
int live = ((mark - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
int live = ((mark - rd->last_collected_time.tv_sec) <
MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
if (unlikely(live != rd->state->aclk_live_status)) {
if (likely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
if (likely(!queue_dimension_to_aclk(rd))) {
@ -1943,6 +1942,9 @@ after_second_database_work:
delete_dimension_uuid(&rd->state->metric_uuid);
} else {
/* Do not delete this dimension */
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
aclk_send_dimension_update(rd);
#endif
last = rd;
rd = rd->next;
continue;

45
database/sqlite/sqlite_aclk_chart.c
View file

@ -3,7 +3,7 @@
#include "sqlite_functions.h"
#include "sqlite_aclk_chart.h"
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
#include "../../aclk/aclk_charts_api.h"
#include "../../aclk/aclk.h"
@ -277,7 +277,7 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
time_t first_t = rd->state->query_ops.oldest_time(rd);
time_t last_t = rd->state->query_ops.latest_time(rd);
int live = ((now - last_t) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
int live = ((now - last_t) < MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
rc = aclk_upd_dimension_event(
wc,
@ -973,6 +973,47 @@ int queue_dimension_to_aclk(RRDDIM *rd)
return rc;
}
void aclk_send_dimension_update(RRDDIM *rd)
{
if (!aclk_use_new_cloud_arch)
return;
char *claim_id = is_agent_claimed();
if (unlikely(!claim_id))
return;
time_t first_entry_t = rrddim_first_entry_t(rd);
time_t last_entry_t = rrddim_last_entry_t(rd);
time_t now = now_realtime_sec();
int live = ((now - rd->last_collected_time.tv_sec) <
MAX(RRDSET_MINIMUM_LIVE_MULTIPLIER * rd->update_every, rrdset_free_obsolete_time));
if (!live || rd->state->aclk_live_status != live || !first_entry_t) {
(void)aclk_upd_dimension_event(
rd->rrdset->rrdhost->dbsync_worker,
claim_id,
&rd->state->metric_uuid,
rd->id,
rd->name,
rd->rrdset->id,
first_entry_t,
live ? 0 : last_entry_t);
if (!first_entry_t)
debug(D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld)",
rd->rrdset->rrdhost->hostname, rd->rrdset->name, rd->name, live, first_entry_t, last_entry_t);
else
debug(D_ACLK_SYNC, "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld) collected %ld seconds ago",
rd->rrdset->rrdhost->hostname, rd->rrdset->name, rd->name, live, first_entry_t,
last_entry_t, now - last_entry_t);
rd->state->aclk_live_status = live;
}
freez(claim_id);
return;
}
#endif //ENABLE_NEW_CLOUD_PROTOCOL
// ST is read locked

5
database/sqlite/sqlite_aclk_chart.h
View file

@ -12,8 +12,8 @@ typedef enum payload_type {
extern sqlite3 *db_meta;
#ifndef RRDSET_MINIMUM_LIVE_COUNT
#define RRDSET_MINIMUM_LIVE_COUNT 3
#ifndef RRDSET_MINIMUM_LIVE_MULTIPLIER
#define RRDSET_MINIMUM_LIVE_MULTIPLIER (1.5)
#endif
extern int queue_chart_to_aclk(RRDSET *st);
@ -34,4 +34,5 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
void aclk_process_dimension_deletion(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
uint32_t sql_get_pending_count(struct aclk_database_worker_config *wc);
void aclk_send_dimension_update(RRDDIM *rd);
#endif //NETDATA_SQLITE_ACLK_CHART_H