archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-16 10:31:07 +00:00
Code Issues Projects Releases Wiki Activity

Enable additional functionality for the new cloud architecture ()

Browse source
This commit is contained in:
Stelios Fragkakis 2021-10-06 20:55:31 +03:00 • committed by GitHub
parent af93cc31ed
commit 12f16063f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 721 additions and 204 deletions

6
aclk/aclk.c
View file

@ -973,7 +973,7 @@ void ng_aclk_host_state_update(RRDHOST *host, int cmd)
rrdhost_aclk_state_lock(localhost);
create_query->data.node_creation.claim_id = strdupz(localhost->aclk_state.claimed_id);
rrdhost_aclk_state_unlock(localhost);
create_query->data.node_creation.hops = 1; //TODO - real hop count instead of hardcoded
create_query->data.node_creation.hops = (uint32_t) host->system_info->hops;
create_query->data.node_creation.hostname = strdupz(host->hostname);
create_query->data.node_creation.machine_guid = strdupz(host->machine_guid);
aclk_queue_query(create_query);
@ -981,7 +981,7 @@ void ng_aclk_host_state_update(RRDHOST *host, int cmd)
}
aclk_query_t query = aclk_query_new(NODE_STATE_UPDATE);
query->data.node_update.hops = 1; //TODO - real hop count instead of hardcoded
query->data.node_update.hops = (uint32_t) host->system_info->hops;
rrdhost_aclk_state_lock(localhost);
query->data.node_update.claim_id = strdupz(localhost->aclk_state.claimed_id);
rrdhost_aclk_state_unlock(localhost);
@ -1020,7 +1020,7 @@ void aclk_send_node_instances()
rrdhost_aclk_state_lock(localhost);
create_query->data.node_creation.claim_id = strdupz(localhost->aclk_state.claimed_id);
rrdhost_aclk_state_unlock(localhost);
create_query->data.node_creation.hops = uuid_compare(list->host_id, localhost->host_uuid) ? 1 : 0; // TODO - when streaming supports hops
create_query->data.node_creation.hops = list->hops;
create_query->data.node_creation.hostname = list->hostname;
create_query->data.node_creation.machine_guid = mallocz(UUID_STR_LEN);
uuid_unparse_lower(list->host_id, (char*)create_query->data.node_creation.machine_guid);

4
aclk/aclk_charts_api.c
View file

@ -16,6 +16,7 @@ void aclk_chart_inst_update(char **payloads, size_t *payload_sizes, struct aclk_
void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
{
aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
query->data.bin_payload.payload = generate_chart_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, new_positions);
query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
QUEUE_IF_PAYLOAD_PRESENT(query);
@ -24,6 +25,7 @@ void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_m
void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id)
{
aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
query->data.bin_payload.payload = generate_charts_and_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, is_dim, new_positions, batch_id);
query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
QUEUE_IF_PAYLOAD_PRESENT(query);
@ -32,6 +34,7 @@ void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int
void aclk_chart_config_updated(struct chart_config_updated *config_list, int list_size)
{
aclk_query_t query = aclk_query_new(CHART_CONFIG_UPDATED);
query->data.bin_payload.topic = ACLK_TOPICID_CHART_CONFIGS_UPDATED;
query->data.bin_payload.payload = generate_chart_configs_updated(&query->data.bin_payload.size, config_list, list_size);
query->data.bin_payload.msg_name = "ChartConfigsUpdated";
QUEUE_IF_PAYLOAD_PRESENT(query);
@ -40,6 +43,7 @@ void aclk_chart_config_updated(struct chart_config_updated *config_list, int lis
void aclk_chart_reset(chart_reset_t reset)
{
aclk_query_t query = aclk_query_new(CHART_RESET);
query->data.bin_payload.topic = ACLK_TOPICID_CHART_RESET;
query->data.bin_payload.payload = generate_reset_chart_messages(&query->data.bin_payload.size, reset);
query->data.bin_payload.msg_name = "ResetChartMessages";
QUEUE_IF_PAYLOAD_PRESENT(query);

67
aclk/aclk_rx_msgs.c
View file

@ -330,6 +330,73 @@ void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t
return;
}
if (!strcmp(message_type, "StreamChartsAndDimensions")) {
stream_charts_and_dims_t res = parse_stream_charts_and_dims(msg, msg_len);
if (!res.claim_id || !res.node_id) {
error("Error parsing StreamChartsAndDimensions msg");
freez(res.claim_id);
freez(res.node_id);
return;
}
chart_batch_id = res.batch_id;
aclk_start_streaming(res.node_id, res.seq_id, res.seq_id_created_at.tv_sec, res.batch_id);
freez(res.claim_id);
freez(res.node_id);
return;
}
if (!strcmp(message_type, "ChartsAndDimensionsAck")) {
chart_and_dim_ack_t res = parse_chart_and_dimensions_ack(msg, msg_len);
if (!res.claim_id || !res.node_id) {
error("Error parsing StreamChartsAndDimensions msg");
freez(res.claim_id);
freez(res.node_id);
return;
}
aclk_ack_chart_sequence_id(res.node_id, res.last_seq_id);
freez(res.claim_id);
freez(res.node_id);
return;
}
if (!strcmp(message_type, "UpdateChartConfigs")) {
struct update_chart_config res = parse_update_chart_config(msg, msg_len);
if (!res.claim_id || !res.node_id || !res.hashes)
error("Error parsing UpdateChartConfigs msg");
else
aclk_get_chart_config(res.hashes);
destroy_update_chart_config(&res);
return;
}
if (!strcmp(message_type, "StartAlarmStreaming")) {
struct start_alarm_streaming res = parse_start_alarm_streaming(msg, msg_len);
if (!res.node_id || !res.batch_id) {
error("Error parsing StartAlarmStreaming");
freez(res.node_id);
return;
}
aclk_start_alert_streaming(res.node_id, res.batch_id, res.start_seq_id);
freez(res.node_id);
return;
}
if (!strcmp(message_type, "SendAlarmLogHealth")) {
char *node_id = parse_send_alarm_log_health(msg, msg_len);
if (!node_id) {
error("Error parsing SendAlarmLogHealth");
return;
}
aclk_send_alarm_health_log(node_id);
freez(node_id);
return;
}
if (!strcmp(message_type, "SendAlarmConfiguration")) {
char *config_hash = parse_send_alarm_configuration(msg, msg_len);
if (!config_hash || !*config_hash) {
error("Error parsing SendAlarmConfiguration");
return;
}
aclk_send_alarm_configuration(config_hash);
freez(config_hash);
return;
}
error ("Unknown new cloud arch message type received \"%s\"", message_type);
}
#endif

4
aclk/schema-wrappers/alarm_stream.cc
View file

@ -97,8 +97,8 @@ static alarmstream::v1::AlarmStatus aclk_alarm_status_to_proto(enum aclk_alarm_s
void destroy_alarm_log_entry(struct alarm_log_entry *entry)
{
freez(entry->node_id);
freez(entry->claim_id);
//freez(entry->node_id);
//freez(entry->claim_id);
freez(entry->chart);
freez(entry->name);

2
aclk/schema-wrappers/chart_stream.cc
View file

@ -75,8 +75,6 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance)
{
freez((char*)instance->id);
freez((char*)instance->claim_id);
freez((char*)instance->node_id);
freez((char*)instance->name);
free_label_list(instance->label_head);

1
database/engine/rrdengine.c
View file

@ -861,6 +861,7 @@ static void after_delete_old_data(struct rrdengine_worker_config* wc)
wc->now_deleting_files = NULL;
wc->cleanup_thread_deleting_files = 0;
aclk_data_rotated();
/* interrupt event loop */
uv_stop(wc->loop);

5
database/rrd.h
View file

@ -50,6 +50,7 @@ struct context_param {
uint8_t flags;
};
#define RRDSET_MINIMUM_LIVE_COUNT 3
#define META_CHART_UPDATED 1
#define META_PLUGIN_UPDATED 2
#define META_MODULE_UPDATED 4
@ -1359,4 +1360,8 @@ extern void set_host_properties(
#endif
#include "sqlite/sqlite_functions.h"
#include "sqlite/sqlite_aclk.h"
#include "sqlite/sqlite_aclk_chart.h"
#include "sqlite/sqlite_aclk_alert.h"
#include "sqlite/sqlite_aclk_node.h"
#include "sqlite/sqlite_health.h"
#endif /* NETDATA_RRD_H */

17
database/rrddim.c
View file

@ -210,7 +210,7 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
}
}
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
}
@ -387,6 +387,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rd->last_collected_time.tv_usec = 0;
rd->rrdset = st;
rd->state = mallocz(sizeof(*rd->state));
#ifdef ENABLE_ACLK
rd->state->aclk_live_status = -1;
#endif
(void) find_dimension_uuid(st, rd, &(rd->state->metric_uuid));
if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
@ -453,7 +456,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrdset_unlock(st);
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
return(rd);
}
@ -521,7 +524,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
}
#ifdef ENABLE_ACLK
if (db_rotated || RRD_MEMORY_MODE_DBENGINE != rrd_memory_mode)
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
}
@ -542,7 +545,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
return 0;
}
@ -559,7 +562,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
return 0;
}
@ -574,7 +577,7 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE);
rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
}
@ -583,7 +586,7 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
#ifdef ENABLE_ACLK
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
}

2
database/rrdhost.c
View file

@ -746,7 +746,9 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
fatal("Failed to initialize dbengine");
}
#endif
#ifdef ACLK_NEWARCH_DEVMODE
sql_aclk_sync_init();
#endif
rrd_unlock();
web_client_api_v1_management_init();

32
database/rrdset.c
View file

@ -649,7 +649,7 @@ RRDSET *rrdset_create_custom(
aclk_add_collector(host, st->plugin_name, st->module_name);
}
}
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
}
#endif
freez(old_plugin);
@ -935,12 +935,13 @@ RRDSET *rrdset_create_custom(
update_chart_metadata(st->chart_uuid, st, id, name);
store_active_chart(st->chart_uuid);
compute_chart_hash(st);
rrdhost_unlock(host);
#ifdef ENABLE_ACLK
if (netdata_cloud_setting)
aclk_add_collector(host, plugin, module);
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#endif
return(st);
}
@ -1377,10 +1378,19 @@ void rrdset_done(RRDSET *st) {
rrdset_rdlock(st);
#ifdef ENABLE_ACLK
if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
if (st->counter_done >= RRDSET_MINIMUM_LIVE_COUNT) {
if (likely(!sql_queue_chart_to_aclk(st)))
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
}
}
#else
if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
rrdset_flag_set(st, RRDSET_FLAG_ACLK);
aclk_update_chart(st->rrdhost, st->id, 1);
}
#endif
#endif
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
@ -1783,9 +1793,23 @@ after_first_database_work:
after_second_database_work:
st->last_collected_total = st->collected_total;
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
time_t mark = now_realtime_sec();
#endif
rrddim_foreach_read(rd, st) {
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue;
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
int live = ((mark - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
if (unlikely(live != rd->state->aclk_live_status)) {
if (likely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
if (likely(!sql_queue_dimension_to_aclk(rd))) {
rd->state->aclk_live_status = live;
}
}
}
#endif
if(unlikely(!rd->updated))
continue;

144
database/sqlite/sqlite_aclk.c
View file

@ -3,17 +3,17 @@
#include "sqlite_functions.h"
#include "sqlite_aclk.h"
// TODO: To be added
#include "sqlite_aclk_chart.h"
//#include "sqlite_aclk_alert.h"
#include "sqlite_aclk_node.h"
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
#include "../../aclk/aclk.h"
#endif
const char *aclk_sync_config[] = {
NULL,
};
int aclk_architecture = 0;
uv_mutex_t aclk_async_lock;
struct aclk_database_worker_config *aclk_thread_head = NULL;
@ -78,24 +78,6 @@ void aclk_database_init_cmd_queue(struct aclk_database_worker_config *wc)
fatal_assert(0 == uv_mutex_init(&wc->cmd_mutex));
}
void aclk_database_enq_cmd_nowake(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
{
unsigned queue_size;
/* wait for free space in queue */
uv_mutex_lock(&wc->cmd_mutex);
while ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {
uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
}
fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
/* enqueue command */
wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
wc->cmd_queue.tail = wc->cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
wc->cmd_queue.tail + 1 : 0;
wc->queue_size = queue_size + 1;
uv_mutex_unlock(&wc->cmd_mutex);
}
int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
{
unsigned queue_size;
@ -207,7 +189,7 @@ int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)
void sql_aclk_sync_init(void)
{
#ifdef ACLK_NEWARCH_DEVMODE
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
char *err_msg = NULL;
int rc;
@ -251,35 +233,49 @@ static void async_cb(uv_async_t *handle)
static void timer_cb(uv_timer_t* handle)
{
struct aclk_database_worker_config *wc = handle->data;
uv_stop(handle->loop);
uv_update_time(handle->loop);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
struct aclk_database_worker_config *wc = handle->data;
struct aclk_database_cmd cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.opcode = ACLK_DATABASE_TIMER;
aclk_database_enq_cmd_noblock(wc, &cmd);
if (wc->cleanup_after && wc->cleanup_after < now_realtime_sec()) {
time_t now = now_realtime_sec();
if (wc->cleanup_after && wc->cleanup_after < now) {
cmd.opcode = ACLK_DATABASE_CLEANUP;
if (!aclk_database_enq_cmd_noblock(wc, &cmd))
wc->cleanup_after += ACLK_DATABASE_CLEANUP_INTERVAL;
}
if (wc->chart_updates && !wc->chart_pending) {
cmd.opcode = ACLK_DATABASE_PUSH_CHART;
cmd.count = ACLK_MAX_CHART_BATCH;
cmd.completion = NULL;
cmd.param1 = ACLK_MAX_CHART_BATCH_COUNT;
if (!aclk_database_enq_cmd_noblock(wc, &cmd))
wc->chart_pending = 1;
}
if (aclk_use_new_cloud_arch && aclk_connected) {
if (wc->rotation_after && wc->rotation_after < now) {
cmd.opcode = ACLK_DATABASE_NODE_INFO;
aclk_database_enq_cmd_noblock(wc, &cmd);
if (wc->alert_updates) {
cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
cmd.count = ACLK_MAX_ALERT_UPDATES;
aclk_database_enq_cmd_noblock(wc, &cmd);
cmd.opcode = ACLK_DATABASE_UPD_RETENTION;
if (!aclk_database_enq_cmd_noblock(wc, &cmd))
wc->rotation_after += ACLK_DATABASE_ROTATION_INTERVAL;
}
if (wc->chart_updates && !wc->chart_pending) {
cmd.opcode = ACLK_DATABASE_PUSH_CHART;
cmd.count = ACLK_MAX_CHART_BATCH;
cmd.param1 = ACLK_MAX_CHART_BATCH_COUNT;
if (!aclk_database_enq_cmd_noblock(wc, &cmd))
wc->chart_pending = 1;
}
if (wc->alert_updates) {
cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
cmd.count = ACLK_MAX_ALERT_UPDATES;
aclk_database_enq_cmd_noblock(wc, &cmd);
}
}
#endif
}
#define MAX_CMD_BATCH_SIZE (256)
@ -334,8 +330,14 @@ void aclk_database_worker(void *arg)
wc->node_info_send = (wc->host && !localhost);
aclk_add_worker_thread(wc);
info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, sizeof(*wc));
// TODO: To be added
// sql_get_last_chart_sequence(wc, cmd);
memset(&cmd, 0, sizeof(cmd));
sql_get_last_chart_sequence(wc, cmd);
wc->chart_updates = 0;
wc->alert_updates = 0;
wc->startup_time = now_realtime_sec();
wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
wc->rotation_after = wc->startup_time + ACLK_DATABASE_ROTATION_DELAY;
while (likely(shutdown == 0)) {
uv_run(loop, UV_RUN_DEFAULT);
@ -345,11 +347,8 @@ void aclk_database_worker(void *arg)
/* wait for commands */
cmd_batch_size = 0;
do {
if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE)) {
info("DEBUG: %s Processed %u commands, current queue about %u",
wc->uuid_str, cmd_batch_size, wc->queue_size);
if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))
break;
}
cmd = aclk_database_deq_cmd(wc);
opcode = cmd.opcode;
++cmd_batch_size;
@ -365,14 +364,6 @@ void aclk_database_worker(void *arg)
if (wc->host == localhost)
sql_check_aclk_table_list(wc);
break;
case ACLK_DATABASE_CHECK:
debug(D_ACLK_SYNC, "Checking database dimensions for %s", wc->host_guid);
// sql_check_dimension_state(wc, cmd);
break;
case ACLK_DATABASE_CHECK_ROTATION:
debug(D_ACLK_SYNC, "Checking database for rotation %s", wc->host_guid);
// sql_check_rotation_state(wc, cmd);
break;
case ACLK_DATABASE_DELETE_HOST:
debug(D_ACLK_SYNC,"Cleaning ACLK tables for %s", (char *) cmd.data);
sql_delete_aclk_table_list(wc, cmd);
@ -407,19 +398,19 @@ void aclk_database_worker(void *arg)
// ALERTS
case ACLK_DATABASE_ADD_ALERT:
debug(D_ACLK_SYNC,"Adding alert event for %s", wc->host_guid);
// aclk_add_alert_event(wc, cmd);
aclk_add_alert_event(wc, cmd);
break;
case ACLK_DATABASE_PUSH_ALERT_CONFIG:
debug(D_ACLK_SYNC,"Pushing chart config info to the cloud for %s", wc->host_guid);
// aclk_push_alert_config_event(wc, cmd);
aclk_push_alert_config_event(wc, cmd);
break;
case ACLK_DATABASE_PUSH_ALERT:
debug(D_ACLK_SYNC, "Pushing alert info to the cloud for %s", wc->host_guid);
// aclk_push_alert_event(wc, cmd);
aclk_push_alert_event(wc, cmd);
break;
case ACLK_DATABASE_ALARM_HEALTH_LOG:
debug(D_ACLK_SYNC, "Pushing alarm health log to the cloud for %s", wc->host_guid);
// aclk_push_alarm_health_log(wc, cmd);
aclk_push_alarm_health_log(wc, cmd);
break;
// NODE OPERATIONS
@ -427,8 +418,9 @@ void aclk_database_worker(void *arg)
debug(D_ACLK_SYNC,"Sending node info for %s", wc->uuid_str);
sql_build_node_info(wc, cmd);
break;
case ACLK_DATABASE_UPD_STATS:
// sql_update_metric_statistics(wc, cmd);
case ACLK_DATABASE_UPD_RETENTION:
debug(D_ACLK_SYNC,"Sending retention info for %s", wc->uuid_str);
aclk_update_retention(wc, cmd);
break;
// NODE_INSTANCE DETECTION
@ -446,7 +438,7 @@ void aclk_database_worker(void *arg)
}
}
}
if (wc->node_info_send && wc->host && localhost && claimed()) {
if (wc->node_info_send && wc->host && localhost && claimed() && aclk_connected) {
cmd.opcode = ACLK_DATABASE_NODE_INFO;
cmd.completion = NULL;
wc->node_info_send = aclk_database_enq_cmd_noblock(wc, &cmd);
@ -471,7 +463,7 @@ void aclk_database_worker(void *arg)
/*
* uv_async_send after uv_close does not seem to crash in linux at the moment,
* it is however undocumented behaviour and we need to be aware if this becomes
* it is however undocumented behaviour we need to be aware if this becomes
* an issue in the future.
*/
uv_close((uv_handle_t *)&wc->async, NULL);
@ -509,13 +501,9 @@ error_after_loop_init:
// -------------------------------------------------------------
void aclk_set_architecture(int mode)
{
aclk_architecture = mode;
}
void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
{
#ifdef ENABLE_ACLK
char uuid_str[GUID_LEN + 1];
char host_guid[GUID_LEN + 1];
@ -568,15 +556,17 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
if (likely(host))
host->dbsync_worker = (void *) wc;
wc->host = host;
wc->chart_updates = 0;
wc->alert_updates = 0;
wc->startup_time = now_realtime_sec();
wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
strcpy(wc->uuid_str, uuid_str);
strcpy(wc->host_guid, host_guid);
if (node_id && !uuid_is_null(*node_id))
uuid_unparse_lower(*node_id, wc->node_id);
fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));
#else
UNUSED(host);
UNUSED(host_uuid);
UNUSED(node_id);
#endif
return;
}
void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@ -722,19 +712,20 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
return;
}
void aclk_data_rotated(RRDHOST *host)
void aclk_data_rotated(void)
{
UNUSED(host);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
debug(D_ACLK_SYNC,"Processing data base rotation event");
struct aclk_database_cmd cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.opcode = ACLK_DATABASE_UPD_STATS;
if (!aclk_use_new_cloud_arch || !aclk_connected)
return;
time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY;
rrd_wrlock();
RRDHOST *this_host = localhost;
while (this_host) {
aclk_database_enq_cmd((struct aclk_database_worker_config *)this_host->dbsync_worker, &cmd);
struct aclk_database_worker_config *wc = this_host->dbsync_worker;
if (wc)
wc->rotation_after = next_rotation_time;
this_host = this_host->next;
}
rrd_unlock();
@ -743,9 +734,10 @@ void aclk_data_rotated(RRDHOST *host)
uv_mutex_lock(&aclk_async_lock);
while (tmp) {
aclk_database_enq_cmd(tmp, &cmd);
tmp->rotation_after = next_rotation_time;
tmp = tmp->next;
}
uv_mutex_unlock(&aclk_async_lock);
#endif
return;
}

16
database/sqlite/sqlite_aclk.h
View file

@ -15,9 +15,10 @@
#define ACLK_MAX_CHART_BATCH_COUNT (10)
#endif
#define ACLK_MAX_ALERT_UPDATES (5)
#define ACLK_SYNC_RETRY_COUNT "10"
#define ACLK_DATABASE_CLEANUP_FIRST (60)
#define ACLK_DATABASE_ROTATION_DELAY (60)
#define ACLK_DATABASE_CLEANUP_INTERVAL (3600)
#define ACLK_DATABASE_ROTATION_INTERVAL (3600)
#define ACLK_DELETE_ACK_INTERNAL (600)
#define ACLK_SYNC_QUERY_SIZE 512
@ -60,8 +61,6 @@ static inline void aclk_complete(struct aclk_completion *p)
extern uv_mutex_t aclk_async_lock;
extern int aclk_architecture;
static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
{
uuid_unparse_lower(*uuid, out);
@ -120,8 +119,6 @@ enum aclk_database_opcode {
ACLK_DATABASE_ADD_DIMENSION,
ACLK_DATABASE_ALARM_HEALTH_LOG,
ACLK_DATABASE_CHART_ACK,
ACLK_DATABASE_CHECK,
ACLK_DATABASE_CHECK_ROTATION,
ACLK_DATABASE_CLEANUP,
ACLK_DATABASE_DELETE_HOST,
ACLK_DATABASE_NODE_INFO,
@ -130,11 +127,9 @@ enum aclk_database_opcode {
ACLK_DATABASE_PUSH_CHART,
ACLK_DATABASE_PUSH_CHART_CONFIG,
ACLK_DATABASE_RESET_CHART,
ACLK_DATABASE_RESET_NODE,
ACLK_DATABASE_SHUTDOWN,
ACLK_DATABASE_TIMER,
ACLK_DATABASE_UPD_STATS,
ACLK_DATABASE_MAX_OPCODE
ACLK_DATABASE_UPD_RETENTION
};
struct aclk_chart_payload_t {
@ -170,6 +165,7 @@ struct aclk_database_worker_config {
time_t chart_timestamp; // last chart timestamp
time_t cleanup_after; // Start a cleanup after this timestamp
time_t startup_time; // When the sync thread started
time_t rotation_after;
uint64_t batch_id; // batch id to use
uint64_t alerts_batch_id; // batch id for alerts to use
uint64_t alerts_start_seq_id; // cloud has asked to start streaming from
@ -215,13 +211,11 @@ extern sqlite3 *db_meta;
extern int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
extern void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
extern void aclk_set_architecture(int mode);
extern void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd);
void aclk_data_rotated(RRDHOST *host);
void aclk_data_rotated(void);
void sql_aclk_sync_init(void);
void sql_check_aclk_table_list(struct aclk_database_worker_config *wc);
void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
void sql_drop_host_aclk_table_list(uuid_t *host_uuid);
void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
#endif //NETDATA_SQLITE_ACLK_H

112
database/sqlite/sqlite_aclk_alert.c
View file

@ -3,7 +3,10 @@
#include "sqlite_functions.h"
#include "sqlite_aclk_alert.h"
#ifdef ENABLE_ACLK
#include "../../aclk/aclk_alarm_api.h"
#include "../../aclk/aclk.h"
#endif
// will replace call to aclk_update_alarm in health/health_log.c
// and handle both cases
@ -11,8 +14,14 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
{
//check aclk architecture and handle old json alarm update to cloud
//include also the valid statuses for this case
/* if (!aclk_architecture)
aclk_update_alarm(host, ae); */
#ifdef ENABLE_ACLK
if (!aclk_use_new_cloud_arch) {
if ((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) ||
((ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL))) {
aclk_update_alarm(host, ae);
}
return;
}
if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)
return;
@ -33,6 +42,10 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
cmd.completion = NULL;
aclk_database_enq_cmd((struct aclk_database_worker_config *) host->dbsync_worker, &cmd);
ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
#else
UNUSED(host);
UNUSED(ae);
#endif
return;
}
@ -79,6 +92,7 @@ bind_fail:
int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
{
#ifdef ENABLE_ACLK
switch(status) {
case RRDCALC_STATUS_REMOVED:
return ALARM_STATUS_REMOVED;
@ -98,6 +112,10 @@ int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
default:
return ALARM_STATUS_UNKNOWN;
}
#else
UNUSED(status);
return 1;
#endif
}
void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@ -122,8 +140,8 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
if (wc->alerts_start_seq_id != 0) {
buffer_sprintf(
sql,
"UPDATE aclk_alert_%s SET date_submitted = NULL, date_cloud_ack = NULL WHERE sequence_id >= %" PRIu64
"; UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id < %" PRIu64
"UPDATE aclk_alert_%s SET date_submitted = NULL, date_cloud_ack = NULL WHERE sequence_id >= %"PRIu64
"; UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id < %"PRIu64
" and date_cloud_ack is null",
wc->uuid_str,
wc->alerts_start_seq_id,
@ -163,7 +181,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
char old_value_string[100 + 1];
char new_value_string[100 + 1];
alarm_log.node_id = strdupz(wc->node_id);
alarm_log.node_id = wc->node_id;
alarm_log.claim_id = claim_id;
alarm_log.chart = strdupz((char *)sqlite3_column_text(res, 12));
@ -179,10 +197,13 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
alarm_log.utc_offset = wc->host->utc_offset;
alarm_log.timezone = strdupz((char *)wc->host->abbrev_timezone);
alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) : strdupz((char *)wc->host->health_default_exec);
alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
strdupz((char *)wc->host->health_default_exec);
alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16));
char *edit_command = sqlite3_column_bytes(res, 16) > 0 ? health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) : strdupz("UNKNOWN=0");
char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) :
strdupz("UNKNOWN=0");
alarm_log.command = strdupz(edit_command);
alarm_log.duration = (time_t) sqlite3_column_int64(res, 6);
@ -193,10 +214,23 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
alarm_log.delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 10);
alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 25);
alarm_log.silenced = ( (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) || ( sqlite3_column_type(res, 15) != SQLITE_NULL && !strncmp((char *)sqlite3_column_text(res,15), "silent", 6)) ) ? 1 : 0;
alarm_log.silenced = ((sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) ||
(sqlite3_column_type(res, 15) != SQLITE_NULL &&
!strncmp((char *)sqlite3_column_text(res, 15), "silent", 6))) ?
1 :
0;
alarm_log.value_string = sqlite3_column_type(res, 23) == SQLITE_NULL ? strdupz((char *)"-") : strdupz((char *)format_value_and_unit(new_value_string, 100, sqlite3_column_double(res, 23), (char *) sqlite3_column_text(res, 17), -1));
alarm_log.old_value_string = sqlite3_column_type(res, 24) == SQLITE_NULL ? strdupz((char *)"-") : strdupz((char *)format_value_and_unit(old_value_string, 100, sqlite3_column_double(res, 24), (char *) sqlite3_column_text(res, 17), -1));
alarm_log.value_string =
sqlite3_column_type(res, 23) == SQLITE_NULL ?
strdupz((char *)"-") :
strdupz((char *)format_value_and_unit(
new_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 17), -1));
alarm_log.old_value_string =
sqlite3_column_type(res, 24) == SQLITE_NULL ?
strdupz((char *)"-") :
strdupz((char *)format_value_and_unit(
old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
alarm_log.value = (calculated_number) sqlite3_column_double(res, 23);
alarm_log.old_value = (calculated_number) sqlite3_column_double(res, 24);
@ -204,7 +238,6 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log.rendered_info = strdupz((char *)sqlite3_column_text(res, 18));
info("DEBUG: %s pushing alert seq %" PRIu64 " - %" PRIu64"", wc->uuid_str, (uint64_t) sqlite3_column_int64(res, 0), (uint64_t) sqlite3_column_int64(res, 1));
aclk_send_alarm_log_entry(&alarm_log);
if (first_sequence_id == 0)
@ -214,12 +247,14 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
destroy_alarm_log_entry(&alarm_log);
freez(edit_command);
}
buffer_flush(sql);
buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=strftime('%%s') "
"WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
wc->uuid_str, first_sequence_id, last_sequence_id);
db_execute(buffer_tostring(sql));
if (first_sequence_id) {
buffer_flush(sql);
buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=strftime('%%s') "
"WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
wc->uuid_str, first_sequence_id, last_sequence_id);
db_execute(buffer_tostring(sql));
}
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK))
@ -278,12 +313,8 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
sqlite3_stmt *res = NULL;
//TODO: make this better: include info from health log too
buffer_sprintf(sql, "select aa.sequence_id, aa.date_created, \
(select laa.sequence_id from aclk_alert_%s laa \
order by laa.sequence_id desc limit 1), \
(select laa.date_created from aclk_alert_%s laa \
order by laa.sequence_id desc limit 1) \
from aclk_alert_%s aa order by aa.sequence_id asc limit 1;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
buffer_sprintf(sql, "SELECT MIN(sequence_id), MIN(date_created), MAX(sequence_id), MAX(date_created) " \
"FROM aclk_alert_%s;", wc->uuid_str);
rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
if (rc != SQLITE_OK) {
@ -318,7 +349,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
struct alarm_log_health alarm_log;
alarm_log.claim_id = claim_id;
alarm_log.node_id = strdupz(wc->node_id);
alarm_log.node_id = wc->node_id;
alarm_log.log_entries = log_entries;
alarm_log.status = wc->alert_updates == 0 ? 2 : 1;
@ -330,7 +361,6 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement to get health log statistics from the database, rc = %d", rc);
freez((char *)alarm_log.node_id);
freez(claim_id);
buffer_free(sql);
#endif
@ -359,6 +389,11 @@ void aclk_send_alarm_configuration(char *config_hash)
return;
}
#define SQL_SELECT_ALERT_CONFIG "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin," \
"module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info," \
"options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after," \
"p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;"
int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
{
UNUSED(wc);
@ -372,20 +407,16 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
sqlite3_stmt *res = NULL;
char *config_hash = (char *) cmd.data_param;
BUFFER *sql = buffer_create(1024);
buffer_sprintf(
sql,
"SELECT alarm, template, on_key, class, type, component, os, hosts, plugin, module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info, options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;");
rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_ALERT_CONFIG, -1, &res, 0);
if (rc != SQLITE_OK) {
error_report("Failed to prepare statement when trying to fetch a chart hash configuration");
goto fail;
error_report("Failed to prepare statement when trying to fetch an alarm hash configuration");
return 1;
}
uuid_t hash_uuid;
if (uuid_parse(config_hash, hash_uuid))
goto fail;
return 1;
rc = sqlite3_bind_blob(res, 1, &hash_uuid , sizeof(hash_uuid), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
@ -463,16 +494,13 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
destroy_aclk_alarm_configuration(&alarm_config);
}
else
info("DEBUG: Alert config for %s not found", config_hash);
info("Alert config for %s not found", config_hash);
bind_fail:
bind_fail:
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement when pushing alarm config hash, rc = %d", rc);
fail:
buffer_free(sql);
return rc;
#endif
return 0;
@ -497,6 +525,11 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
wc = (struct aclk_database_worker_config *)host->dbsync_worker;
rrd_unlock();
if (unlikely(!host->health_enabled)) {
info("Ignoring request to stream alert state changes, health is disabled for %s", host->machine_guid);
return;
}
if (likely(wc)) {
info("START streaming alerts for %s enabled with batch_id %"PRIu64" and start_seq_id %"PRIu64, node_id, batch_id, start_seq_id);
__sync_synchronize();
@ -528,7 +561,10 @@ int sql_queue_removed_alerts_to_aclk(RRDHOST *host)
BUFFER *sql = buffer_create(1024);
buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created) " \
"select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s where new_status = -2 and updated_by_id = 0 and unique_id not in (select alert_unique_id from aclk_alert_%s) order by unique_id asc on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
"select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s " \
"where new_status = -2 and updated_by_id = 0 and unique_id not in " \
"(select alert_unique_id from aclk_alert_%s) order by unique_id asc " \
"on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
db_execute(buffer_tostring(sql));

119
database/sqlite/sqlite_aclk_chart.c
View file

@ -3,8 +3,12 @@
#include "sqlite_functions.h"
#include "sqlite_aclk_chart.h"
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
#include "../../aclk/aclk_charts_api.h"
#include "../../aclk/aclk.h"
#endif
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
static inline int sql_queue_chart_payload(struct aclk_database_worker_config *wc,
void *data, enum aclk_database_opcode opcode)
{
@ -117,6 +121,7 @@ bind_fail:
error_report("Failed to reset statement in store chart payload, rc = %d", rc);
return (rc != SQLITE_DONE);
}
#endif
int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
{
@ -134,8 +139,8 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat
chart_payload.config_hash = get_str_from_uuid(&st->state->hash_id);
chart_payload.update_every = st->update_every;
chart_payload.memory_mode = st->rrd_memory_mode;
chart_payload.name = strdupz((char *)st->name);
chart_payload.node_id = strdupz(wc->node_id);
chart_payload.name = (char *)st->name;
chart_payload.node_id = wc->node_id;
chart_payload.claim_id = claim_id;
chart_payload.id = strdupz(st->id);
@ -186,12 +191,12 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
size_t size;
memset(&dim_payload, 0, sizeof(dim_payload));
dim_payload.node_id = strdupz(wc->node_id);
dim_payload.node_id = wc->node_id;
dim_payload.claim_id = claim_id;
dim_payload.name = strdupz(rd->name);
dim_payload.id = strdupz(rd->id);
dim_payload.name = rd->name;
dim_payload.id = rd->id;
dim_payload.chart_id = strdupz(rd->rrdset->name);
dim_payload.chart_id = rd->rrdset->name;
dim_payload.created_at.tv_sec = first_t;
if (unlikely(!live))
dim_payload.last_timestamp.tv_sec = last_t;
@ -199,10 +204,6 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
char *payload = generate_chart_dimension_updated(&size, &dim_payload);
if (likely(payload))
rc = aclk_add_chart_payload(wc->uuid_str, &rd->state->metric_uuid, claim_id, ACLK_PAYLOAD_DIMENSION, (void *)payload, size);
freez((char *)dim_payload.node_id);
freez((char *)dim_payload.chart_id);
freez((char *)dim_payload.name);
freez((char *)dim_payload.id);
freez(payload);
freez(claim_id);
}
@ -229,6 +230,10 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
if (unlikely(!claim_id))
return;
uuid_t claim_uuid;
if (uuid_parse(claim_id, claim_uuid))
return;
int limit = cmd.count > 0 ? cmd.count : 1;
uint64_t first_sequence;
@ -252,7 +257,7 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
return;
}
rc = sqlite3_bind_text(res, 1, claim_id , -1, SQLITE_STATIC);
rc = sqlite3_bind_blob(res, 1, claim_uuid , sizeof(claim_uuid), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@ -404,7 +409,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d
destroy_chart_config_updated(&chart_config);
}
else
info("DEBUG: Chart config for %s not found", hash_id);
info("Chart config for %s not found", hash_id);
bind_fail:
rc = sqlite3_finalize(res);
@ -455,6 +460,7 @@ void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_
void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
{
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
BUFFER *sql = buffer_create(1024);
buffer_sprintf(sql, "UPDATE aclk_chart_%s SET status = NULL, date_submitted = NULL WHERE sequence_id >= %"PRIu64";",
wc->uuid_str, cmd.param1);
@ -462,15 +468,18 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
if (cmd.param1 == 1) {
db_lock();
buffer_flush(sql);
info("DEBUG: Deleting all data for %s", wc->uuid_str);
buffer_sprintf(sql, "DELETE FROM aclk_chart_payload_%s; DELETE FROM aclk_chart_%s; DELETE FROM aclk_chart_latest_%s;",
wc->uuid_str, wc->uuid_str, wc->uuid_str);
info("Received full resync for %s", wc->uuid_str);
buffer_sprintf(sql, "DELETE FROM aclk_chart_payload_%s; DELETE FROM aclk_chart_%s; " \
"DELETE FROM aclk_chart_latest_%s;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
db_execute("BEGIN TRANSACTION;");
db_execute(buffer_tostring(sql));
db_execute("COMMIT TRANSACTION;");
db_unlock();
wc->chart_sequence_id = 0;
wc->chart_timestamp = 0;
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
RRDHOST *host = wc->host;
rrdhost_rdlock(host);
RRDSET *st;
@ -484,7 +493,6 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
rrdset_unlock(st);
}
rrdhost_unlock(host);
#endif
}
else {
//sql_chart_deduplicate(wc, cmd);
@ -492,6 +500,11 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
}
buffer_free(sql);
wc->chart_updates = 1;
#else
UNUSED(wc);
UNUSED(cmd);
#endif
return;
}
@ -555,6 +568,7 @@ void aclk_ack_chart_sequence_id(char *node_id, uint64_t last_sequence_id)
return;
}
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
void aclk_reset_chart_event(char *node_id, uint64_t last_sequence_id)
{
if (unlikely(!node_id))
@ -564,21 +578,47 @@ void aclk_reset_chart_event(char *node_id, uint64_t last_sequence_id)
aclk_submit_param_command(node_id, ACLK_DATABASE_RESET_CHART, last_sequence_id);
return;
}
#endif
// ST is read locked
int sql_queue_chart_to_aclk(RRDSET *st)
{
#ifdef ENABLE_ACLK
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
if (!aclk_use_new_cloud_arch)
#endif
{
rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
aclk_update_chart(st->rrdhost, st->id, 1);
return 0;
}
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
return sql_queue_chart_payload((struct aclk_database_worker_config *) st->rrdhost->dbsync_worker,
st, ACLK_DATABASE_ADD_CHART);
#else
return 0;
#endif
#else
UNUSED(st);
return 0;
#endif
}
int sql_queue_dimension_to_aclk(RRDDIM *rd)
{
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
if (!aclk_use_new_cloud_arch)
return 0;
int rc = sql_queue_chart_payload((struct aclk_database_worker_config *) rd->rrdset->rrdhost->dbsync_worker,
rd, ACLK_DATABASE_ADD_DIMENSION);
if (likely(!rc))
rrddim_flag_set(rd, RRDDIM_FLAG_ACLK);
return rc;
#else
UNUSED(rd);
return 0;
#endif
}
void sql_chart_deduplicate(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@ -674,6 +714,7 @@ fail:
// Start streaming charts / dimensions for node_id
void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at, uint64_t batch_id)
{
UNUSED(created_at);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
if (unlikely(!node_id))
return;
@ -692,52 +733,41 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
rrd_unlock();
wc = (struct aclk_database_worker_config *)host->dbsync_worker;
if (likely(wc)) {
// if (unlikely(!wc->chart_updates)) {
// struct aclk_database_cmd cmd;
// cmd.opcode = ACLK_DATABASE_NODE_INFO;
// cmd.completion = NULL;
// aclk_database_enq_cmd(wc, &cmd);
// }
wc->chart_reset_count++;
__sync_synchronize();
wc->chart_updates = 0;
wc->batch_id = batch_id;
__sync_synchronize();
wc->batch_created = now_realtime_sec();
info("DEBUG: START streaming charts for %s (%s) enabled -- last streamed sequence %"PRIu64" t=%ld (reset count=%d)", node_id, wc->uuid_str,
wc->chart_sequence_id, wc->chart_timestamp, wc->chart_reset_count);
// If mismatch detected
if (sequence_id > wc->chart_sequence_id || wc->chart_reset_count > 10) {
info("DEBUG: Full resync requested -- reset_count=%d", wc->chart_reset_count);
debug(D_ACLK_SYNC,"Requesting full resync from the cloud -- reset_count=%d", wc->chart_reset_count);
chart_reset_t chart_reset;
chart_reset.node_id = strdupz(node_id);
chart_reset.claim_id = is_agent_claimed();
chart_reset.reason = SEQ_ID_NOT_EXISTS;
aclk_chart_reset(chart_reset);
// wc->chart_updates = 0;
wc->chart_reset_count = -1;
if (chart_reset.claim_id) {
chart_reset.node_id = node_id;
chart_reset.reason = SEQ_ID_NOT_EXISTS;
aclk_chart_reset(chart_reset);
freez(chart_reset.claim_id);
wc->chart_reset_count = -1;
}
return;
} else {
struct aclk_database_cmd cmd;
memset(&cmd, 0, sizeof(cmd));
// TODO: handle timestamp
// if (!wc->chart_reset_count)
// wc->chart_delay = now_realtime_sec() + 60;
// else
// wc->chart_delay = 0;
if (sequence_id < wc->chart_sequence_id) { // || created_at != wc->chart_timestamp) {
// wc->chart_updates = 0;
if (sequence_id)
info("DEBUG: Synchonization mismatch detected");
else
info("DEBUG: Synchonization mismatch detected; full resync ACKed from the cloud");
cmd.opcode = ACLK_DATABASE_RESET_CHART;
cmd.param1 = sequence_id + 1;
cmd.completion = NULL;
aclk_database_enq_cmd(wc, &cmd);
}
else
else {
debug(D_ACLK_SYNC,"START streaming charts for %s enabled -- last streamed sequence %"PRIu64 \
" t=%ld (reset count=%d)", wc->host_guid, wc->chart_sequence_id,
wc->chart_timestamp, wc->chart_reset_count);
wc->chart_reset_count = 0;
wc->chart_updates = 1;
}
}
}
else
@ -750,7 +780,6 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
#else
UNUSED(node_id);
UNUSED(sequence_id);
UNUSED(created_at);
UNUSED(batch_id);
#endif
return;

160
database/sqlite/sqlite_aclk_node.c
View file

@ -5,6 +5,7 @@
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
#include "../../aclk/aclk_charts_api.h"
#include "../../aclk/aclk.h"
#endif
void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@ -14,6 +15,9 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
struct update_node_info node_info;
if (!wc->host)
return;
rrd_wrlock();
node_info.node_id = wc->node_id;
node_info.claim_id = is_agent_claimed();
@ -59,3 +63,159 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
return;
}
#define SQL_SELECT_HOST_MEMORY_MODE "select memory_mode from chart where host_id = @host_id limit 1;"
static RRD_MEMORY_MODE sql_get_host_memory_mode(uuid_t *host_id)
{
int rc;
RRD_MEMORY_MODE memory_mode = RRD_MEMORY_MODE_RAM;
sqlite3_stmt *res = NULL;
rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_HOST_MEMORY_MODE, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement to read host memory mode");
return memory_mode;
}
rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind host parameter to fetch host memory mode");
goto failed;
}
while (sqlite3_step(res) == SQLITE_ROW) {
memory_mode = (RRD_MEMORY_MODE) sqlite3_column_int(res, 0);
}
failed:
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to finalize the prepared statement when reading host memory mode");
return memory_mode;
}
#define SELECT_HOST_DIMENSION_LIST "SELECT d.dim_id, c.update_every, c.type||'.'||c.id FROM chart c, dimension d, host h " \
"WHERE d.chart_id = c.chart_id AND c.host_id = h.host_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
#define SELECT_HOST_CHART_LIST "SELECT distinct h.host_id, c.update_every, c.type||'.'||c.id FROM chart c, host h " \
"WHERE c.host_id = h.host_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
void aclk_update_retention(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
{
UNUSED(cmd);
#ifdef ENABLE_NEW_CLOUD_PROTOCOL
int rc;
if (!aclk_use_new_cloud_arch || !aclk_connected)
return;
char *claim_id = is_agent_claimed();
if (unlikely(!claim_id))
return;
sqlite3_stmt *res = NULL;
RRD_MEMORY_MODE memory_mode;
uuid_t host_uuid;
rc = uuid_parse(wc->host_guid, host_uuid);
if (unlikely(rc))
return;
if (wc->host)
memory_mode = wc->host->rrd_memory_mode;
else
memory_mode = sql_get_host_memory_mode(&host_uuid);
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_DIMENSION_LIST, -1, &res, 0);
else
rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_CHART_LIST, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement to fetch host dimensions");
freez(claim_id);
return;
}
rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind host parameter to fetch host dimensions");
goto failed;
}
time_t start_time = LONG_MAX;
time_t first_entry_t;
uint32_t update_every = 0;
struct retention_updated rotate_data;
memset(&rotate_data, 0, sizeof(rotate_data));
int max_intervals = 32;
rotate_data.interval_duration_count = 0;
rotate_data.interval_durations = callocz(max_intervals, sizeof(*rotate_data.interval_durations));
now_realtime_timeval(&rotate_data.rotation_timestamp);
rotate_data.memory_mode = memory_mode;
rotate_data.claim_id = claim_id;
rotate_data.node_id = strdupz(wc->node_id);
while (sqlite3_step(res) == SQLITE_ROW) {
if (!update_every || update_every != (uint32_t) sqlite3_column_int(res, 1)) {
if (update_every) {
debug(D_ACLK_SYNC,"Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
rotate_data.interval_durations[rotate_data.interval_duration_count].retention = rotate_data.rotation_timestamp.tv_sec - start_time;
rotate_data.interval_duration_count++;
}
update_every = (uint32_t) sqlite3_column_int(res, 1);
rotate_data.interval_durations[rotate_data.interval_duration_count].update_every = update_every;
start_time = LONG_MAX;
}
#ifdef ENABLE_DBENGINE
time_t last_entry_t;
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
rc = rrdeng_metric_latest_time_by_uuid((uuid_t *)sqlite3_column_blob(res, 0), &first_entry_t, &last_entry_t);
else
#endif
{
if (wc->host) {
RRDSET *st = NULL;
rc = (st = rrdset_find(wc->host, (const char *)sqlite3_column_text(res, 2))) ? 0 : 1;
if (!rc)
first_entry_t = rrdset_first_entry_t(st);
}
else {
rc = 0;
first_entry_t = rotate_data.rotation_timestamp.tv_sec;
}
}
if (likely(!rc && first_entry_t))
start_time = MIN(start_time, first_entry_t);
}
if (update_every) {
debug(D_ACLK_SYNC, "Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
rotate_data.interval_durations[rotate_data.interval_duration_count].retention = rotate_data.rotation_timestamp.tv_sec - start_time;
rotate_data.interval_duration_count++;
}
for (int i = 0; i < rotate_data.interval_duration_count; ++i) {
debug(D_ACLK_SYNC,"%d --> Update %s for %u Retention = %u", i, wc->host_guid,
rotate_data.interval_durations[i].update_every, rotate_data.interval_durations[i].retention);
};
aclk_retention_updated(&rotate_data);
freez(rotate_data.node_id);
freez(rotate_data.interval_durations);
failed:
freez(claim_id);
rc = sqlite3_finalize(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to finalize the prepared statement when reading host dimensions");
#else
UNUSED(wc);
#endif
return;
}

1
database/sqlite/sqlite_aclk_node.h
View file

@ -4,4 +4,5 @@
#define NETDATA_SQLITE_ACLK_NODE_H
void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
void aclk_update_retention(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
#endif //NETDATA_SQLITE_ACLK_NODE_H

208
database/sqlite/sqlite_functions.c
View file

@ -26,6 +26,25 @@ const char *database_config[] = {
"red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
"repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
"p_db_lookup_after int, p_db_lookup_before int, p_update_every int);",
"CREATE TABLE IF NOT EXISTS chart_hash_map(chart_id blob , hash_id blob, UNIQUE (chart_id, hash_id));",
"CREATE TABLE IF NOT EXISTS chart_hash(hash_id blob PRIMARY KEY,type text, id text, name text, "
"family text, context text, title text, unit text, plugin text, "
"module text, priority integer, chart_type, last_used);",
"CREATE VIEW IF NOT EXISTS v_chart_hash as SELECT ch.*, chm.chart_id FROM chart_hash ch, chart_hash_map chm "
"WHERE ch.hash_id = chm.hash_id;",
"CREATE TRIGGER IF NOT EXISTS tr_v_chart_hash INSTEAD OF INSERT on v_chart_hash BEGIN "
"INSERT INTO chart_hash (hash_id, type, id, name, family, context, title, unit, plugin, "
"module, priority, chart_type, last_used) "
"values (new.hash_id, new.type, new.id, new.name, new.family, new.context, new.title, new.unit, new.plugin, "
"new.module, new.priority, new.chart_type, strftime('%s')) "
"ON CONFLICT (hash_id) DO UPDATE SET last_used = strftime('%s'); "
"INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) "
"on conflict (chart_id, hash_id) do nothing; END; ",
"delete from chart_active;",
"delete from dimension_active;",
"delete from chart where chart_id not in (select chart_id from dimension);",
@ -1331,6 +1350,174 @@ failed:
return;
}
/*
* Store a chart hash in the database
*/
#define SQL_STORE_CHART_HASH "insert into v_chart_hash (hash_id, type, id, " \
"name, family, context, title, unit, plugin, module, priority, chart_type, last_used, chart_id) " \
"values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11, ?12, strftime('%s'), ?13);"
int sql_store_chart_hash(
uuid_t *hash_id, uuid_t *chart_id, const char *type, const char *id, const char *name, const char *family,
const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
RRDSET_TYPE chart_type)
{
static __thread sqlite3_stmt *res = NULL;
int rc, param = 0;
if (unlikely(!db_meta)) {
if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
return 0;
error_report("Database has not been initialized");
return 1;
}
if (unlikely(!res)) {
rc = prepare_statement(db_meta, SQL_STORE_CHART_HASH, &res);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement to store chart, rc = %d", rc);
return 1;
}
}
param++;
rc = sqlite3_bind_blob(res, 1, hash_id, sizeof(*hash_id), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 2, type, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 3, id, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
if (name && *name)
rc = sqlite3_bind_text(res, 4, name, -1, SQLITE_STATIC);
else
rc = sqlite3_bind_null(res, 4);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 5, family, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 6, context, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 7, title, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 8, units, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 9, plugin, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_text(res, 10, module, -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_int(res, 11, (int) priority);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_int(res, 12, chart_type);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
param++;
rc = sqlite3_bind_blob(res, 13, chart_id, sizeof(*chart_id), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
rc = execute_insert(res);
if (unlikely(rc != SQLITE_DONE))
error_report("Failed to store chart hash_id, rc = %d", rc);
rc = sqlite3_reset(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc);
return 0;
bind_fail:
error_report("Failed to bind parameter %d to store chart hash_id, rc = %d", param, rc);
rc = sqlite3_reset(res);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc);
return 1;
}
void compute_chart_hash(RRDSET *st)
{
EVP_MD_CTX *evpctx;
unsigned char hash_value[EVP_MAX_MD_SIZE];
unsigned int hash_len;
char priority_str[32];
sprintf(priority_str, "%ld", st->priority);
evpctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
//EVP_DigestUpdate(evpctx, st->type, strlen(st->type));
EVP_DigestUpdate(evpctx, st->id, strlen(st->id));
EVP_DigestUpdate(evpctx, st->name, strlen(st->name));
EVP_DigestUpdate(evpctx, st->family, strlen(st->family));
EVP_DigestUpdate(evpctx, st->context, strlen(st->context));
EVP_DigestUpdate(evpctx, st->title, strlen(st->title));
EVP_DigestUpdate(evpctx, st->units, strlen(st->units));
EVP_DigestUpdate(evpctx, st->plugin_name, strlen(st->plugin_name));
if (st->module_name)
EVP_DigestUpdate(evpctx, st->module_name, strlen(st->module_name));
// EVP_DigestUpdate(evpctx, priority_str, strlen(priority_str));
EVP_DigestUpdate(evpctx, &st->priority, sizeof(st->priority));
EVP_DigestUpdate(evpctx, &st->chart_type, sizeof(st->chart_type));
EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
EVP_MD_CTX_destroy(evpctx);
fatal_assert(hash_len > sizeof(uuid_t));
char uuid_str[GUID_LEN + 1];
uuid_unparse_lower(*((uuid_t *) &hash_value), uuid_str);
//info("Calculating HASH %s for chart %s", uuid_str, st->name);
uuid_copy(st->state->hash_id, *((uuid_t *) &hash_value));
(void)sql_store_chart_hash(
(uuid_t *)&hash_value,
st->chart_uuid,
st->type,
st->id,
st->name,
st->family,
st->context,
st->title,
st->units,
st->plugin_name,
st->module_name,
st->priority,
st->chart_type);
return;
}
#define SQL_STORE_CLAIM_ID "insert into node_instance " \
"(host_id, claim_id, date_created) values (@host_id, @claim_id, strftime('%s')) " \
"on conflict(host_id) do update set claim_id = excluded.claim_id;"
@ -1389,9 +1576,16 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
return;
}
struct aclk_database_worker_config *wc = host->dbsync_worker;
if (unlikely(!host->node_id))
host->node_id = mallocz(sizeof(*host->node_id));
uuid_copy(*(host->node_id), *node_id);
if (unlikely(!wc))
sql_create_aclk_table(host, &host->host_uuid, node_id);
else
uuid_unparse_lower(*node_id, wc->node_id);
return;
}
@ -1581,7 +1775,7 @@ struct node_instance_list *get_node_list(void)
rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement store chart labels");
error_report("Failed to prepare statement to get node instance information");
return NULL;
};
@ -1591,7 +1785,7 @@ struct node_instance_list *get_node_list(void)
row++;
if (sqlite3_reset(res) != SQLITE_OK) {
error_report("Failed to reset the prepared statement fetching storing node instance information");
error_report("Failed to reset the prepared statement while fetching node instance information");
goto failed;
}
node_list = callocz(row + 1, sizeof(*node_list));
@ -1603,7 +1797,7 @@ struct node_instance_list *get_node_list(void)
if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
uuid_copy(node_list[row].host_id, *host_id);
node_list[row].querable = 1;
node_list[row].queryable = 1;
uuid_unparse_lower(*host_id, host_guid);
node_list[row].live = rrdhost_find_by_guid(host_guid, 0) ? 1 : 0;
node_list[row].hops = uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0;
@ -1617,7 +1811,7 @@ struct node_instance_list *get_node_list(void)
failed:
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
error_report("Failed to finalize the prepared statement when storing node instance information");
error_report("Failed to finalize the prepared statement when fetching node instance information");
return node_list;
};
@ -1637,13 +1831,13 @@ void sql_load_node_id(RRDHOST *host)
rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to prepare statement store chart labels");
error_report("Failed to prepare statement to fetch node id");
return;
};
rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK)) {
error_report("Failed to bind host_id parameter to store node instance information");
error_report("Failed to bind host_id parameter to load node instance information");
goto failed;
}
@ -1657,7 +1851,7 @@ void sql_load_node_id(RRDHOST *host)
failed:
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
error_report("Failed to finalize the prepared statement when storing node instance information");
error_report("Failed to finalize the prepared statement when loading node instance information");
return;
};

5
database/sqlite/sqlite_functions.h
View file

@ -12,7 +12,7 @@ struct node_instance_list {
uuid_t host_id;
char *hostname;
int live;
int querable;
int queryable;
int hops;
};
@ -90,6 +90,5 @@ extern int get_host_id(uuid_t *node_id, uuid_t *host_id);
extern void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
extern struct node_instance_list *get_node_list(void);
extern void sql_load_node_id(RRDHOST *host);
extern int execute_insert(sqlite3_stmt *res);
extern int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
extern void compute_chart_hash(RRDSET *st);
#endif //NETDATA_SQLITE_FUNCTIONS_H

15
database/sqlite/sqlite_health.c
View file

@ -109,7 +109,12 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
/* Health related SQL queries
Inserts an entry in the table
*/
#define SQL_INSERT_HEALTH_LOG(guid) "INSERT INTO health_log_%s(hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", guid
#define SQL_INSERT_HEALTH_LOG(guid) "INSERT INTO health_log_%s(hostname, unique_id, alarm_id, alarm_event_id, " \
"config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, " \
"exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, " \
"units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, " \
"class, component, type) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", guid
void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
sqlite3_stmt *res = NULL;
int rc;
@ -639,7 +644,13 @@ void sql_health_alarm_log_load(RRDHOST *host) {
/*
* Store an alert config hash in the database
*/
#define SQL_STORE_ALERT_CONFIG_HASH "insert or replace into alert_hash (hash_id, date_updated, alarm, template, on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every) values (?1,strftime('%s'),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34);"
#define SQL_STORE_ALERT_CONFIG_HASH "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \
"on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, " \
"charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
"p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
"p_db_lookup_before, p_update_every) values (?1,strftime('%s'),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
"?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34);"
int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
{
static __thread sqlite3_stmt *res = NULL;

5
health/health_log.c
View file

@ -162,10 +162,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
#ifdef ENABLE_ACLK
if (netdata_cloud_setting) {
if ((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) ||
((ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL))) {
aclk_update_alarm(host, ae);
}
sql_queue_alarm_to_aclk(host, ae);
}
#endif
}