mirror of
https://github.com/netdata/netdata.git
synced 2025-04-20 12:00:08 +00:00

* Tier part 1 * Tier part 2 * Tier part 3 * Tier part 4 * Tier part 5 * Fix some ML compilation errors * fix more conflicts * pass proper tier * move metric_uuid from state to RRDDIM * move aclk_live_status from state to RRDDIM * move ml_dimension from state to RRDDIM * abstracted the data collection interface * support flushing for mem db too * abstracted the query api * abstracted latest/oldest time per metric * cleanup * store_metric for tier1 * fix for store_metric * allow multiple tiers, more than 2 * state to tier * Change storage type in db. Query param to request min, max, sum or average * Store tier data correctly * Fix skipping tier page type * Add tier grouping in the tier * Fix to handle archived charts (part 1) * Temp fix for query granularity when requesting tier1 data * Fix parameters in the correct order and calculate the anomaly based on the anomaly count * Proper tiering grouping * Anomaly calculation based on anomaly count * force type checking on storage handles * update cmocka tests * fully dynamic number of storage tiers * fix static allocation * configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode * use default page dt using the tiering info * automatic selection of tier * fix for automatic selection of tier * working prototype of dynamic tier selection * automatic selection of tier done right (I hope) * ask for the proper tier value, based on the grouping function * fixes for unittests and load_metric_next() * fixes for lgtm findings * minor renames * add dbengine to page cache size setting * add dbengine to page cache with malloc * query engine optimized to loop as little are required based on the view_update_every * query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA * report db points per tier in jsonwrap * query planer that switches database tiers on the fly to satisfy the query for the entire timeframe * dbegnine statistics and documentation (in progress) * calculate average point duration in db * handle single point pages the best we can * handle single point pages even better * Keep page type in the rrdeng_page_descr * updated doc * handle future backwards compatibility - improved statistics * support &tier=X in queries * enfore increasing iterations on tiers * tier 1 is always 1 iteration * backfilling higher tiers on first data collection * reversed anomaly bit * set up to 5 tiers * natural points should only be offered on tier 0, except a specific tier is selected * do not allow more than 65535 points of tier0 to be aggregated on any tier * Work only on actually activated tiers * fix query interpolation * fix query interpolation again * fix lgtm finding * Activate one tier for now * backfilling of higher tiers using raw metrics from lower tiers * fix for crash on start when storage tiers is increased from the default * more statistics on exit * fix bug that prevented higher tiers to get any values; added backfilling options * fixed the statistics log line * removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle * fixed division by zero on zero points_wanted * removed dead code * Decide on the descr->type for the type of metric * dont store metrics on unknown page types * free db_metric_handle on sql based context queries * Disable STORAGE_POINT value check in the exporting engine unit tests * fix for db modes other than dbengine * fix for aclk archived chart queries destroying db_metric_handles of valid rrddims * fix left-over freez() instead of OWA freez on median queries Co-authored-by: Costa Tsaousis <costa@netdata.cloud> Co-authored-by: Vladimir Kobal <vlad@prokk.net>
134 lines
3.6 KiB
C
134 lines
3.6 KiB
C
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
#include "storage_engine.h"
|
|
#include "ram/rrddim_mem.h"
|
|
#ifdef ENABLE_DBENGINE
|
|
#include "engine/rrdengineapi.h"
|
|
#endif
|
|
|
|
#define im_collect_ops { \
|
|
.init = rrddim_collect_init,\
|
|
.store_metric = rrddim_collect_store_metric,\
|
|
.flush = rrddim_store_metric_flush,\
|
|
.finalize = rrddim_collect_finalize\
|
|
}
|
|
|
|
#define im_query_ops { \
|
|
.init = rrddim_query_init, \
|
|
.next_metric = rrddim_query_next_metric, \
|
|
.is_finished = rrddim_query_is_finished, \
|
|
.finalize = rrddim_query_finalize, \
|
|
.latest_time = rrddim_query_latest_time, \
|
|
.oldest_time = rrddim_query_oldest_time \
|
|
}
|
|
|
|
static STORAGE_ENGINE engines[] = {
|
|
{
|
|
.id = RRD_MEMORY_MODE_NONE,
|
|
.name = RRD_MEMORY_MODE_NONE_NAME,
|
|
.api = {
|
|
.init = rrddim_metric_init,
|
|
.free = rrddim_metric_free,
|
|
.collect_ops = im_collect_ops,
|
|
.query_ops = im_query_ops
|
|
}
|
|
},
|
|
{
|
|
.id = RRD_MEMORY_MODE_RAM,
|
|
.name = RRD_MEMORY_MODE_RAM_NAME,
|
|
.api = {
|
|
.init = rrddim_metric_init,
|
|
.free = rrddim_metric_free,
|
|
.collect_ops = im_collect_ops,
|
|
.query_ops = im_query_ops
|
|
}
|
|
},
|
|
{
|
|
.id = RRD_MEMORY_MODE_MAP,
|
|
.name = RRD_MEMORY_MODE_MAP_NAME,
|
|
.api = {
|
|
.init = rrddim_metric_init,
|
|
.free = rrddim_metric_free,
|
|
.collect_ops = im_collect_ops,
|
|
.query_ops = im_query_ops
|
|
}
|
|
},
|
|
{
|
|
.id = RRD_MEMORY_MODE_SAVE,
|
|
.name = RRD_MEMORY_MODE_SAVE_NAME,
|
|
.api = {
|
|
.init = rrddim_metric_init,
|
|
.free = rrddim_metric_free,
|
|
.collect_ops = im_collect_ops,
|
|
.query_ops = im_query_ops
|
|
}
|
|
},
|
|
{
|
|
.id = RRD_MEMORY_MODE_ALLOC,
|
|
.name = RRD_MEMORY_MODE_ALLOC_NAME,
|
|
.api = {
|
|
.init = rrddim_metric_init,
|
|
.free = rrddim_metric_free,
|
|
.collect_ops = im_collect_ops,
|
|
.query_ops = im_query_ops
|
|
}
|
|
},
|
|
#ifdef ENABLE_DBENGINE
|
|
{
|
|
.id = RRD_MEMORY_MODE_DBENGINE,
|
|
.name = RRD_MEMORY_MODE_DBENGINE_NAME,
|
|
.api = {
|
|
.init = rrdeng_metric_init,
|
|
.free = rrdeng_metric_free,
|
|
.collect_ops = {
|
|
.init = rrdeng_store_metric_init,
|
|
.store_metric = rrdeng_store_metric_next,
|
|
.flush = rrdeng_store_metric_flush_current_page,
|
|
.finalize = rrdeng_store_metric_finalize
|
|
},
|
|
.query_ops = {
|
|
.init = rrdeng_load_metric_init,
|
|
.next_metric = rrdeng_load_metric_next,
|
|
.is_finished = rrdeng_load_metric_is_finished,
|
|
.finalize = rrdeng_load_metric_finalize,
|
|
.latest_time = rrdeng_metric_latest_time,
|
|
.oldest_time = rrdeng_metric_oldest_time
|
|
}
|
|
}
|
|
},
|
|
#endif
|
|
{ .id = RRD_MEMORY_MODE_NONE, .name = NULL }
|
|
};
|
|
|
|
STORAGE_ENGINE* storage_engine_find(const char* name)
|
|
{
|
|
for (STORAGE_ENGINE* it = engines; it->name; it++) {
|
|
if (strcmp(it->name, name) == 0)
|
|
return it;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode)
|
|
{
|
|
for (STORAGE_ENGINE* it = engines; it->name; it++) {
|
|
if (it->id == mmode)
|
|
return it;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
STORAGE_ENGINE* storage_engine_foreach_init()
|
|
{
|
|
// Assuming at least one engine exists
|
|
return &engines[0];
|
|
}
|
|
|
|
STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it)
|
|
{
|
|
if (!it || !it->name)
|
|
return NULL;
|
|
|
|
it++;
|
|
return it->name ? it : NULL;
|
|
}
|