0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-20 12:00:08 +00:00
netdata_netdata/database/storage_engine.c
Stelios Fragkakis 49234f23de
Multi-Tier database backend for long term metrics storage ()
* Tier part 1

* Tier part 2

* Tier part 3

* Tier part 4

* Tier part 5

* Fix some ML compilation errors

* fix more conflicts

* pass proper tier

* move metric_uuid from state to RRDDIM

* move aclk_live_status from state to RRDDIM

* move ml_dimension from state to RRDDIM

* abstracted the data collection interface

* support flushing for mem db too

* abstracted the query api

* abstracted latest/oldest time per metric

* cleanup

* store_metric for tier1

* fix for store_metric

* allow multiple tiers, more than 2

* state to tier

* Change storage type in db. Query param to request min, max, sum or average

* Store tier data correctly

* Fix skipping tier page type

* Add tier grouping in the tier

* Fix to handle archived charts (part 1)

* Temp fix for query granularity when requesting tier1 data

* Fix parameters in the correct order and calculate the anomaly based on the anomaly count

* Proper tiering grouping

* Anomaly calculation based on anomaly count

* force type checking on storage handles

* update cmocka tests

* fully dynamic number of storage tiers

* fix static allocation

* configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode

* use default page dt using the tiering info

* automatic selection of tier

* fix for automatic selection of tier

* working prototype of dynamic tier selection

* automatic selection of tier done right (I hope)

* ask for the proper tier value, based on the grouping function

* fixes for unittests and load_metric_next()

* fixes for lgtm findings

* minor renames

* add dbengine to page cache size setting

* add dbengine to page cache with malloc

* query engine optimized to loop as little are required based on the view_update_every

* query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA

* report db points per tier in jsonwrap

* query planer that switches database tiers on the fly to satisfy the query for the entire timeframe

* dbegnine statistics and documentation (in progress)

* calculate average point duration in db

* handle single point pages the best we can

* handle single point pages even better

* Keep page type in the rrdeng_page_descr

* updated doc

* handle future backwards compatibility - improved statistics

* support &tier=X in queries

* enfore increasing iterations on tiers

* tier 1 is always 1 iteration

* backfilling higher tiers on first data collection

* reversed anomaly bit

* set up to 5 tiers

* natural points should only be offered on tier 0, except a specific tier is selected

* do not allow more than 65535 points of tier0 to be aggregated on any tier

* Work only on actually activated tiers

* fix query interpolation

* fix query interpolation again

* fix lgtm finding

* Activate one tier for now

* backfilling of higher tiers using raw metrics from lower tiers

* fix for crash on start when storage tiers is increased from the default

* more statistics on exit

* fix bug that prevented higher tiers to get any values; added backfilling options

* fixed the statistics log line

* removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle

* fixed division by zero on zero points_wanted

* removed dead code

* Decide on the descr->type for the type of metric

* dont store metrics on unknown page types

* free db_metric_handle on sql based context queries

* Disable STORAGE_POINT value check in the exporting engine unit tests

* fix for db modes other than dbengine

* fix for aclk archived chart queries destroying db_metric_handles of valid rrddims

* fix left-over freez() instead of OWA freez on median queries

Co-authored-by: Costa Tsaousis <costa@netdata.cloud>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>
2022-07-06 14:01:53 +03:00

134 lines
3.6 KiB
C

// SPDX-License-Identifier: GPL-3.0-or-later
#include "storage_engine.h"
#include "ram/rrddim_mem.h"
#ifdef ENABLE_DBENGINE
#include "engine/rrdengineapi.h"
#endif
#define im_collect_ops { \
.init = rrddim_collect_init,\
.store_metric = rrddim_collect_store_metric,\
.flush = rrddim_store_metric_flush,\
.finalize = rrddim_collect_finalize\
}
#define im_query_ops { \
.init = rrddim_query_init, \
.next_metric = rrddim_query_next_metric, \
.is_finished = rrddim_query_is_finished, \
.finalize = rrddim_query_finalize, \
.latest_time = rrddim_query_latest_time, \
.oldest_time = rrddim_query_oldest_time \
}
static STORAGE_ENGINE engines[] = {
{
.id = RRD_MEMORY_MODE_NONE,
.name = RRD_MEMORY_MODE_NONE_NAME,
.api = {
.init = rrddim_metric_init,
.free = rrddim_metric_free,
.collect_ops = im_collect_ops,
.query_ops = im_query_ops
}
},
{
.id = RRD_MEMORY_MODE_RAM,
.name = RRD_MEMORY_MODE_RAM_NAME,
.api = {
.init = rrddim_metric_init,
.free = rrddim_metric_free,
.collect_ops = im_collect_ops,
.query_ops = im_query_ops
}
},
{
.id = RRD_MEMORY_MODE_MAP,
.name = RRD_MEMORY_MODE_MAP_NAME,
.api = {
.init = rrddim_metric_init,
.free = rrddim_metric_free,
.collect_ops = im_collect_ops,
.query_ops = im_query_ops
}
},
{
.id = RRD_MEMORY_MODE_SAVE,
.name = RRD_MEMORY_MODE_SAVE_NAME,
.api = {
.init = rrddim_metric_init,
.free = rrddim_metric_free,
.collect_ops = im_collect_ops,
.query_ops = im_query_ops
}
},
{
.id = RRD_MEMORY_MODE_ALLOC,
.name = RRD_MEMORY_MODE_ALLOC_NAME,
.api = {
.init = rrddim_metric_init,
.free = rrddim_metric_free,
.collect_ops = im_collect_ops,
.query_ops = im_query_ops
}
},
#ifdef ENABLE_DBENGINE
{
.id = RRD_MEMORY_MODE_DBENGINE,
.name = RRD_MEMORY_MODE_DBENGINE_NAME,
.api = {
.init = rrdeng_metric_init,
.free = rrdeng_metric_free,
.collect_ops = {
.init = rrdeng_store_metric_init,
.store_metric = rrdeng_store_metric_next,
.flush = rrdeng_store_metric_flush_current_page,
.finalize = rrdeng_store_metric_finalize
},
.query_ops = {
.init = rrdeng_load_metric_init,
.next_metric = rrdeng_load_metric_next,
.is_finished = rrdeng_load_metric_is_finished,
.finalize = rrdeng_load_metric_finalize,
.latest_time = rrdeng_metric_latest_time,
.oldest_time = rrdeng_metric_oldest_time
}
}
},
#endif
{ .id = RRD_MEMORY_MODE_NONE, .name = NULL }
};
STORAGE_ENGINE* storage_engine_find(const char* name)
{
for (STORAGE_ENGINE* it = engines; it->name; it++) {
if (strcmp(it->name, name) == 0)
return it;
}
return NULL;
}
STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode)
{
for (STORAGE_ENGINE* it = engines; it->name; it++) {
if (it->id == mmode)
return it;
}
return NULL;
}
STORAGE_ENGINE* storage_engine_foreach_init()
{
// Assuming at least one engine exists
return &engines[0];
}
STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it)
{
if (!it || !it->name)
return NULL;
it++;
return it->name ? it : NULL;
}