0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-05-19 23:41:46 +00:00
netdata_netdata/web/api/queries/rrdr.c
Stelios Fragkakis 49234f23de
Multi-Tier database backend for long term metrics storage ()
* Tier part 1

* Tier part 2

* Tier part 3

* Tier part 4

* Tier part 5

* Fix some ML compilation errors

* fix more conflicts

* pass proper tier

* move metric_uuid from state to RRDDIM

* move aclk_live_status from state to RRDDIM

* move ml_dimension from state to RRDDIM

* abstracted the data collection interface

* support flushing for mem db too

* abstracted the query api

* abstracted latest/oldest time per metric

* cleanup

* store_metric for tier1

* fix for store_metric

* allow multiple tiers, more than 2

* state to tier

* Change storage type in db. Query param to request min, max, sum or average

* Store tier data correctly

* Fix skipping tier page type

* Add tier grouping in the tier

* Fix to handle archived charts (part 1)

* Temp fix for query granularity when requesting tier1 data

* Fix parameters in the correct order and calculate the anomaly based on the anomaly count

* Proper tiering grouping

* Anomaly calculation based on anomaly count

* force type checking on storage handles

* update cmocka tests

* fully dynamic number of storage tiers

* fix static allocation

* configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode

* use default page dt using the tiering info

* automatic selection of tier

* fix for automatic selection of tier

* working prototype of dynamic tier selection

* automatic selection of tier done right (I hope)

* ask for the proper tier value, based on the grouping function

* fixes for unittests and load_metric_next()

* fixes for lgtm findings

* minor renames

* add dbengine to page cache size setting

* add dbengine to page cache with malloc

* query engine optimized to loop as little are required based on the view_update_every

* query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA

* report db points per tier in jsonwrap

* query planer that switches database tiers on the fly to satisfy the query for the entire timeframe

* dbegnine statistics and documentation (in progress)

* calculate average point duration in db

* handle single point pages the best we can

* handle single point pages even better

* Keep page type in the rrdeng_page_descr

* updated doc

* handle future backwards compatibility - improved statistics

* support &tier=X in queries

* enfore increasing iterations on tiers

* tier 1 is always 1 iteration

* backfilling higher tiers on first data collection

* reversed anomaly bit

* set up to 5 tiers

* natural points should only be offered on tier 0, except a specific tier is selected

* do not allow more than 65535 points of tier0 to be aggregated on any tier

* Work only on actually activated tiers

* fix query interpolation

* fix query interpolation again

* fix lgtm finding

* Activate one tier for now

* backfilling of higher tiers using raw metrics from lower tiers

* fix for crash on start when storage tiers is increased from the default

* more statistics on exit

* fix bug that prevented higher tiers to get any values; added backfilling options

* fixed the statistics log line

* removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle

* fixed division by zero on zero points_wanted

* removed dead code

* Decide on the descr->type for the type of metric

* dont store metrics on unknown page types

* free db_metric_handle on sql based context queries

* Disable STORAGE_POINT value check in the exporting engine unit tests

* fix for db modes other than dbengine

* fix for aclk archived chart queries destroying db_metric_handles of valid rrddims

* fix left-over freez() instead of OWA freez on median queries

Co-authored-by: Costa Tsaousis <costa@netdata.cloud>
Co-authored-by: Vladimir Kobal <vlad@prokk.net>
2022-07-06 14:01:53 +03:00

131 lines
3.9 KiB
C

// SPDX-License-Identifier: GPL-3.0-or-later
#include "rrdr.h"
/*
static void rrdr_dump(RRDR *r)
{
long c, i;
RRDDIM *d;
fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
, d->id
, d->name
, (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
, (r->od[c] & RRDR_RESET)?"RESET ":""
, (r->od[c] & RRDR_DIMENSION_HIDDEN)?"HIDDEN ":""
, (r->od[c] & RRDR_DIMENSION_NONZERO)?"NONZERO ":""
);
}
if(r->rows <= 0) {
fprintf(stderr, "RRDR does not have any values in it.\n");
return;
}
fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
// for each line in the array
for(i = 0; i < r->rows ;i++) {
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_DIMENSION_FLAGS *co = &r->o[ i * r->d ];
// print the id and the timestamp of the line
fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
// for each dimension
for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely(!(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
if(co[c] & RRDR_EMPTY)
fprintf(stderr, "null ");
else
fprintf(stderr, NETDATA_DOUBLE_FORMAT " %s%s%s%s "
, cn[c]
, (co[c] & RRDR_EMPTY)?"E":" "
, (co[c] & RRDR_RESET)?"R":" "
, (co[c] & RRDR_DIMENSION_HIDDEN)?"H":" "
, (co[c] & RRDR_DIMENSION_NONZERO)?"N":" "
);
}
fprintf(stderr, "\n");
}
}
*/
inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
if(unlikely(!r)) return;
if(likely(r->st_locked_by_rrdr_create))
rrdset_unlock(r->st);
onewayalloc_freez(owa, r->t);
onewayalloc_freez(owa, r->v);
onewayalloc_freez(owa, r->o);
onewayalloc_freez(owa, r->od);
onewayalloc_freez(owa, r->ar);
onewayalloc_freez(owa, r);
}
RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
r->internal.owa = owa;
r->d = dimensions;
r->n = points;
r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(uint8_t));
r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
r->group = 1;
r->update_every = 1;
return r;
}
RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
if (unlikely(!st)) return NULL;
bool st_locked_by_rrdr_create = false;
if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
rrdset_rdlock(st);
st_locked_by_rrdr_create = true;
}
// count the number of dimensions
int dimensions = 0;
RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
RRDDIM *rd;
if (temp_rd) {
RRDDIM *t = temp_rd;
while (t) {
dimensions++;
t = t->next;
}
} else
rrddim_foreach_read(rd, st) dimensions++;
// create the rrdr
RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
r->st = st;
r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
// set the hidden flag on hidden dimensions
int c;
for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) {
if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
r->od[c] = RRDR_DIMENSION_HIDDEN;
else
r->od[c] = RRDR_DIMENSION_DEFAULT;
}
return r;
}