mirror of
https://github.com/netdata/netdata.git
synced 2025-05-19 23:41:46 +00:00

* Tier part 1 * Tier part 2 * Tier part 3 * Tier part 4 * Tier part 5 * Fix some ML compilation errors * fix more conflicts * pass proper tier * move metric_uuid from state to RRDDIM * move aclk_live_status from state to RRDDIM * move ml_dimension from state to RRDDIM * abstracted the data collection interface * support flushing for mem db too * abstracted the query api * abstracted latest/oldest time per metric * cleanup * store_metric for tier1 * fix for store_metric * allow multiple tiers, more than 2 * state to tier * Change storage type in db. Query param to request min, max, sum or average * Store tier data correctly * Fix skipping tier page type * Add tier grouping in the tier * Fix to handle archived charts (part 1) * Temp fix for query granularity when requesting tier1 data * Fix parameters in the correct order and calculate the anomaly based on the anomaly count * Proper tiering grouping * Anomaly calculation based on anomaly count * force type checking on storage handles * update cmocka tests * fully dynamic number of storage tiers * fix static allocation * configure grouping for all tiers; disable tiers for unittest; disable statsd configuration for private charts mode * use default page dt using the tiering info * automatic selection of tier * fix for automatic selection of tier * working prototype of dynamic tier selection * automatic selection of tier done right (I hope) * ask for the proper tier value, based on the grouping function * fixes for unittests and load_metric_next() * fixes for lgtm findings * minor renames * add dbengine to page cache size setting * add dbengine to page cache with malloc * query engine optimized to loop as little are required based on the view_update_every * query engine grouping methods now do not assume a constant number of points per group and they allocate memory with OWA * report db points per tier in jsonwrap * query planer that switches database tiers on the fly to satisfy the query for the entire timeframe * dbegnine statistics and documentation (in progress) * calculate average point duration in db * handle single point pages the best we can * handle single point pages even better * Keep page type in the rrdeng_page_descr * updated doc * handle future backwards compatibility - improved statistics * support &tier=X in queries * enfore increasing iterations on tiers * tier 1 is always 1 iteration * backfilling higher tiers on first data collection * reversed anomaly bit * set up to 5 tiers * natural points should only be offered on tier 0, except a specific tier is selected * do not allow more than 65535 points of tier0 to be aggregated on any tier * Work only on actually activated tiers * fix query interpolation * fix query interpolation again * fix lgtm finding * Activate one tier for now * backfilling of higher tiers using raw metrics from lower tiers * fix for crash on start when storage tiers is increased from the default * more statistics on exit * fix bug that prevented higher tiers to get any values; added backfilling options * fixed the statistics log line * removed limit of 255 iterations per tier; moved the code of freezing rd->tiers[x]->db_metric_handle * fixed division by zero on zero points_wanted * removed dead code * Decide on the descr->type for the type of metric * dont store metrics on unknown page types * free db_metric_handle on sql based context queries * Disable STORAGE_POINT value check in the exporting engine unit tests * fix for db modes other than dbengine * fix for aclk archived chart queries destroying db_metric_handles of valid rrddims * fix left-over freez() instead of OWA freez on median queries Co-authored-by: Costa Tsaousis <costa@netdata.cloud> Co-authored-by: Vladimir Kobal <vlad@prokk.net>
131 lines
3.9 KiB
C
131 lines
3.9 KiB
C
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
#include "rrdr.h"
|
|
|
|
/*
|
|
static void rrdr_dump(RRDR *r)
|
|
{
|
|
long c, i;
|
|
RRDDIM *d;
|
|
|
|
fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
|
|
|
|
for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
|
|
fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
|
|
, d->id
|
|
, d->name
|
|
, (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
|
|
, (r->od[c] & RRDR_RESET)?"RESET ":""
|
|
, (r->od[c] & RRDR_DIMENSION_HIDDEN)?"HIDDEN ":""
|
|
, (r->od[c] & RRDR_DIMENSION_NONZERO)?"NONZERO ":""
|
|
);
|
|
}
|
|
|
|
if(r->rows <= 0) {
|
|
fprintf(stderr, "RRDR does not have any values in it.\n");
|
|
return;
|
|
}
|
|
|
|
fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
|
|
|
|
// for each line in the array
|
|
for(i = 0; i < r->rows ;i++) {
|
|
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
|
|
RRDR_DIMENSION_FLAGS *co = &r->o[ i * r->d ];
|
|
|
|
// print the id and the timestamp of the line
|
|
fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
|
|
|
|
// for each dimension
|
|
for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
|
|
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
|
|
if(unlikely(!(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
|
|
|
|
if(co[c] & RRDR_EMPTY)
|
|
fprintf(stderr, "null ");
|
|
else
|
|
fprintf(stderr, NETDATA_DOUBLE_FORMAT " %s%s%s%s "
|
|
, cn[c]
|
|
, (co[c] & RRDR_EMPTY)?"E":" "
|
|
, (co[c] & RRDR_RESET)?"R":" "
|
|
, (co[c] & RRDR_DIMENSION_HIDDEN)?"H":" "
|
|
, (co[c] & RRDR_DIMENSION_NONZERO)?"N":" "
|
|
);
|
|
}
|
|
|
|
fprintf(stderr, "\n");
|
|
}
|
|
}
|
|
*/
|
|
|
|
inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
|
|
if(unlikely(!r)) return;
|
|
|
|
if(likely(r->st_locked_by_rrdr_create))
|
|
rrdset_unlock(r->st);
|
|
|
|
onewayalloc_freez(owa, r->t);
|
|
onewayalloc_freez(owa, r->v);
|
|
onewayalloc_freez(owa, r->o);
|
|
onewayalloc_freez(owa, r->od);
|
|
onewayalloc_freez(owa, r->ar);
|
|
onewayalloc_freez(owa, r);
|
|
}
|
|
|
|
RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
|
|
RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
|
|
r->internal.owa = owa;
|
|
|
|
r->d = dimensions;
|
|
r->n = points;
|
|
|
|
r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
|
|
r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
|
|
r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS));
|
|
r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(uint8_t));
|
|
r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS));
|
|
|
|
r->group = 1;
|
|
r->update_every = 1;
|
|
|
|
return r;
|
|
}
|
|
|
|
RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
|
|
if (unlikely(!st)) return NULL;
|
|
|
|
bool st_locked_by_rrdr_create = false;
|
|
if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
|
|
rrdset_rdlock(st);
|
|
st_locked_by_rrdr_create = true;
|
|
}
|
|
|
|
// count the number of dimensions
|
|
int dimensions = 0;
|
|
RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
|
|
RRDDIM *rd;
|
|
if (temp_rd) {
|
|
RRDDIM *t = temp_rd;
|
|
while (t) {
|
|
dimensions++;
|
|
t = t->next;
|
|
}
|
|
} else
|
|
rrddim_foreach_read(rd, st) dimensions++;
|
|
|
|
// create the rrdr
|
|
RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
|
|
r->st = st;
|
|
r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
|
|
|
|
// set the hidden flag on hidden dimensions
|
|
int c;
|
|
for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) {
|
|
if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
|
|
r->od[c] = RRDR_DIMENSION_HIDDEN;
|
|
else
|
|
r->od[c] = RRDR_DIMENSION_DEFAULT;
|
|
}
|
|
|
|
return r;
|
|
}
|