mirror of
https://github.com/netdata/netdata.git
synced 2025-04-24 05:13:08 +00:00
986a1abf68
* faster correlations * 4x times faster correlations * a little bit more help * 10x times faster metrics correlations * 6 digits precision; better comments * enabled metrics correlations by default * abstracted DIFFS_NUMBER to allow easily changing it * reworked the entire logic to have more accuracy and support a baseline that is power of two multiple of highlight * properly calculate shifts * even more improved version * added support for timeout; fixed another memory leak; skipped hidden dimensions * default timeout 1min * reduce memory even further * use dictionary for the list of charts and optimize locks * return 403 forbidden, when mc is not enabled * added query options * dont process zero dimensions * added volume method as an option to metric correlations ; now metric correlations can support multiple implementations * make sure we will never crash * spread results evenly for both kstwo and volume * fixed bug in query engine that was missing misaligned queries when a single point was requested from the db; improved comments; improved query flags * updated swagger and added sane defaults; query options are now supported, including anomaly-bit * added "raw" option to allow cross node correlations; added "group" option to allow different time aggregations; allowed calling metric correlations without any parameters; allowed calling metric correlations with relative timestamps; added timeout to volume method; properly handled timeout on ks2 method; json output now sends all parameters back - same for json_wrap; modified query engine to use present time for relative timestamps; modified "allow_past" to mean both past backwards and forwards * emulate the old behaviour about zero points * 100% accuracy against python ks_2samp(); now the default is volume and the default points are 500 * added config option to change default metric correlations method * removed work-arounds now that rrdlabels are merged
435 lines
14 KiB
C
435 lines
14 KiB
C
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
#include "web/api/web_api_v1.h"
|
|
|
|
static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
|
|
{
|
|
if (unlikely(!temp_rd))
|
|
return;
|
|
|
|
onewayalloc_freez(owa, (char *)temp_rd->id);
|
|
|
|
if (unlikely(archive_mode)) {
|
|
temp_rd->rrdset->counter--;
|
|
if (!temp_rd->rrdset->counter) {
|
|
onewayalloc_freez(owa, (char *)temp_rd->rrdset->name);
|
|
onewayalloc_freez(owa, temp_rd->rrdset->context);
|
|
onewayalloc_freez(owa, temp_rd->rrdset);
|
|
}
|
|
}
|
|
|
|
onewayalloc_freez(owa, temp_rd->state);
|
|
onewayalloc_freez(owa, temp_rd);
|
|
}
|
|
|
|
static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
|
|
{
|
|
if (unlikely(!temp_rd))
|
|
return;
|
|
|
|
RRDDIM *t;
|
|
while (temp_rd) {
|
|
t = temp_rd->next;
|
|
free_single_rrdrim(owa, temp_rd, archive_mode);
|
|
temp_rd = t;
|
|
}
|
|
}
|
|
|
|
void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list)
|
|
{
|
|
if (unlikely(!param_list || !*param_list))
|
|
return;
|
|
|
|
free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
|
|
onewayalloc_freez(owa, (*param_list));
|
|
*param_list = NULL;
|
|
}
|
|
|
|
void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested)
|
|
{
|
|
RRDDIM *temp_rd = context_param_list->rd;
|
|
RRDDIM *new_rd_list = NULL, *t;
|
|
int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE);
|
|
while (temp_rd) {
|
|
t = temp_rd->next;
|
|
RRDSET *st = temp_rd->rrdset;
|
|
time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
|
|
|
|
if (last_entry_t >= after_requested) {
|
|
temp_rd->next = new_rd_list;
|
|
new_rd_list = temp_rd;
|
|
} else
|
|
free_single_rrdrim(owa, temp_rd, is_archived);
|
|
temp_rd = t;
|
|
}
|
|
context_param_list->rd = new_rd_list;
|
|
};
|
|
|
|
void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st)
|
|
{
|
|
if (unlikely(!param_list || !st))
|
|
return;
|
|
|
|
if (unlikely(!(*param_list))) {
|
|
*param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
|
|
(*param_list)->first_entry_t = LONG_MAX;
|
|
(*param_list)->last_entry_t = 0;
|
|
(*param_list)->flags = CONTEXT_FLAGS_CONTEXT;
|
|
(*param_list)->rd = NULL;
|
|
}
|
|
|
|
RRDDIM *rd1;
|
|
st->last_accessed_time = now_realtime_sec();
|
|
rrdset_rdlock(st);
|
|
|
|
(*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st));
|
|
(*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
|
|
|
|
rrddim_foreach_read(rd1, st) {
|
|
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize);
|
|
rd->id = onewayalloc_strdupz(owa, rd1->id);
|
|
rd->name = onewayalloc_strdupz(owa, rd1->name);
|
|
rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
|
|
rd->next = (*param_list)->rd;
|
|
(*param_list)->rd = rd;
|
|
}
|
|
|
|
rrdset_unlock(st);
|
|
}
|
|
|
|
void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
|
|
rrdset2json(st, wb, NULL, NULL, 0);
|
|
}
|
|
|
|
void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) {
|
|
switch(format) {
|
|
case DATASOURCE_JSON:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
|
|
break;
|
|
|
|
case DATASOURCE_DATATABLE_JSON:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
|
|
break;
|
|
|
|
case DATASOURCE_DATATABLE_JSONP:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
|
|
break;
|
|
|
|
case DATASOURCE_JSONP:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
|
|
break;
|
|
|
|
case DATASOURCE_SSV:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
|
|
break;
|
|
|
|
case DATASOURCE_CSV:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
|
|
break;
|
|
|
|
case DATASOURCE_TSV:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
|
|
break;
|
|
|
|
case DATASOURCE_HTML:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
|
|
break;
|
|
|
|
case DATASOURCE_JS_ARRAY:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
|
|
break;
|
|
|
|
case DATASOURCE_SSV_COMMA:
|
|
buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
|
|
break;
|
|
|
|
default:
|
|
buffer_strcat(wb, "unknown");
|
|
break;
|
|
}
|
|
}
|
|
|
|
int rrdset2value_api_v1(
|
|
RRDSET *st
|
|
, BUFFER *wb
|
|
, calculated_number *n
|
|
, const char *dimensions
|
|
, long points
|
|
, long long after
|
|
, long long before
|
|
, int group_method
|
|
, long group_time
|
|
, uint32_t options
|
|
, time_t *db_after
|
|
, time_t *db_before
|
|
, size_t *db_points_read
|
|
, size_t *result_points_generated
|
|
, int *value_is_null
|
|
, int timeout
|
|
) {
|
|
int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
|
|
|
|
ONEWAYALLOC *owa = onewayalloc_create(0);
|
|
|
|
RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
|
|
|
|
if(!r) {
|
|
if(value_is_null) *value_is_null = 1;
|
|
ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
|
|
goto cleanup;
|
|
}
|
|
|
|
if(db_points_read)
|
|
*db_points_read += r->internal.db_points_read;
|
|
|
|
if(result_points_generated)
|
|
*result_points_generated += r->internal.result_points_generated;
|
|
|
|
if(rrdr_rows(r) == 0) {
|
|
if(db_after) *db_after = 0;
|
|
if(db_before) *db_before = 0;
|
|
if(value_is_null) *value_is_null = 1;
|
|
|
|
ret = HTTP_RESP_BAD_REQUEST;
|
|
goto cleanup;
|
|
}
|
|
|
|
if(wb) {
|
|
if (r->result_options & RRDR_RESULT_OPTION_RELATIVE)
|
|
buffer_no_cacheable(wb);
|
|
else if (r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
|
|
buffer_cacheable(wb);
|
|
}
|
|
|
|
if(db_after) *db_after = r->after;
|
|
if(db_before) *db_before = r->before;
|
|
|
|
long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
|
|
*n = rrdr2value(r, i, options, value_is_null, NULL);
|
|
ret = HTTP_RESP_OK;
|
|
|
|
cleanup:
|
|
if(r) rrdr_free(owa, r);
|
|
onewayalloc_destroy(owa);
|
|
return ret;
|
|
}
|
|
|
|
int rrdset2anything_api_v1(
|
|
ONEWAYALLOC *owa
|
|
, RRDSET *st
|
|
, QUERY_PARAMS *query_params
|
|
, BUFFER *dimensions
|
|
, uint32_t format
|
|
, long points
|
|
, long long after
|
|
, long long before
|
|
, int group_method
|
|
, long group_time
|
|
, uint32_t options
|
|
, time_t *latest_timestamp
|
|
)
|
|
{
|
|
BUFFER *wb = query_params->wb;
|
|
if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
|
|
st->last_accessed_time = now_realtime_sec();
|
|
|
|
RRDR *r = rrd2rrdr(
|
|
owa,
|
|
st,
|
|
points,
|
|
after,
|
|
before,
|
|
group_method,
|
|
group_time,
|
|
options,
|
|
dimensions ? buffer_tostring(dimensions) : NULL,
|
|
query_params->context_param_list,
|
|
query_params->timeout);
|
|
if(!r) {
|
|
buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
|
|
return HTTP_RESP_INTERNAL_SERVER_ERROR;
|
|
}
|
|
|
|
if (r->result_options & RRDR_RESULT_OPTION_CANCEL) {
|
|
rrdr_free(owa, r);
|
|
return HTTP_RESP_BACKEND_FETCH_FAILED;
|
|
}
|
|
|
|
if (st && st->state && st->state->is_ar_chart)
|
|
ml_process_rrdr(r, query_params->max_anomaly_rates);
|
|
|
|
RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
|
|
|
|
if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
|
|
buffer_no_cacheable(wb);
|
|
else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
|
|
buffer_cacheable(wb);
|
|
|
|
if(latest_timestamp && rrdr_rows(r) > 0)
|
|
*latest_timestamp = r->before;
|
|
|
|
switch(format) {
|
|
case DATASOURCE_SSV:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_PLAIN;
|
|
rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_SSV_COMMA:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_PLAIN;
|
|
rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_JS_ARRAY:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_CSV:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_PLAIN;
|
|
rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_CSV_MARKDOWN:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_PLAIN;
|
|
rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_CSV_JSON_ARRAY:
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
buffer_strcat(wb, "[\n");
|
|
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
|
|
buffer_strcat(wb, "\n]");
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
buffer_strcat(wb, "[\n");
|
|
rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
|
|
buffer_strcat(wb, "\n]");
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_TSV:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_PLAIN;
|
|
rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd);
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_HTML:
|
|
if(options & RRDR_OPTION_JSON_WRAP) {
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
|
|
buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
|
|
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
|
|
buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
|
|
rrdr_json_wrapper_end(r, wb, format, options, 1);
|
|
}
|
|
else {
|
|
wb->contenttype = CT_TEXT_HTML;
|
|
buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
|
|
rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "", temp_rd);
|
|
buffer_strcat(wb, "</table>\n</center>\n</html>\n");
|
|
}
|
|
break;
|
|
|
|
case DATASOURCE_DATATABLE_JSONP:
|
|
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
|
|
rrdr2json(r, wb, options, 1, query_params->context_param_list);
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
break;
|
|
|
|
case DATASOURCE_DATATABLE_JSON:
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
|
|
rrdr2json(r, wb, options, 1, query_params->context_param_list);
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
break;
|
|
|
|
case DATASOURCE_JSONP:
|
|
wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
|
|
rrdr2json(r, wb, options, 0, query_params->context_param_list);
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
break;
|
|
|
|
case DATASOURCE_JSON:
|
|
default:
|
|
wb->contenttype = CT_APPLICATION_JSON;
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
|
|
|
|
rrdr2json(r, wb, options, 0, query_params->context_param_list);
|
|
|
|
if(options & RRDR_OPTION_JSON_WRAP)
|
|
rrdr_json_wrapper_end(r, wb, format, options, 0);
|
|
break;
|
|
}
|
|
|
|
rrdr_free(owa, r);
|
|
return HTTP_RESP_OK;
|
|
}
|