archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-16 10:31:07 +00:00
Code Issues Projects Releases Wiki Activity

Trimmed-median, trimmed-mean and percentile ()

Browse source
This commit is contained in:
Costa Tsaousis 2022-08-05 12:50:11 +03:00 • committed by GitHub
parent 6a52383565
commit b863e5062e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 1546 additions and 51 deletions

4
CMakeLists.txt
View file

@ -814,12 +814,16 @@ set(API_PLUGIN_FILES
web/api/queries/sum/sum.h
web/api/queries/median/median.c
web/api/queries/median/median.h
web/api/queries/percentile/percentile.c
web/api/queries/percentile/percentile.h
web/api/queries/stddev/stddev.c
web/api/queries/stddev/stddev.h
web/api/queries/ses/ses.c
web/api/queries/ses/ses.h
web/api/queries/des/des.c
web/api/queries/des/des.h
web/api/queries/trimmed_mean/trimmed_mean.c
web/api/queries/trimmed_mean/trimmed_mean.h
web/api/queries/weights.c
web/api/queries/weights.h
web/api/formatters/rrd2json.c

4
Makefile.am
View file

@ -587,6 +587,10 @@ API_PLUGIN_FILES = \
web/api/queries/median/median.h \
web/api/queries/min/min.c \
web/api/queries/min/min.h \
web/api/queries/percentile/percentile.c \
web/api/queries/percentile/percentile.h \
web/api/queries/trimmed_mean/trimmed_mean.c \
web/api/queries/trimmed_mean/trimmed_mean.h \
web/api/queries/query.c \
web/api/queries/query.h \
web/api/queries/rrdr.c \

2
configure.ac
View file

@ -1748,9 +1748,11 @@ AC_CONFIG_FILES([
web/api/queries/max/Makefile
web/api/queries/median/Makefile
web/api/queries/min/Makefile
web/api/queries/percentile/Makefile
web/api/queries/ses/Makefile
web/api/queries/stddev/Makefile
web/api/queries/sum/Makefile
web/api/queries/trimmed_mean/Makefile
web/api/health/Makefile
web/gui/Makefile
web/gui/dashboard/Makefile

444
web/api/netdata-swagger.json
View file

@ -453,7 +453,35 @@
"ses",
"des",
"cv",
"countif"
"countif",
"percentile",
"percentile25",
"percentile50",
"percentile75",
"percentile80",
"percentile90",
"percentile95",
"percentile97",
"percentile98",
"percentile99",
"trimmed-mean",
"trimmed-mean1",
"trimmed-mean2",
"trimmed-mean3",
"trimmed-mean5",
"trimmed-mean10",
"trimmed-mean15",
"trimmed-mean20",
"trimmed-mean25",
"trimmed-median",
"trimmed-median1",
"trimmed-median2",
"trimmed-median3",
"trimmed-median5",
"trimmed-median10",
"trimmed-median15",
"trimmed-median20",
"trimmed-median25"
],
"default": "average"
}
@ -690,7 +718,39 @@
"median",
"stddev",
"sum",
"incremental-sum"
"incremental-sum",
"ses",
"des",
"cv",
"countif",
"percentile",
"percentile25",
"percentile50",
"percentile75",
"percentile80",
"percentile90",
"percentile95",
"percentile97",
"percentile98",
"percentile99",
"trimmed-mean",
"trimmed-mean1",
"trimmed-mean2",
"trimmed-mean3",
"trimmed-mean5",
"trimmed-mean10",
"trimmed-mean15",
"trimmed-mean20",
"trimmed-mean25",
"trimmed-median",
"trimmed-median1",
"trimmed-median2",
"trimmed-median3",
"trimmed-median5",
"trimmed-median10",
"trimmed-median15",
"trimmed-median20",
"trimmed-median25"
],
"default": "average"
}
@ -1362,7 +1422,7 @@
"/metric_correlations": {
"get": {
"summary": "Analyze all the metrics to find their correlations",
"description": "Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
"description": "THIS ENDPOINT IS OBSOLETE. Use the /weights endpoint. Given two time-windows (baseline, highlight), it goes through all the available metrics, querying both windows and tries to find how these two windows relate to each other. It supports multiple algorithms to do so. The result is a list of all metrics evaluated, weighted for 0.0 (the two windows are more different) to 1.0 (the two windows are similar). The algorithm adjusts automatically the baseline window to be a power of two multiple of the highlighted (1, 2, 4, 8, etc).",
"parameters": [
{
"name": "baseline_after",
@ -1499,7 +1559,35 @@
"ses",
"des",
"cv",
"countif"
"countif",
"percentile",
"percentile25",
"percentile50",
"percentile75",
"percentile80",
"percentile90",
"percentile95",
"percentile97",
"percentile98",
"percentile99",
"trimmed-mean",
"trimmed-mean1",
"trimmed-mean2",
"trimmed-mean3",
"trimmed-mean5",
"trimmed-mean10",
"trimmed-mean15",
"trimmed-mean20",
"trimmed-mean25",
"trimmed-median",
"trimmed-median1",
"trimmed-median2",
"trimmed-median3",
"trimmed-median5",
"trimmed-median10",
"trimmed-median15",
"trimmed-median20",
"trimmed-median25"
],
"default": "average"
}
@ -1540,6 +1628,236 @@
}
}
}
},
"/weights": {
"get": {
"summary": "Analyze all the metrics using an algorithm and score them accordingly",
"description": "This endpoint goes through all metrics and scores them according to an algorithm.",
"parameters": [
{
"name": "baseline_after",
"in": "query",
"description": "This parameter can either be an absolute timestamp specifying the starting point of baseline window, or a relative number of seconds (negative, relative to parameter baseline_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "number",
"format": "integer",
"default": -300
}
},
{
"name": "baseline_before",
"in": "query",
"description": "This parameter can either be an absolute timestamp specifying the ending point of the baseline window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds). This parameter is used in KS2 and VOLUME algorithms.",
"required": false,
"schema": {
"type": "number",
"format": "integer",
"default": -60
}
},
{
"name": "after",
"in": "query",
"description": "This parameter can either be an absolute timestamp specifying the starting point of highlighted window, or a relative number of seconds (negative, relative to parameter highlight_before). Netdata will assume it is a relative number if it is less that 3 years (in seconds).",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "number",
"format": "integer",
"default": -60
}
},
{
"name": "before",
"in": "query",
"description": "This parameter can either be an absolute timestamp specifying the ending point of the highlighted window, or a relative number of seconds (negative), relative to the last collected timestamp. Netdata will assume it is a relative number if it is less than 3 years (in seconds).",
"required": false,
"schema": {
"type": "number",
"format": "integer",
"default": 0
}
},
{
"name": "context",
"in": "query",
"description": "A simple pattern matching the contexts to evaluate.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "string"
}
},
{
"name": "points",
"in": "query",
"description": "The number of points to be evaluated for the highlighted window. The baseline window will be adjusted automatically to receive a proportional amount of points. This parameter is only used by the KS2 algorithm.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "number",
"format": "integer",
"default": 500
}
},
{
"name": "method",
"in": "query",
"description": "the algorithm to run",
"required": false,
"schema": {
"type": "string",
"enum": [
"ks2",
"volume",
"anomaly-rate"
],
"default": "anomaly-rate"
}
},
{
"name": "tier",
"in": "query",
"description": "Use the specified database tier",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "number",
"format": "integer"
}
},
{
"name": "timeout",
"in": "query",
"description": "Cancel the query if to takes more that this amount of milliseconds.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "number",
"format": "integer",
"default": 60000
}
},
{
"name": "options",
"in": "query",
"description": "Options that affect data generation.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "array",
"items": {
"type": "string",
"enum": [
"min2max",
"abs",
"absolute",
"absolute-sum",
"null2zero",
"percentage",
"unaligned",
"nonzero",
"anomaly-bit",
"raw"
]
},
"default": [
"null2zero",
"nonzero",
"unaligned"
]
}
},
{
"name": "group",
"in": "query",
"description": "The grouping method. If multiple collected values are to be grouped in order to return fewer points, this parameters defines the method of grouping. methods supported \"min\", \"max\", \"average\", \"sum\", \"incremental-sum\". \"max\" is actually calculated on the absolute value collected (so it works for both positive and negative dimensions to return the most extreme value in either direction).",
"required": true,
"allowEmptyValue": false,
"schema": {
"type": "string",
"enum": [
"min",
"max",
"average",
"median",
"stddev",
"sum",
"incremental-sum",
"ses",
"des",
"cv",
"countif",
"percentile",
"percentile25",
"percentile50",
"percentile75",
"percentile80",
"percentile90",
"percentile95",
"percentile97",
"percentile98",
"percentile99",
"trimmed-mean",
"trimmed-mean1",
"trimmed-mean2",
"trimmed-mean3",
"trimmed-mean5",
"trimmed-mean10",
"trimmed-mean15",
"trimmed-mean20",
"trimmed-mean25",
"trimmed-median",
"trimmed-median1",
"trimmed-median2",
"trimmed-median3",
"trimmed-median5",
"trimmed-median10",
"trimmed-median15",
"trimmed-median20",
"trimmed-median25"
],
"default": "average"
}
},
{
"name": "group_options",
"in": "query",
"description": "When the group function supports additional parameters, this field can be used to pass them to it. Currently only \"countif\" supports this.",
"required": false,
"allowEmptyValue": false,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "JSON object with weights for each context, chart and dimension.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/weights"
}
}
}
},
"400": {
"description": "The given parameters are invalid."
},
"403": {
"description": "metrics correlations are not enabled on this Netdata Agent."
},
"404": {
"description": "No charts could be found, or the method that correlated the metrics did not produce any result."
},
"504": {
"description": "Timeout - the query took too long and has been cancelled."
}
}
}
}
},
"servers": [
@ -2785,6 +3103,124 @@
}
}
}
},
"weights": {
"type": "object",
"properties": {
"after": {
"description": "the start time of the highlighted window",
"type": "integer"
},
"before": {
"description": "the end time of the highlighted window",
"type": "integer"
},
"duration": {
"description": "the duration of the highlighted window",
"type": "integer"
},
"points": {
"description": "the points of the highlighted window",
"type": "integer"
},
"baseline_after": {
"description": "the start time of the baseline window",
"type": "integer"
},
"baseline_before": {
"description": "the end time of the baseline window",
"type": "integer"
},
"baseline_duration": {
"description": "the duration of the baseline window",
"type": "integer"
},
"baseline_points": {
"description": "the points of the baseline window",
"type": "integer"
},
"group": {
"description": "the grouping method across time",
"type": "string"
},
"method": {
"description": "the correlation method used",
"type": "string"
},
"options": {
"description": "a comma separated list of the query options set",
"type": "string"
},
"correlated_dimensions": {
"description": "the number of dimensions returned in the result"
},
"total_dimensions_count": {
"description": "the total number of dimensions evaluated",
"type": "integer"
},
"statistics": {
"type": "object",
"properties": {
"query_time_ms": {
"type": "number"
},
"db_queries": {
"type": "integer"
},
"db_points_read": {
"type": "integer"
},
"query_result_points": {
"type": "integer"
},
"binary_searches": {
"type": "integer"
}
}
},
"contexts": {
"description": "A dictionary of weighted context objects.",
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/weighted_context"
}
}
}
},
"weighted_context": {
"type": "object",
"properties": {
"weight": {
"description": "The average weight of the context.",
"type": "number"
},
"charts": {
"description": "A dictionary of weighted chart objects.",
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/weighted_chart"
}
}
}
},
"weighted_chart": {
"type": "object",
"properties": {
"weight": {
"description": "The average weight of the context.",
"type": "number"
},
"dimensions": {
"description": "A dictionary of weighted dimensions.",
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/weighted_dimension"
}
}
}
},
"weighted_dimension": {
"type": "number"
}
}
}

169
web/api/netdata-swagger.yaml
View file

@ -365,6 +365,34 @@ paths:
- des
- cv
- countif
- percentile
- percentile25
- percentile50
- percentile75
- percentile80
- percentile90
- percentile95
- percentile97
- percentile98
- percentile99
- trimmed-mean
- trimmed-mean1
- trimmed-mean2
- trimmed-mean3
- trimmed-mean5
- trimmed-mean10
- trimmed-mean15
- trimmed-mean20
- trimmed-mean25
- trimmed-median
- trimmed-median1
- trimmed-median2
- trimmed-median3
- trimmed-median5
- trimmed-median10
- trimmed-median15
- trimmed-median20
- trimmed-median25
default: average
- name: group_options
in: query
@ -575,6 +603,38 @@ paths:
- stddev
- sum
- incremental-sum
- ses
- des
- cv
- countif
- percentile
- percentile25
- percentile50
- percentile75
- percentile80
- percentile90
- percentile95
- percentile97
- percentile98
- percentile99
- trimmed-mean
- trimmed-mean1
- trimmed-mean2
- trimmed-mean3
- trimmed-mean5
- trimmed-mean10
- trimmed-mean15
- trimmed-mean20
- trimmed-mean25
- trimmed-median
- trimmed-median1
- trimmed-median2
- trimmed-median3
- trimmed-median5
- trimmed-median10
- trimmed-median15
- trimmed-median20
- trimmed-median25
default: average
- name: options
in: query
@ -1238,6 +1298,34 @@ paths:
- des
- cv
- countif
- percentile
- percentile25
- percentile50
- percentile75
- percentile80
- percentile90
- percentile95
- percentile97
- percentile98
- percentile99
- trimmed-mean
- trimmed-mean1
- trimmed-mean2
- trimmed-mean3
- trimmed-mean5
- trimmed-mean10
- trimmed-mean15
- trimmed-mean20
- trimmed-mean25
- trimmed-median
- trimmed-median1
- trimmed-median2
- trimmed-median3
- trimmed-median5
- trimmed-median10
- trimmed-median15
- trimmed-median20
- trimmed-median25
default: average
- name: group_options
in: query
@ -1413,6 +1501,34 @@ paths:
- des
- cv
- countif
- percentile
- percentile25
- percentile50
- percentile75
- percentile80
- percentile90
- percentile95
- percentile97
- percentile98
- percentile99
- trimmed-mean
- trimmed-mean1
- trimmed-mean2
- trimmed-mean3
- trimmed-mean5
- trimmed-mean10
- trimmed-mean15
- trimmed-mean20
- trimmed-mean25
- trimmed-median
- trimmed-median1
- trimmed-median2
- trimmed-median3
- trimmed-median5
- trimmed-median10
- trimmed-median15
- trimmed-median20
- trimmed-median25
default: average
- name: group_options
in: query
@ -1428,7 +1544,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/weight"
$ref: "#/components/schemas/weights"
"400":
description: The given parameters are invalid.
"403":
@ -2373,7 +2489,7 @@ components:
type: number
dimension2-name:
type: number
weight:
weights:
type: object
properties:
after:
@ -2428,26 +2544,31 @@ components:
binary_searches:
type: integer
contexts:
description: A dictionary of weighted context objects.
type: object
description: An object containing context objects.
properties:
contextX:
type: object
properties:
charts:
type: object
properties:
chartX:
type: object
properties:
dimensions:
type: object
properties:
dimensionX:
type: number
weight:
description: The average chart weight
type: number
weight:
description: The average context weight
type: number
additionalProperties:
$ref: '#/components/schemas/weighted_context'
weighted_context:
type: object
properties:
weight:
description: The average weight of the context.
type: number
charts:
description: A dictionary of weighted chart objects.
type: object
additionalProperties:
$ref: '#/components/schemas/weighted_chart'
weighted_chart:
type: object
properties:
weight:
description: The average weight of the context.
type: number
dimensions:
description: A dictionary of weighted dimensions.
type: object
additionalProperties:
$ref: '#/components/schemas/weighted_dimension'
weighted_dimension:
type: number

2
web/api/queries/Makefile.am
View file

@ -12,8 +12,10 @@ SUBDIRS = \
min \
sum \
median \
percentile \
ses \
stddev \
trimmed_mean \
$(NULL)
dist_noinst_DATA = \

17
web/api/queries/median/README.md
View file

@ -13,6 +13,20 @@ The median is the value separating the higher half from the lower half of a data
`median` is not an accurate average. However, it eliminates all spikes, by sorting
all the values in a period, and selecting the value in the middle of the sorted array.
Netdata also supports `trimmed-median`, which trims a percentage of the smaller and bigger values prior to finding the
median. The following `trimmed-median` functions are defined:
- `trimmed-median1`
- `trimmed-median2`
- `trimmed-median3`
- `trimmed-median5`
- `trimmed-median10`
- `trimmed-median15`
- `trimmed-median20`
- `trimmed-median25`
The function `trimmed-median` is an alias for `trimmed-median5`.
## how to use
Use it in alarms like this:
@ -27,7 +41,8 @@ lookup: median -1m unaligned of my_dimension
`median` does not change the units. For example, if the chart units is `requests/sec`, the result
will be again expressed in the same units.
It can also be used in APIs and badges as `&group=median` in the URL.
It can also be used in APIs and badges as `&group=median` in the URL. Additionally, a percentage may be given with
`&group_options=` to trim all small and big values before finding the median.
## Examples

100
web/api/queries/median/median.c
View file

@ -2,28 +2,65 @@
#include "median.h"
// ----------------------------------------------------------------------------
// median
struct grouping_median {
size_t series_size;
size_t next_pos;
NETDATA_DOUBLE percent;
NETDATA_DOUBLE *series;
};
void grouping_create_median(RRDR *r, const char *options __maybe_unused) {
void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
long entries = r->group;
if(entries < 0) entries = 0;
if(entries < 10) entries = 10;
struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median));
g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
g->series_size = (size_t)entries;
g->percent = def;
if(options && *options) {
g->percent = str2ndd(options, NULL);
if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
if(g->percent < 0.0) g->percent = 0.0;
if(g->percent > 50.0) g->percent = 50.0;
}
g->percent = g->percent / 100.0;
r->internal.grouping_data = g;
}
void grouping_create_median(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 0.0);
}
void grouping_create_trimmed_median1(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 1.0);
}
void grouping_create_trimmed_median2(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 2.0);
}
void grouping_create_trimmed_median3(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 3.0);
}
void grouping_create_trimmed_median5(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 5.0);
}
void grouping_create_trimmed_median10(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 10.0);
}
void grouping_create_trimmed_median15(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 15.0);
}
void grouping_create_trimmed_median20(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 20.0);
}
void grouping_create_trimmed_median25(RRDR *r, const char *options) {
grouping_create_median_internal(r, options, 25.0);
}
// resets when switches dimensions
// so, clear everything to restart
void grouping_reset_median(RRDR *r) {
@ -46,37 +83,58 @@ void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) {
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
g->series_size *= 2;
}
else
g->series[g->next_pos++] = (NETDATA_DOUBLE)value;
g->series[g->next_pos++] = value;
}
NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data;
size_t available_slots = g->next_pos;
NETDATA_DOUBLE value;
if(unlikely(!g->next_pos)) {
if(unlikely(!available_slots)) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
else {
if(g->next_pos > 1) {
sort_series(g->series, g->next_pos);
value = (NETDATA_DOUBLE)median_on_sorted_series(g->series, g->next_pos);
}
else
value = (NETDATA_DOUBLE)g->series[0];
if(!netdata_double_isnumber(value)) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
//log_series_to_stderr(g->series, g->next_pos, value, "median");
else if(available_slots == 1) {
value = g->series[0];
}
else {
sort_series(g->series, available_slots);
size_t start_slot = 0;
size_t end_slot = available_slots - 1;
if(g->percent > 0.0) {
NETDATA_DOUBLE min = g->series[0];
NETDATA_DOUBLE max = g->series[available_slots - 1];
NETDATA_DOUBLE delta = (max - min) * g->percent;
NETDATA_DOUBLE wanted_min = min + delta;
NETDATA_DOUBLE wanted_max = max - delta;
for (start_slot = 0; start_slot < available_slots; start_slot++)
if (g->series[start_slot] >= wanted_min) break;
for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--)
if (g->series[end_slot] <= wanted_max) break;
}
if(start_slot == end_slot)
value = g->series[start_slot];
else
value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1);
}
if(unlikely(!netdata_double_isnumber(value))) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
//log_series_to_stderr(g->series, g->next_pos, value, "median");
g->next_pos = 0;
return value;
}

10
web/api/queries/median/median.h
View file

@ -6,7 +6,15 @@
#include "../query.h"
#include "../rrdr.h"
extern void grouping_create_median(RRDR *r, const char *options __maybe_unused);
extern void grouping_create_median(RRDR *r, const char *options);
extern void grouping_create_trimmed_median1(RRDR *r, const char *options);
extern void grouping_create_trimmed_median2(RRDR *r, const char *options);
extern void grouping_create_trimmed_median3(RRDR *r, const char *options);
extern void grouping_create_trimmed_median5(RRDR *r, const char *options);
extern void grouping_create_trimmed_median10(RRDR *r, const char *options);
extern void grouping_create_trimmed_median15(RRDR *r, const char *options);
extern void grouping_create_trimmed_median20(RRDR *r, const char *options);
extern void grouping_create_trimmed_median25(RRDR *r, const char *options);
extern void grouping_reset_median(RRDR *r);
extern void grouping_free_median(RRDR *r);
extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);

8
web/api/queries/percentile/Makefile.am Normal file
View file

@ -0,0 +1,8 @@
# SPDX-License-Identifier: GPL-3.0-or-later
AUTOMAKE_OPTIONS = subdir-objects
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
dist_noinst_DATA = \
README.md \
$(NULL)

58
web/api/queries/percentile/README.md Normal file
View file

@ -0,0 +1,58 @@
<!--
title: "Percentile"
description: "Use percentile in API queries and health entities to find the 'percentile' value from a sample, eliminating any unwanted spikes in the returned metrics."
custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/percentile/README.md
-->
# Percentile
The percentile is the average value of a series using only the smaller N percentile of the values.
(a population or a probability distribution).
Netdata applies linear interpolation on the last point, if the percentile requested does not give a round number of
points.
The following percentile aliases are defined:
- `percentile25`
- `percentile50`
- `percentile75`
- `percentile80`
- `percentile90`
- `percentile95`
- `percentile97`
- `percentile98`
- `percentile99`
The default `percentile` is an alias for `percentile95`.
Any percentile may be requested using the `group_options` query parameter.
## how to use
Use it in alarms like this:
```
alarm: my_alarm
on: my_chart
lookup: percentile95 -1m unaligned of my_dimension
warn: $this > 1000
```
`percentile` does not change the units. For example, if the chart units is `requests/sec`, the result
will be again expressed in the same units.
It can also be used in APIs and badges as `&group=percentile` in the URL and the additional parameter `group_options`
may be used to request any percentile (e.g. `&group=percentile&group_options=96`).
## Examples
Examining last 1 minute `successful` web server responses:
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=min&after=-60&label=min)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=average&after=-60&label=average)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=percentile95&after=-60&label=percentile95&value_color=orange)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=max&after=-60&label=max)
## References
- <https://en.wikipedia.org/wiki/Percentile>.

169
web/api/queries/percentile/percentile.c Normal file
View file

@ -0,0 +1,169 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "percentile.h"
// ----------------------------------------------------------------------------
// median
struct grouping_percentile {
size_t series_size;
size_t next_pos;
NETDATA_DOUBLE percent;
NETDATA_DOUBLE *series;
};
static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
long entries = r->group;
if(entries < 10) entries = 10;
struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile));
g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
g->series_size = (size_t)entries;
g->percent = def;
if(options && *options) {
g->percent = str2ndd(options, NULL);
if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
if(g->percent < 0.0) g->percent = 0.0;
if(g->percent > 100.0) g->percent = 100.0;
}
g->percent = g->percent / 100.0;
r->internal.grouping_data = g;
}
void grouping_create_percentile25(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 25.0);
}
void grouping_create_percentile50(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 50.0);
}
void grouping_create_percentile75(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 75.0);
}
void grouping_create_percentile80(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 80.0);
}
void grouping_create_percentile90(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 90.0);
}
void grouping_create_percentile95(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 95.0);
}
void grouping_create_percentile97(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 97.0);
}
void grouping_create_percentile98(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 98.0);
}
void grouping_create_percentile99(RRDR *r, const char *options) {
grouping_create_percentile_internal(r, options, 99.0);
}
// resets when switches dimensions
// so, clear everything to restart
void grouping_reset_percentile(RRDR *r) {
struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
g->next_pos = 0;
}
void grouping_free_percentile(RRDR *r) {
struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
if(g) onewayalloc_freez(r->internal.owa, g->series);
onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
r->internal.grouping_data = NULL;
}
void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
if(unlikely(g->next_pos >= g->series_size)) {
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
g->series_size *= 2;
}
g->series[g->next_pos++] = value;
}
NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data;
NETDATA_DOUBLE value;
size_t available_slots = g->next_pos;
if(unlikely(!available_slots)) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
else if(available_slots == 1) {
value = g->series[0];
}
else {
sort_series(g->series, available_slots);
NETDATA_DOUBLE min = g->series[0];
NETDATA_DOUBLE max = g->series[available_slots - 1];
if (min != max) {
size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
if(!slots_to_use) slots_to_use = 1;
NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
NETDATA_DOUBLE percent_interpolation_slot = 0.0;
NETDATA_DOUBLE percent_last_slot = 0.0;
if(percent_delta > 0.0) {
NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
percent_interpolation_slot = percent_delta / percent_1slot;
percent_last_slot = 1 - percent_interpolation_slot;
}
int start_slot, stop_slot, step, last_slot, interpolation_slot;
if(min >= 0.0 && max >= 0.0) {
start_slot = 0;
stop_slot = start_slot + (int)slots_to_use;
last_slot = stop_slot - 1;
interpolation_slot = stop_slot;
step = 1;
}
else {
start_slot = (int)available_slots - 1;
stop_slot = start_slot - (int)slots_to_use;
last_slot = stop_slot + 1;
interpolation_slot = stop_slot;
step = -1;
}
value = 0.0;
for(int slot = start_slot; slot != stop_slot ; slot += step)
value += g->series[slot];
size_t counted = slots_to_use;
if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
value += g->series[interpolation_slot] * percent_interpolation_slot;
value += g->series[last_slot] * percent_last_slot;
counted++;
}
value = value / (NETDATA_DOUBLE)counted;
}
else
value = min;
}
if(unlikely(!netdata_double_isnumber(value))) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
//log_series_to_stderr(g->series, g->next_pos, value, "percentile");
g->next_pos = 0;
return value;
}

23
web/api/queries/percentile/percentile.h Normal file
View file

@ -0,0 +1,23 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_API_QUERIES_PERCENTILE_H
#define NETDATA_API_QUERIES_PERCENTILE_H
#include "../query.h"
#include "../rrdr.h"
extern void grouping_create_percentile25(RRDR *r, const char *options);
extern void grouping_create_percentile50(RRDR *r, const char *options);
extern void grouping_create_percentile75(RRDR *r, const char *options);
extern void grouping_create_percentile80(RRDR *r, const char *options);
extern void grouping_create_percentile90(RRDR *r, const char *options);
extern void grouping_create_percentile95(RRDR *r, const char *options);
extern void grouping_create_percentile97(RRDR *r, const char *options);
extern void grouping_create_percentile98(RRDR *r, const char *options);
extern void grouping_create_percentile99(RRDR *r, const char *options );
extern void grouping_reset_percentile(RRDR *r);
extern void grouping_free_percentile(RRDR *r);
extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_PERCENTILE_H

310
web/api/queries/query.c
View file

@ -14,6 +14,8 @@
#include "stddev/stddev.h"
#include "ses/ses.h"
#include "des/des.h"
#include "percentile/percentile.h"
#include "trimmed_mean/trimmed_mean.h"
// ----------------------------------------------------------------------------
@ -74,6 +76,105 @@ static struct {
.flush = grouping_flush_average,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean1",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN1,
.init = NULL,
.create= grouping_create_trimmed_mean1,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean2",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN2,
.init = NULL,
.create= grouping_create_trimmed_mean2,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean3",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN3,
.init = NULL,
.create= grouping_create_trimmed_mean3,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean5",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN5,
.init = NULL,
.create= grouping_create_trimmed_mean5,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean10",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN10,
.init = NULL,
.create= grouping_create_trimmed_mean10,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean15",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN15,
.init = NULL,
.create= grouping_create_trimmed_mean15,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean20",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN20,
.init = NULL,
.create= grouping_create_trimmed_mean20,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean25",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN25,
.init = NULL,
.create= grouping_create_trimmed_mean25,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-mean",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEAN5,
.init = NULL,
.create= grouping_create_trimmed_mean5,
.reset = grouping_reset_trimmed_mean,
.free = grouping_free_trimmed_mean,
.add = grouping_add_trimmed_mean,
.flush = grouping_flush_trimmed_mean,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "incremental_sum",
.hash = 0,
.value = RRDR_GROUPING_INCREMENTAL_SUM,
@ -107,6 +208,215 @@ static struct {
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median1",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN1,
.init = NULL,
.create= grouping_create_trimmed_median1,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median2",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN2,
.init = NULL,
.create= grouping_create_trimmed_median2,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median3",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN3,
.init = NULL,
.create= grouping_create_trimmed_median3,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median5",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN5,
.init = NULL,
.create= grouping_create_trimmed_median5,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median10",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN10,
.init = NULL,
.create= grouping_create_trimmed_median10,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median15",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN15,
.init = NULL,
.create= grouping_create_trimmed_median15,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median20",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN20,
.init = NULL,
.create= grouping_create_trimmed_median20,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median25",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN25,
.init = NULL,
.create= grouping_create_trimmed_median25,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "trimmed-median",
.hash = 0,
.value = RRDR_GROUPING_TRIMMED_MEDIAN5,
.init = NULL,
.create= grouping_create_trimmed_median5,
.reset = grouping_reset_median,
.free = grouping_free_median,
.add = grouping_add_median,
.flush = grouping_flush_median,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile25",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE25,
.init = NULL,
.create= grouping_create_percentile25,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile50",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE50,
.init = NULL,
.create= grouping_create_percentile50,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile75",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE75,
.init = NULL,
.create= grouping_create_percentile75,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile80",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE80,
.init = NULL,
.create= grouping_create_percentile80,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile90",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE90,
.init = NULL,
.create= grouping_create_percentile90,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile95",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE95,
.init = NULL,
.create= grouping_create_percentile95,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile97",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE97,
.init = NULL,
.create= grouping_create_percentile97,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile98",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE98,
.init = NULL,
.create= grouping_create_percentile98,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile99",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE99,
.init = NULL,
.create= grouping_create_percentile99,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "percentile",
.hash = 0,
.value = RRDR_GROUPING_PERCENTILE95,
.init = NULL,
.create= grouping_create_percentile95,
.reset = grouping_reset_percentile,
.free = grouping_free_percentile,
.add = grouping_add_percentile,
.flush = grouping_flush_percentile,
.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE
},
{.name = "min",
.hash = 0,
.value = RRDR_GROUPING_MIN,

25
web/api/queries/query.h
View file

@ -14,7 +14,32 @@ typedef enum rrdr_grouping {
RRDR_GROUPING_MAX,
RRDR_GROUPING_SUM,
RRDR_GROUPING_INCREMENTAL_SUM,
RRDR_GROUPING_TRIMMED_MEAN1,
RRDR_GROUPING_TRIMMED_MEAN2,
RRDR_GROUPING_TRIMMED_MEAN3,
RRDR_GROUPING_TRIMMED_MEAN5,
RRDR_GROUPING_TRIMMED_MEAN10,
RRDR_GROUPING_TRIMMED_MEAN15,
RRDR_GROUPING_TRIMMED_MEAN20,
RRDR_GROUPING_TRIMMED_MEAN25,
RRDR_GROUPING_MEDIAN,
RRDR_GROUPING_TRIMMED_MEDIAN1,
RRDR_GROUPING_TRIMMED_MEDIAN2,
RRDR_GROUPING_TRIMMED_MEDIAN3,
RRDR_GROUPING_TRIMMED_MEDIAN5,
RRDR_GROUPING_TRIMMED_MEDIAN10,
RRDR_GROUPING_TRIMMED_MEDIAN15,
RRDR_GROUPING_TRIMMED_MEDIAN20,
RRDR_GROUPING_TRIMMED_MEDIAN25,
RRDR_GROUPING_PERCENTILE25,
RRDR_GROUPING_PERCENTILE50,
RRDR_GROUPING_PERCENTILE75,
RRDR_GROUPING_PERCENTILE80,
RRDR_GROUPING_PERCENTILE90,
RRDR_GROUPING_PERCENTILE95,
RRDR_GROUPING_PERCENTILE97,
RRDR_GROUPING_PERCENTILE98,
RRDR_GROUPING_PERCENTILE99,
RRDR_GROUPING_STDDEV,
RRDR_GROUPING_CV,
RRDR_GROUPING_SES,

8
web/api/queries/trimmed_mean/Makefile.am Normal file
View file

@ -0,0 +1,8 @@
# SPDX-License-Identifier: GPL-3.0-or-later
AUTOMAKE_OPTIONS = subdir-objects
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
dist_noinst_DATA = \
README.md \
$(NULL)

56
web/api/queries/trimmed_mean/README.md Normal file
View file

@ -0,0 +1,56 @@
<!--
title: "Trimmed Mean"
description: "Use trimmed-mean in API queries and health entities to find the average value from a sample, eliminating any unwanted spikes in the returned metrics."
custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/trimmed_mean/README.md
-->
# Trimmed Mean
The trimmed mean is the average value of a series excluding the smallest and biggest points.
Netdata applies linear interpolation on the last point, if the percentage requested to be excluded does not give a
round number of points.
The following percentile aliases are defined:
- `trimmed-mean1`
- `trimmed-mean2`
- `trimmed-mean3`
- `trimmed-mean5`
- `trimmed-mean10`
- `trimmed-mean15`
- `trimmed-mean20`
- `trimmed-mean25`
The default `trimmed-mean` is an alias for `trimmed-mean5`.
Any percentage may be requested using the `group_options` query parameter.
## how to use
Use it in alarms like this:
```
alarm: my_alarm
on: my_chart
lookup: trimmed-mean5 -1m unaligned of my_dimension
warn: $this > 1000
```
`trimmed-mean` does not change the units. For example, if the chart units is `requests/sec`, the result
will be again expressed in the same units.
It can also be used in APIs and badges as `&group=trimmed-mean` in the URL and the additional parameter `group_options`
may be used to request any percentage (e.g. `&group=trimmed-mean&group_options=29`).
## Examples
Examining last 1 minute `successful` web server responses:
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=min&after=-60&label=min)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=average&after=-60&label=average)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=trimmed-mean5&after=-60&label=trimmed-mean5&value_color=orange)
- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=max&after=-60&label=max)
## References
- <https://en.wikipedia.org/wiki/Truncated_mean>.

166
web/api/queries/trimmed_mean/trimmed_mean.c Normal file
View file

@ -0,0 +1,166 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "trimmed_mean.h"
// ----------------------------------------------------------------------------
// median
struct grouping_trimmed_mean {
size_t series_size;
size_t next_pos;
NETDATA_DOUBLE percent;
NETDATA_DOUBLE *series;
};
static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) {
long entries = r->group;
if(entries < 10) entries = 10;
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean));
g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE));
g->series_size = (size_t)entries;
g->percent = def;
if(options && *options) {
g->percent = str2ndd(options, NULL);
if(!netdata_double_isnumber(g->percent)) g->percent = 0.0;
if(g->percent < 0.0) g->percent = 0.0;
if(g->percent > 50.0) g->percent = 50.0;
}
g->percent = 1.0 - ((g->percent / 100.0) * 2.0);
r->internal.grouping_data = g;
}
void grouping_create_trimmed_mean1(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 1.0);
}
void grouping_create_trimmed_mean2(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 2.0);
}
void grouping_create_trimmed_mean3(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 3.0);
}
void grouping_create_trimmed_mean5(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 5.0);
}
void grouping_create_trimmed_mean10(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 10.0);
}
void grouping_create_trimmed_mean15(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 15.0);
}
void grouping_create_trimmed_mean20(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 20.0);
}
void grouping_create_trimmed_mean25(RRDR *r, const char *options) {
grouping_create_trimmed_mean_internal(r, options, 25.0);
}
// resets when switches dimensions
// so, clear everything to restart
void grouping_reset_trimmed_mean(RRDR *r) {
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
g->next_pos = 0;
}
void grouping_free_trimmed_mean(RRDR *r) {
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
if(g) onewayalloc_freez(r->internal.owa, g->series);
onewayalloc_freez(r->internal.owa, r->internal.grouping_data);
r->internal.grouping_data = NULL;
}
void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
if(unlikely(g->next_pos >= g->series_size)) {
g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE));
g->series_size *= 2;
}
g->series[g->next_pos++] = value;
}
NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data;
NETDATA_DOUBLE value;
size_t available_slots = g->next_pos;
if(unlikely(!available_slots)) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
else if(available_slots == 1) {
value = g->series[0];
}
else {
sort_series(g->series, available_slots);
NETDATA_DOUBLE min = g->series[0];
NETDATA_DOUBLE max = g->series[available_slots - 1];
if (min != max) {
size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent);
if(!slots_to_use) slots_to_use = 1;
NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots;
NETDATA_DOUBLE percent_delta = g->percent - percent_to_use;
NETDATA_DOUBLE percent_interpolation_slot = 0.0;
NETDATA_DOUBLE percent_last_slot = 0.0;
if(percent_delta > 0.0) {
NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots;
NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use;
percent_interpolation_slot = percent_delta / percent_1slot;
percent_last_slot = 1 - percent_interpolation_slot;
}
int start_slot, stop_slot, step, last_slot, interpolation_slot;
if(min >= 0.0 && max >= 0.0) {
start_slot = (int)((available_slots - slots_to_use) / 2);
stop_slot = start_slot + (int)slots_to_use;
last_slot = stop_slot - 1;
interpolation_slot = stop_slot;
step = 1;
}
else {
start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2);
stop_slot = start_slot - (int)slots_to_use;
last_slot = stop_slot + 1;
interpolation_slot = stop_slot;
step = -1;
}
value = 0.0;
for(int slot = start_slot; slot != stop_slot ; slot += step)
value += g->series[slot];
size_t counted = slots_to_use;
if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) {
value += g->series[interpolation_slot] * percent_interpolation_slot;
value += g->series[last_slot] * percent_last_slot;
counted++;
}
value = value / (NETDATA_DOUBLE)counted;
}
else
value = min;
}
if(unlikely(!netdata_double_isnumber(value))) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
//log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean");
g->next_pos = 0;
return value;
}

22
web/api/queries/trimmed_mean/trimmed_mean.h Normal file
View file

@ -0,0 +1,22 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_API_QUERIES_TRIMMED_MEAN_H
#define NETDATA_API_QUERIES_TRIMMED_MEAN_H
#include "../query.h"
#include "../rrdr.h"
extern void grouping_create_trimmed_mean1(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean2(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean3(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean5(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean10(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean15(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean20(RRDR *r, const char *options);
extern void grouping_create_trimmed_mean25(RRDR *r, const char *options);
extern void grouping_reset_trimmed_mean(RRDR *r);
extern void grouping_free_trimmed_mean(RRDR *r);
extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H