mirror of
https://github.com/netdata/netdata.git
synced 2025-04-27 06:10:43 +00:00
Use chart labels to filter alerts (#14982)
* use chart labels to filter alerts * add entry to readme * support chart_label=val val2 val3 * docs updates * more docs * use rc not rt
This commit is contained in:
Emmanuel Vasilakis
GitHub
parent
128fe42762
commit
10cad04d2d
7 changed files with 148 additions and 2 deletions
|
|
@ -369,6 +369,10 @@ static inline bool rrdcalc_check_if_it_matches_rrdset(RRDCALC *rc, RRDSET *st) {
|
|||
st->rrdhost->rrdlabels, rc->host_labels_pattern, '=', NULL))
|
||||
return false;
|
||||
|
||||
if (st->rrdlabels && rc->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(
|
||||
st->rrdlabels, rc->chart_labels_pattern, '=', NULL))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -605,11 +609,13 @@ static void rrdcalc_free_internals(RRDCALC *rc) {
|
|||
string_freez(rc->host_labels);
|
||||
string_freez(rc->module_match);
|
||||
string_freez(rc->plugin_match);
|
||||
string_freez(rc->chart_labels);
|
||||
|
||||
simple_pattern_free(rc->foreach_dimension_pattern);
|
||||
simple_pattern_free(rc->host_labels_pattern);
|
||||
simple_pattern_free(rc->module_pattern);
|
||||
simple_pattern_free(rc->plugin_pattern);
|
||||
simple_pattern_free(rc->chart_labels_pattern);
|
||||
}
|
||||
|
||||
static void rrdcalc_rrdhost_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdhost __maybe_unused) {
|
||||
|
|
|
|||
|
|
@ -109,6 +109,9 @@ struct rrdcalc {
|
|||
STRING *host_labels; // the label read from an alarm file
|
||||
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
|
||||
|
||||
STRING *chart_labels; // the chart label read from an alarm file
|
||||
SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// runtime information
|
||||
|
||||
|
|
@ -168,6 +171,7 @@ struct rrdcalc {
|
|||
#define rrdcalc_dimensions(rc) string2str((rc)->dimensions)
|
||||
#define rrdcalc_foreachdim(rc) string2str((rc)->foreach_dimension)
|
||||
#define rrdcalc_host_labels(rc) string2str((rc)->host_labels)
|
||||
#define rrdcalc_chart_labels(rc) string2str((rc)->chart_labels)
|
||||
|
||||
#define foreach_rrdcalc_in_rrdhost_read(host, rc) \
|
||||
dfe_start_read((host)->rrdcalc_root_index, rc) \
|
||||
|
|
@ -206,6 +210,7 @@ struct alert_config {
|
|||
STRING *options;
|
||||
STRING *repeat;
|
||||
STRING *host_labels;
|
||||
STRING *chart_labels;
|
||||
|
||||
STRING *p_db_lookup_dimensions;
|
||||
STRING *p_db_lookup_method;
|
||||
|
|
|
|||
|
|
@ -51,6 +51,11 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR
|
|||
'=', NULL))
|
||||
return false;
|
||||
|
||||
if(st->rrdlabels && rt->chart_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdlabels,
|
||||
rt->chart_labels_pattern,
|
||||
'=', NULL))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -120,8 +125,10 @@ static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) {
|
|||
string_freez(rt->dimensions);
|
||||
string_freez(rt->foreach_dimension);
|
||||
string_freez(rt->host_labels);
|
||||
string_freez(rt->chart_labels);
|
||||
simple_pattern_free(rt->foreach_dimension_pattern);
|
||||
simple_pattern_free(rt->host_labels_pattern);
|
||||
simple_pattern_free(rt->chart_labels_pattern);
|
||||
}
|
||||
|
||||
void rrdcalctemplate_free_unused_rrdcalctemplate_loaded_from_config(RRDCALCTEMPLATE *rt) {
|
||||
|
|
|
|||
|
|
@ -74,6 +74,9 @@ struct rrdcalctemplate {
|
|||
STRING *host_labels; // the label read from an alarm file
|
||||
SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
|
||||
|
||||
STRING *chart_labels; // the chart label read from an alarm file
|
||||
SIMPLE_PATTERN *chart_labels_pattern; // the simple pattern of chart labels
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// expressions related to the alarm
|
||||
|
||||
|
|
@ -107,6 +110,7 @@ struct rrdcalctemplate {
|
|||
#define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)
|
||||
#define rrdcalctemplate_foreachdim(rt) string2str((rt)->foreach_dimension)
|
||||
#define rrdcalctemplate_host_labels(rt) string2str((rt)->host_labels)
|
||||
#define rrdcalctemplate_chart_labels(rt) string2str((rt)->chart_labels)
|
||||
|
||||
#define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after)
|
||||
|
||||
|
|
|
|||
|
|
@ -1070,6 +1070,7 @@ int alert_hash_and_store_config(
|
|||
DIGEST_ALERT_CONFIG_VAL(cfg->options);
|
||||
DIGEST_ALERT_CONFIG_VAL(cfg->repeat);
|
||||
DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);
|
||||
DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);
|
||||
|
||||
EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
|
||||
EVP_MD_CTX_destroy(evpctx);
|
||||
|
|
|
|||
|
|
@ -241,7 +241,8 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation
|
|||
| [`delay`](#alarm-line-delay) | no | Optional hysteresis settings to prevent floods of notifications. |
|
||||
| [`repeat`](#alarm-line-repeat) | no | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode. |
|
||||
| [`options`](#alarm-line-options) | no | Add an option to not clear alarms. |
|
||||
| [`host labels`](#alarm-line-host-labels) | no | List of labels present on a host. |
|
||||
| [`host labels`](#alarm-line-host-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. |
|
||||
| [`chart labels`](#alarm-line-chart-labels) | no | Restrict an alarm or template to a list of matching labels present on a host. |
|
||||
| [`info`](#alarm-line-info) | no | A brief description of the alarm. |
|
||||
|
||||
The `alarm` or `template` line must be the first line of any entity.
|
||||
|
|
@ -446,6 +447,9 @@ For example, you can create a template on the `disk.io` context, but filter it t
|
|||
families: sda sdb
|
||||
```
|
||||
|
||||
Please note that the use of the `families` filter is planned to be deprecated in upcoming Netdata releases.
|
||||
Please use [`chart labels`](#alarm-line-chart-labels) instead.
|
||||
|
||||
#### Alarm line `lookup`
|
||||
|
||||
This line makes a database lookup to find a value. This result of this lookup is available as `$this`.
|
||||
|
|
@ -696,6 +700,35 @@ host labels: installed = 201*
|
|||
|
||||
See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
|
||||
|
||||
#### Alarm line `chart labels`
|
||||
|
||||
Similar to host labels, the `chart labels` key can be used to filter if an alarm will load or not for a specific chart, based on
|
||||
whether these chart labels match or not.
|
||||
|
||||
The list of chart labels present on each chart can be obtained from http://localhost:19999/api/v1/charts?all
|
||||
|
||||
For example, each `disk_space` chart defines a chart label called `mount_point` with each instance of this chart having
|
||||
a value there of which mount point it monitors.
|
||||
|
||||
If you have an e.g. external disk mounted on `/mnt/disk1` and you don't wish any related disk space alerts running for
|
||||
it (but you do for all other mount points), you can add the following to the alert's configuration:
|
||||
|
||||
```yaml
|
||||
chart labels: mount_point=!/mnt/disk1 *`
|
||||
```
|
||||
|
||||
The `chart labels` is a space-separated list that accepts simple patterns. If you use multiple different chart labels,
|
||||
then the result is an OR between them. i.e. the following:
|
||||
|
||||
```yaml
|
||||
chart labels: mount_point=/mnt/disk1 device=sda`
|
||||
```
|
||||
|
||||
Will create the alert if the `mount_point` is `/mnt/disk1` or the `device` is `sda`. Furthermore, if a chart label name
|
||||
is specified that does not exist in the chart, the chart won't be matched.
|
||||
|
||||
See our [simple patterns docs](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) for more examples.
|
||||
|
||||
#### Alarm line `info`
|
||||
|
||||
The info field can contain a small piece of text describing the alarm or template. This will be rendered in
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
#define HEALTH_REPEAT_KEY "repeat"
|
||||
#define HEALTH_HOST_LABEL_KEY "host labels"
|
||||
#define HEALTH_FOREACH_KEY "foreach"
|
||||
#define HEALTH_CHART_LABEL_KEY "chart labels"
|
||||
|
||||
static inline int health_parse_delay(
|
||||
size_t line, const char *filename, char *string,
|
||||
|
|
@ -192,6 +193,49 @@ static inline int isvariableterm(const char s) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
// If needed, add a prefix key to all possible values in the range
|
||||
static inline char *health_config_add_key_to_values(char *value) {
|
||||
BUFFER *wb = buffer_create(HEALTH_CONF_MAX_LINE + 1, NULL);
|
||||
char key[HEALTH_CONF_MAX_LINE + 1];
|
||||
char data[HEALTH_CONF_MAX_LINE + 1];
|
||||
|
||||
char *s = value;
|
||||
size_t i = 0;
|
||||
|
||||
while(*s) {
|
||||
if (*s == '=') {
|
||||
//hold the key
|
||||
data[i]='\0';
|
||||
strncpyz(key, data, HEALTH_CONF_MAX_LINE);
|
||||
i=0;
|
||||
} else if (*s == ' ') {
|
||||
data[i]='\0';
|
||||
if (data[0]=='!')
|
||||
buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
|
||||
else
|
||||
buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
|
||||
i=0;
|
||||
} else {
|
||||
data[i++] = *s;
|
||||
}
|
||||
s++;
|
||||
}
|
||||
|
||||
data[i]='\0';
|
||||
if (data[0]) {
|
||||
if (data[0]=='!')
|
||||
buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "!%s=%s ", key, data + 1);
|
||||
else
|
||||
buffer_snprintf(wb, HEALTH_CONF_MAX_LINE, "%s=%s ", key, data);
|
||||
}
|
||||
|
||||
char *final = mallocz(HEALTH_CONF_MAX_LINE + 1);
|
||||
strncpyz(final, buffer_tostring(wb), HEALTH_CONF_MAX_LINE);
|
||||
buffer_free(wb);
|
||||
|
||||
return final;
|
||||
}
|
||||
|
||||
static inline void parse_variables_and_store_in_health_rrdvars(char *value, size_t len) {
|
||||
const char *s = value;
|
||||
char buffer[RRDVAR_MAX_LENGTH];
|
||||
|
|
@ -453,6 +497,7 @@ static inline void alert_config_free(struct alert_config *cfg)
|
|||
string_freez(cfg->host_labels);
|
||||
string_freez(cfg->p_db_lookup_dimensions);
|
||||
string_freez(cfg->p_db_lookup_method);
|
||||
string_freez(cfg->chart_labels);
|
||||
freez(cfg);
|
||||
}
|
||||
|
||||
|
|
@ -489,7 +534,8 @@ static int health_readfile(const char *filename, void *data) {
|
|||
hash_delay = 0,
|
||||
hash_options = 0,
|
||||
hash_repeat = 0,
|
||||
hash_host_label = 0;
|
||||
hash_host_label = 0,
|
||||
hash_chart_label = 0;
|
||||
|
||||
char buffer[HEALTH_CONF_MAX_LINE + 1];
|
||||
|
||||
|
|
@ -521,6 +567,7 @@ static int health_readfile(const char *filename, void *data) {
|
|||
hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
|
||||
hash_repeat = simple_uhash(HEALTH_REPEAT_KEY);
|
||||
hash_host_label = simple_uhash(HEALTH_HOST_LABEL_KEY);
|
||||
hash_chart_label = simple_uhash(HEALTH_CHART_LABEL_KEY);
|
||||
}
|
||||
|
||||
FILE *fp = fopen(filename, "r");
|
||||
|
|
@ -937,6 +984,27 @@ static int health_readfile(const char *filename, void *data) {
|
|||
rc->module_match = string_strdupz(value);
|
||||
rc->module_pattern = simple_pattern_create(rrdcalc_module_match(rc), NULL, SIMPLE_PATTERN_EXACT, true);
|
||||
}
|
||||
else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
|
||||
alert_cfg->chart_labels = string_strdupz(value);
|
||||
if(rc->chart_labels) {
|
||||
if(strcmp(rrdcalc_chart_labels(rc), value) != 0)
|
||||
error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
|
||||
line, filename, rrdcalc_name(rc), key, value, value);
|
||||
|
||||
string_freez(rc->chart_labels);
|
||||
simple_pattern_free(rc->chart_labels_pattern);
|
||||
}
|
||||
|
||||
{
|
||||
char *tmp = simple_pattern_trim_around_equal(value);
|
||||
char *tmp_2 = health_config_add_key_to_values(tmp);
|
||||
rc->chart_labels = string_strdupz(tmp_2);
|
||||
freez(tmp);
|
||||
freez(tmp_2);
|
||||
}
|
||||
rc->chart_labels_pattern = simple_pattern_create(rrdcalc_chart_labels(rc), NULL, SIMPLE_PATTERN_EXACT,
|
||||
true);
|
||||
}
|
||||
else {
|
||||
error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",
|
||||
line, filename, rrdcalc_name(rc), key);
|
||||
|
|
@ -1186,9 +1254,31 @@ static int health_readfile(const char *filename, void *data) {
|
|||
rt->host_labels = string_strdupz(tmp);
|
||||
freez(tmp);
|
||||
}
|
||||
|
||||
rt->host_labels_pattern = simple_pattern_create(rrdcalctemplate_host_labels(rt), NULL,
|
||||
SIMPLE_PATTERN_EXACT, true);
|
||||
}
|
||||
else if(hash == hash_chart_label && !strcasecmp(key, HEALTH_CHART_LABEL_KEY)) {
|
||||
alert_cfg->chart_labels = string_strdupz(value);
|
||||
if(rt->chart_labels) {
|
||||
if(strcmp(rrdcalctemplate_chart_labels(rt), value) != 0)
|
||||
error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
|
||||
line, filename, rrdcalctemplate_name(rt), key, rrdcalctemplate_chart_labels(rt), value, value);
|
||||
|
||||
string_freez(rt->chart_labels);
|
||||
simple_pattern_free(rt->chart_labels_pattern);
|
||||
}
|
||||
|
||||
{
|
||||
char *tmp = simple_pattern_trim_around_equal(value);
|
||||
char *tmp_2 = health_config_add_key_to_values(tmp);
|
||||
rt->chart_labels = string_strdupz(tmp_2);
|
||||
freez(tmp);
|
||||
freez(tmp_2);
|
||||
}
|
||||
rt->chart_labels_pattern = simple_pattern_create(rrdcalctemplate_chart_labels(rt), NULL,
|
||||
SIMPLE_PATTERN_EXACT, true);
|
||||
}
|
||||
else {
|
||||
error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
|
||||
line, filename, rrdcalctemplate_name(rt), key);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue