archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-14 09:38:34 +00:00
Code Issues Projects Releases Wiki Activity

netdata doubles ()

Browse source 
* netdata doubles

* fix cmocka test

* fix cmocka test again

* fix left-overs of long double to NETDATA_DOUBLE

* RRDDIM detached from disk representation; db settings in [db] section of netdata.conf

* update the memory before saving

* rrdset is now detached from file structures too

* on memory mode map, update the memory mapped structures on every iteration

* allow RRD_ID_LENGTH_MAX to be changed

* granularity secs, back to update every

* fix formatting

* more formatting
This commit is contained in:
Costa Tsaousis 2022-06-28 17:04:37 +03:00 • committed by GitHub
parent e86cec2631
commit c3dfbe52a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
114 changed files with 1779 additions and 1495 deletions
aclk
aclk_rx_msgs.c
cli
README.md
collectors
apps.plugin
apps_plugin.c
cgroups.plugin
sys_fs_cgroup.c
tests
test_doubles.c
ebpf.plugin
ebpf_cachestat.cebpf_dcstat.c
perf.plugin
perf_plugin.c
plugins.d
pluginsd_parser.cpluginsd_parser.h
proc.plugin
proc_diskstats.cproc_mdstat.cproc_net_dev.cproc_net_wireless.csys_devices_system_edac_mc.c
statsd.plugin
statsd.c
daemon
config
README.md
global_statistics.cmain.cunit_test.c
database
README.md
engine
README.mdrrdengine.crrdengineapi.crrdengineapi.h
metric_correlations.c
ram
rrddim_mem.crrddim_mem.h
rrd.hrrdcalc.crrdcalc.hrrdcalctemplate.hrrddim.crrdhost.crrdset.crrdsetvar.crrdsetvar.hrrdvar.crrdvar.h
sqlite
sqlite_aclk_alert.csqlite_health.c
docs/guides
longer-metrics-storage.md
exporting
exporting_engine.h
graphite
graphite.c
json
json.c
opentsdb
opentsdb.c
process_data.c
prometheus
prometheus.c
remote_write
remote_write.c
tests
exporting_doubles.cnetdata_doubles.ctest_exporting_engine.h
health
health.chealth.hhealth_config.chealth_log.c
libnetdata
buffer
buffer.cbuffer.h
clocks
clocks.c
config
appconfig.cappconfig.h
eval
eval.ceval.h
inlined.h
json
json.cjson.h
required_dummies.h
statistical
statistical.cstatistical.h
storage_number
storage_number.cstorage_number.h
tests
test_storage_number.c
tests
test_str2ld.c
ml
Dimension.ccml.cc
parser
parser.h
streaming
rrdpush.csender.c
tests/profile
test-eval.c
web/api
badges
web_buffer_svg.cweb_buffer_svg.h
exporters/shell
allmetrics_shell.c
formatters
csv
csv.c
json
json.c
json_wrapper.crrd2json.crrd2json.hrrdset2json.c
ssv
ssv.c
value
value.cvalue.h
queries
average
average.caverage.h
countif
countif.ccountif.h
des
des.cdes.h
incremental_sum
incremental_sum.cincremental_sum.h
max
max.c
Show more

8
aclk/aclk_rx_msgs.c
View file

@ -55,19 +55,19 @@ static int cloud_to_agent_parse(JSON_ENTRY *e)
break;
case JSON_NUMBER:
if (!strcmp(e->name, "version")) {
data->version = e->data.number;
data->version = (int)e->data.number;
break;
}
if (!strcmp(e->name, "timeout")) {
data->timeout = e->data.number;
data->timeout = (int)e->data.number;
break;
}
if (!strcmp(e->name, "min-version")) {
data->min_version = e->data.number;
data->min_version = (int)e->data.number;
break;
}
if (!strcmp(e->name, "max-version")) {
data->max_version = e->data.number;
data->max_version = (int)e->data.number;
break;
}

2
cli/README.md
View file

@ -20,7 +20,7 @@ reload-health
reload-labels
Reload all labels.
save-database
Save internal DB to disk for memory mode save.
Save internal DB to disk for database mode save.
reopen-logs
Close and reopen log files.
shutdown-agent

18
collectors/apps.plugin/apps_plugin.c
View file

@ -173,7 +173,8 @@ static kernel_uint_t
global_gtime = 0;
// the normalization ratios, as calculated by normalize_utilization()
double utime_fix_ratio = 1.0,
NETDATA_DOUBLE
utime_fix_ratio = 1.0,
stime_fix_ratio = 1.0,
gtime_fix_ratio = 1.0,
minflt_fix_ratio = 1.0,
@ -501,7 +502,8 @@ struct file_descriptor {
static int
all_files_len = 0,
all_files_size = 0;
long double currentmaxfds = 0;
long currentmaxfds = 0;
// ----------------------------------------------------------------------------
// read users and groups from files
@ -3021,7 +3023,7 @@ static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) {
reallocate_target_fds(u);
reallocate_target_fds(g);
long double currentfds = 0;
long currentfds = 0;
size_t c, size = p->fds_size;
struct pid_fd *fds = p->fds;
for(c = 0; c < size ;c++) {
@ -3373,7 +3375,7 @@ static void normalize_utilization(struct target *root) {
gtime_fix_ratio =
cutime_fix_ratio =
cstime_fix_ratio =
cgtime_fix_ratio = 1.0; //(double)(global_utime + global_stime) / (double)(utime + cutime + stime + cstime);
cgtime_fix_ratio = 1.0; //(NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + cutime + stime + cstime);
}
else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) {
// children resources are too high
@ -3383,7 +3385,7 @@ static void normalize_utilization(struct target *root) {
gtime_fix_ratio = 1.0;
cutime_fix_ratio =
cstime_fix_ratio =
cgtime_fix_ratio = (double)((global_utime + global_stime) - (utime + stime)) / (double)(cutime + cstime);
cgtime_fix_ratio = (NETDATA_DOUBLE)((global_utime + global_stime) - (utime + stime)) / (NETDATA_DOUBLE)(cutime + cstime);
}
else if(utime || stime) {
// even running processes are unrealistic
@ -3391,7 +3393,7 @@ static void normalize_utilization(struct target *root) {
// lower the running processes resources
utime_fix_ratio =
stime_fix_ratio =
gtime_fix_ratio = (double)(global_utime + global_stime) / (double)(utime + stime);
gtime_fix_ratio = (NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + stime);
cutime_fix_ratio =
cstime_fix_ratio =
cgtime_fix_ratio = 0.0;
@ -3439,14 +3441,14 @@ static void normalize_utilization(struct target *root) {
if(utime || stime || gtime)
majflt_fix_ratio =
minflt_fix_ratio = (double)(utime * utime_fix_ratio + stime * stime_fix_ratio + gtime * gtime_fix_ratio) / (double)(utime + stime + gtime);
minflt_fix_ratio = (NETDATA_DOUBLE)(utime * utime_fix_ratio + stime * stime_fix_ratio + gtime * gtime_fix_ratio) / (NETDATA_DOUBLE)(utime + stime + gtime);
else
minflt_fix_ratio =
majflt_fix_ratio = 1.0;
if(cutime || cstime || cgtime)
cmajflt_fix_ratio =
cminflt_fix_ratio = (double)(cutime * cutime_fix_ratio + cstime * cstime_fix_ratio + cgtime * cgtime_fix_ratio) / (double)(cutime + cstime + cgtime);
cminflt_fix_ratio = (NETDATA_DOUBLE)(cutime * cutime_fix_ratio + cstime * cstime_fix_ratio + cgtime * cgtime_fix_ratio) / (NETDATA_DOUBLE)(cutime + cstime + cgtime);
else
cminflt_fix_ratio =
cmajflt_fix_ratio = 1.0;

24
collectors/cgroups.plugin/sys_fs_cgroup.c
View file

@ -835,7 +835,7 @@ struct cgroup {
unsigned long long cpu_cfs_quota;
RRDSETVAR *chart_var_cpu_limit;
calculated_number prev_cpu_usage;
NETDATA_DOUBLE prev_cpu_usage;
char *filename_memory_limit;
unsigned long long memory_limit;
@ -1012,7 +1012,7 @@ static unsigned long long calc_percentage(unsigned long long value, unsigned lon
if (total == 0) {
return 0;
}
return (calculated_number)value / (calculated_number)total * 100;
return (NETDATA_DOUBLE)value / (NETDATA_DOUBLE)total * 100;
}
static int calc_cgroup_depth(const char *id) {
@ -3686,7 +3686,7 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u
*filename = NULL;
}
else {
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024)));
rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1;
}
} else {
@ -3701,11 +3701,11 @@ static inline int update_memory_limits(char **filename, RRDSETVAR **chart_var, u
char *s = "max\n\0";
if(strcmp(s, buffer) == 0){
*value = UINT64_MAX;
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024)));
rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1;
}
*value = str2ull(buffer);
rrdsetvar_custom_chart_variable_set(*chart_var, (calculated_number)(*value / (1024 * 1024)));
rrdsetvar_custom_chart_variable_set(*chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
return 1;
}
}
@ -3814,17 +3814,17 @@ void update_cgroup_charts(int update_every) {
}
}
else {
calculated_number value = 0, quota = 0;
NETDATA_DOUBLE value = 0, quota = 0;
if(likely( ((!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) && (cg->filename_cpuset_cpus || (cg->filename_cpu_cfs_period && cg->filename_cpu_cfs_quota)))
|| ((cg->options & CGROUP_OPTIONS_IS_UNIFIED) && cg->filename_cpu_cfs_quota))) {
if(unlikely(cg->cpu_cfs_quota > 0))
quota = (calculated_number)cg->cpu_cfs_quota / (calculated_number)cg->cpu_cfs_period;
quota = (NETDATA_DOUBLE)cg->cpu_cfs_quota / (NETDATA_DOUBLE)cg->cpu_cfs_period;
if(unlikely(quota > 0 && quota < cg->cpuset_cpus))
value = quota * 100;
else
value = (calculated_number)cg->cpuset_cpus * 100;
value = (NETDATA_DOUBLE)cg->cpuset_cpus * 100;
}
if(likely(value)) {
rrdsetvar_custom_chart_variable_set(cg->chart_var_cpu_limit, value);
@ -3853,14 +3853,14 @@ void update_cgroup_charts(int update_every) {
rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
else
rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
cg->prev_cpu_usage = (calculated_number)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
cg->prev_cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
}
else
rrdset_next(cg->st_cpu_limit);
calculated_number cpu_usage = 0;
cpu_usage = (calculated_number)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
calculated_number cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
NETDATA_DOUBLE cpu_usage = 0;
cpu_usage = (NETDATA_DOUBLE)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
NETDATA_DOUBLE cpu_used = 100 * (cpu_usage - cg->prev_cpu_usage) / (value * update_every);
rrdset_isnot_obsolete(cg->st_cpu_limit);

2
collectors/cgroups.plugin/tests/test_doubles.c
View file

@ -109,7 +109,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
return NULL;
}
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value)
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value)
{
UNUSED(rs);
UNUSED(value);

8
collectors/ebpf.plugin/ebpf_cachestat.c
View file

@ -367,23 +367,23 @@ void cachestat_update_publish(netdata_publish_cachestat_t *out, uint64_t mpa, ui
uint64_t apcl, uint64_t apd)
{
// Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138
calculated_number total = (calculated_number) (((long long)mpa) - ((long long)mbd));
NETDATA_DOUBLE total = (NETDATA_DOUBLE) (((long long)mpa) - ((long long)mbd));
if (total < 0)
total = 0;
calculated_number misses = (calculated_number) ( ((long long) apcl) - ((long long) apd) );
NETDATA_DOUBLE misses = (NETDATA_DOUBLE) ( ((long long) apcl) - ((long long) apd) );
if (misses < 0)
misses = 0;
// If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding
// more pages than needed. In this case just assume misses as total and reset hits.
calculated_number hits = total - misses;
NETDATA_DOUBLE hits = total - misses;
if (hits < 0 ) {
misses = total;
hits = 0;
}
calculated_number ratio = (total > 0) ? hits/total : 1;
NETDATA_DOUBLE ratio = (total > 0) ? hits/total : 1;
out->ratio = (long long )(ratio*100);
out->hit = (long long)hits;

4
collectors/ebpf.plugin/ebpf_dcstat.c
View file

@ -239,8 +239,8 @@ static inline int ebpf_dc_load_and_attach(struct dc_bpf *obj, ebpf_module_t *em)
*/
void dcstat_update_publish(netdata_publish_dcstat_t *out, uint64_t cache_access, uint64_t not_found)
{
calculated_number successful_access = (calculated_number) (((long long)cache_access) - ((long long)not_found));
calculated_number ratio = (cache_access) ? successful_access/(calculated_number)cache_access : 0;
NETDATA_DOUBLE successful_access = (NETDATA_DOUBLE) (((long long)cache_access) - ((long long)not_found));
NETDATA_DOUBLE ratio = (cache_access) ? successful_access/(NETDATA_DOUBLE)cache_access : 0;
out->ratio = (long long )(ratio*100);
}

4
collectors/perf.plugin/perf_plugin.c
View file

@ -520,8 +520,8 @@ static void perf_send_metrics() {
, "instructions_per_cycle"
);
calculated_number result = ((calculated_number)perf_events[EV_ID_INSTRUCTIONS].value /
(calculated_number)perf_events[EV_ID_CPU_CYCLES].value) * 100.0;
NETDATA_DOUBLE result = ((NETDATA_DOUBLE)perf_events[EV_ID_INSTRUCTIONS].value /
(NETDATA_DOUBLE)perf_events[EV_ID_CPU_CYCLES].value) * 100.0;
printf("SET %s = %lld\n"
, "ipc"
, (collected_number) result

6
collectors/plugins.d/pluginsd_parser.c
View file

@ -96,7 +96,7 @@ PARSER_RC pluginsd_disable_action(void *user)
}
PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value)
PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value)
{
UNUSED(user);
@ -473,7 +473,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
{
char *name = words[1];
char *value = words[2];
calculated_number v;
NETDATA_DOUBLE v;
RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
@ -513,7 +513,7 @@ PARSER_RC pluginsd_variable(char **words, void *user, PLUGINSD_ACTION *plugins_
}
char *endptr = NULL;
v = (calculated_number)str2ld(value, &endptr);
v = (NETDATA_DOUBLE)str2ndd(value, &endptr);
if (unlikely(endptr && *endptr)) {
if (endptr == value)
error(

3
collectors/plugins.d/pluginsd_parser.h
View file

@ -30,8 +30,7 @@ extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *n
char *title, char *units, char *plugin, char *module, int priority,
int update_every, RRDSET_TYPE chart_type, char *options);
extern PARSER_RC pluginsd_disable_action(void *user);
extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global,
calculated_number value);
extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value);
extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source);

2
collectors/proc.plugin/proc_diskstats.c
View file

@ -201,7 +201,7 @@ static unsigned long long int bcache_read_number_with_units(const char *filename
static int unknown_units_error = 10;
char *end = NULL;
long double value = str2ld(buffer, &end);
NETDATA_DOUBLE value = str2ndd(buffer, &end);
if(end && *end) {
if(*end == 'k')
return (unsigned long long int)(value * 1024.0);

4
collectors/proc.plugin/proc_mdstat.c
View file

@ -263,7 +263,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
word = procfile_lineword(ff, l, 3);
remove_trailing_chars(word, '%');
unsigned long long percentage = (unsigned long long)(str2ld(word, NULL) * 100);
unsigned long long percentage = (unsigned long long)(str2ndd(word, NULL) * 100);
// possible operations: check, resync, recovery, reshape
// 4-th character is unique for each operation so it is checked
switch (procfile_lineword(ff, l, 1)[3]) {
@ -287,7 +287,7 @@ int do_proc_mdstat(int update_every, usec_t dt)
word += 7; // skip leading "finish="
if (likely(s > word))
raid->finish_in = (unsigned long long)(str2ld(word, NULL) * 60);
raid->finish_in = (unsigned long long)(str2ndd(word, NULL) * 60);
word = procfile_lineword(ff, l, 6);
s = remove_trailing_chars(word, 'K'); // remove trailing "K/sec"

2
collectors/proc.plugin/proc_net_dev.c
View file

@ -967,7 +967,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
d->filename_speed = NULL;
}
else {
rrdsetvar_custom_chart_variable_set(d->chart_var_speed, (calculated_number) d->speed * KILOBITS_IN_A_MEGABIT);
rrdsetvar_custom_chart_variable_set(d->chart_var_speed, (NETDATA_DOUBLE) d->speed * KILOBITS_IN_A_MEGABIT);
if(d->do_speed != CONFIG_BOOLEAN_NO) {
if(unlikely(!d->st_speed)) {

12
collectors/proc.plugin/proc_net_wireless.c
View file

@ -24,9 +24,9 @@ static struct netwireless {
kernel_uint_t status;
// Quality
calculated_number link;
calculated_number level;
calculated_number noise;
NETDATA_DOUBLE link;
NETDATA_DOUBLE level;
NETDATA_DOUBLE noise;
// Discarded packets
kernel_uint_t nwid;
@ -281,9 +281,9 @@ int do_proc_net_wireless(int update_every, usec_t dt)
}
if (likely(do_quality != CONFIG_BOOLEAN_NO)) {
wireless_dev->link = str2ld(procfile_lineword(ff, l, 2), NULL);
wireless_dev->level = str2ld(procfile_lineword(ff, l, 3), NULL);
wireless_dev->noise = str2ld(procfile_lineword(ff, l, 4), NULL);
wireless_dev->link = str2ndd(procfile_lineword(ff, l, 2), NULL);
wireless_dev->level = str2ndd(procfile_lineword(ff, l, 3), NULL);
wireless_dev->noise = str2ndd(procfile_lineword(ff, l, 4), NULL);
if (unlikely(!wireless_dev->st_link)) {
wireless_dev->st_link = rrdset_create_localhost("wireless",

2
collectors/proc.plugin/sys_devices_system_edac_mc.c
View file

@ -74,7 +74,7 @@ int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) {
}
static int do_ce = -1, do_ue = -1;
calculated_number ce_sum = 0, ue_sum = 0;
NETDATA_DOUBLE ce_sum = 0, ue_sum = 0;
struct mc *m;
if(unlikely(do_ce == -1)) {

40
collectors/statsd.plugin/statsd.c
View file

@ -35,7 +35,7 @@
// data specific to each metric type
typedef struct statsd_metric_gauge {
LONG_DOUBLE value;
NETDATA_DOUBLE value;
} STATSD_METRIC_GAUGE;
typedef struct statsd_metric_counter { // counter and meter
@ -64,7 +64,7 @@ typedef struct statsd_histogram_extensions {
size_t size;
size_t used;
LONG_DOUBLE *values; // dynamic array of values collected
NETDATA_DOUBLE *values; // dynamic array of values collected
} STATSD_METRIC_HISTOGRAM_EXTENSIONS;
typedef struct statsd_metric_histogram { // histogram and timer
@ -431,12 +431,12 @@ static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, cons
// --------------------------------------------------------------------------------------------------------------------
// statsd parsing numbers
static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) {
LONG_DOUBLE value;
static inline NETDATA_DOUBLE statsd_parse_float(const char *v, NETDATA_DOUBLE def) {
NETDATA_DOUBLE value;
if(likely(v && *v)) {
char *e = NULL;
value = str2ld(v, &e);
value = str2ndd(v, &e);
if(unlikely(e && *e))
error("STATSD: excess data '%s' after value '%s'", e, v);
}
@ -446,8 +446,8 @@ static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) {
return value;
}
static inline LONG_DOUBLE statsd_parse_sampling_rate(const char *v) {
LONG_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
static inline NETDATA_DOUBLE statsd_parse_sampling_rate(const char *v) {
NETDATA_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001;
if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
return sampling_rate;
@ -522,7 +522,7 @@ static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char
// magic loading of metric, without affecting anything
}
else {
m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
m->counter.value += llrintndd((NETDATA_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
m->events++;
m->count++;
@ -549,18 +549,18 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha
// magic loading of metric, without affecting anything
}
else {
LONG_DOUBLE v = statsd_parse_float(value, 1.0);
LONG_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
NETDATA_DOUBLE v = statsd_parse_float(value, 1.0);
NETDATA_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01;
if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
long long samples = llrintl(1.0 / sampling_rate);
long long samples = llrintndd(1.0 / sampling_rate);
while(samples-- > 0) {
if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
netdata_mutex_lock(&m->histogram.ext->mutex);
m->histogram.ext->size += statsd.histogram_increase_step;
m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size);
m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(NETDATA_DOUBLE) * m->histogram.ext->size);
netdata_mutex_unlock(&m->histogram.ext->mutex);
}
@ -1945,21 +1945,21 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
netdata_mutex_lock(&m->histogram.ext->mutex);
size_t len = m->histogram.ext->used;
LONG_DOUBLE *series = m->histogram.ext->values;
NETDATA_DOUBLE *series = m->histogram.ext->values;
sort_series(series, len);
m->histogram.ext->last_min = (collected_number)roundl(series[0] * statsd.decimal_detail);
m->histogram.ext->last_max = (collected_number)roundl(series[len - 1] * statsd.decimal_detail);
m->last = (collected_number)roundl(average(series, len) * statsd.decimal_detail);
m->histogram.ext->last_median = (collected_number)roundl(median_on_sorted_series(series, len) * statsd.decimal_detail);
m->histogram.ext->last_stddev = (collected_number)roundl(standard_deviation(series, len) * statsd.decimal_detail);
m->histogram.ext->last_sum = (collected_number)roundl(sum(series, len) * statsd.decimal_detail);
m->histogram.ext->last_min = (collected_number)roundndd(series[0] * statsd.decimal_detail);
m->histogram.ext->last_max = (collected_number)roundndd(series[len - 1] * statsd.decimal_detail);
m->last = (collected_number)roundndd(average(series, len) * statsd.decimal_detail);
m->histogram.ext->last_median = (collected_number)roundndd(median_on_sorted_series(series, len) * statsd.decimal_detail);
m->histogram.ext->last_stddev = (collected_number)roundndd(standard_deviation(series, len) * statsd.decimal_detail);
m->histogram.ext->last_sum = (collected_number)roundndd(sum(series, len) * statsd.decimal_detail);
size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0);
if(pct_len < 1)
m->histogram.ext->last_percentile = (collected_number)(series[0] * statsd.decimal_detail);
else
m->histogram.ext->last_percentile = (collected_number)roundl(series[pct_len - 1] * statsd.decimal_detail);
m->histogram.ext->last_percentile = (collected_number)roundndd(series[pct_len - 1] * statsd.decimal_detail);
netdata_mutex_unlock(&m->histogram.ext->mutex);

78
daemon/config/README.md
View file

@ -26,20 +26,21 @@ the [web server access lists](/web/server/README.md#access-lists).
`netdata.conf` has sections stated with `[section]`. You will see the following sections:
1. `[global]` to [configure](#global-section-options) the [Netdata daemon](/daemon/README.md).
2. `[directories]` to [configure](#directories-section-options) the directories used by Netdata.
3. `[logs]` to [configure](#logs-section-options) the Netdata logging.
4. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used
2. `[db]` to [configure](#db-section-options) the database of Netdata.
3. `[directories]` to [configure](#directories-section-options) the directories used by Netdata.
4. `[logs]` to [configure](#logs-section-options) the Netdata logging.
5. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used
Netdata.
5. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings.
6. `[ml]` to configure settings for [machine learning](/ml/README.md).
7. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md).
8. `[web]` to [configure the web server](/web/server/README.md).
9. `[registry]` for the [Netdata registry](/registry/README.md).
10. `[global statistics]` for the [Netdata registry](/registry/README.md).
11. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md).
12. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH
6. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings.
7. `[ml]` to configure settings for [machine learning](/ml/README.md).
8. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md).
9. `[web]` to [configure the web server](/web/server/README.md).
10. `[registry]` for the [Netdata registry](/registry/README.md).
11. `[global statistics]` for the [Netdata registry](/registry/README.md).
12. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md).
13. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH
settings.
13. `[plugin:NAME]` sections for each collector plugin, under the
14. `[plugin:NAME]` sections for each collector plugin, under the
comment [Per plugin configuration](#per-plugin-configuration).
The configuration file is a `name = value` dictionary. Netdata will not complain if you set options unknown to it. When
@ -67,30 +68,35 @@ Please note that your data history will be lost if you have modified `history` p
### [global] section options
| setting | default | info |
|:-------------------------------------:|:------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) |
| OOM score | `0` | |
| glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| hostname | auto-detected | The hostname of the computer running Netdata. |
| history | `3996` | Used with `memory mode = save/map/ram/alloc`, not the default `memory mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. This setting can also be configured per chart. Check [Memory Requirements](/database/README.md) for more information. |
| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). |
| memory mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size` and `dbengine disk space`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. |
| page cache size | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. |
| dbengine disk space | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
| dbengine multihost disk space | 256 | Same functionality as `dbengine disk space`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. |
| host access prefix | | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) |
| timezone | auto-detected | The timezone retrieved from the environment variable |
| run as user | `netdata` | The user Netdata will run as. |
| pthread stack size | auto-detected | |
| cleanup obsolete charts after seconds | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
| gap when lost iterations above | `1` | |
| cleanup orphan hosts after seconds | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
| setting | default | info |
|:-------------------------------------:|:-------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) |
| OOM score | `0` | |
| glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). |
| hostname | auto-detected | The hostname of the computer running Netdata. |
| host access prefix | empty | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). |
| timezone | auto-detected | The timezone retrieved from the environment variable |
| run as user | `netdata` | The user Netdata will run as. |
| pthread stack size | auto-detected | |
### [db] section options
| setting | default | info |
|:----------------------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size MB` and `dbengine disk space MB`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. |
| retention | `3600` | Used with `mode = save/map/ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](/database/README.md) for more information. |
| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). |
| page cache size MB | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. |
| dbengine disk space MB | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. |
| dbengine multihost disk space MB | 256 | Same functionality as `dbengine disk space MB`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. |
| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database and the dbengine page cache to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) |
| cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions |
| gap when lost iterations above | `1` | |
| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. |
| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions |
| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. |
| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. |
### [directories] section options

2
daemon/global_statistics.c
View file

@ -1146,7 +1146,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
if(wu->workers_cpu_registered == 0)
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0);
else
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (calculated_number)wu->workers_cpu_registered ));
rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (NETDATA_DOUBLE)wu->workers_cpu_registered ));
rrdset_done(wu->st_workers_cpu);
}

139
daemon/main.c
View file

@ -522,6 +522,58 @@ static void backwards_compatible_config() {
config_move(CONFIG_SECTION_STATSD, "enabled",
CONFIG_SECTION_PLUGINS, "statsd");
config_move(CONFIG_SECTION_GLOBAL, "memory mode",
CONFIG_SECTION_DB, "mode");
config_move(CONFIG_SECTION_GLOBAL, "history",
CONFIG_SECTION_DB, "retention");
config_move(CONFIG_SECTION_GLOBAL, "update every",
CONFIG_SECTION_DB, "update every");
config_move(CONFIG_SECTION_GLOBAL, "page cache size",
CONFIG_SECTION_DB, "page cache size MB");
config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc",
CONFIG_SECTION_DB, "page cache with malloc");
config_move(CONFIG_SECTION_GLOBAL, "dbengine disk space",
CONFIG_SECTION_DB, "dbengine disk space MB");
config_move(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space",
CONFIG_SECTION_DB, "dbengine multihost disk space MB");
config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)",
CONFIG_SECTION_DB, "memory deduplication (ksm)");
config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout",
CONFIG_SECTION_DB, "dbengine page fetch timeout secs");
config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries",
CONFIG_SECTION_DB, "dbengine page fetch retries");
config_move(CONFIG_SECTION_GLOBAL, "dbengine extent pages",
CONFIG_SECTION_DB, "dbengine pages per extent");
config_move(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds",
CONFIG_SECTION_DB, "cleanup obsolete charts after secs");
config_move(CONFIG_SECTION_GLOBAL, "gap when lost iterations above",
CONFIG_SECTION_DB, "gap when lost iterations above");
config_move(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds",
CONFIG_SECTION_DB, "cleanup orphan hosts after secs");
config_move(CONFIG_SECTION_GLOBAL, "delete obsolete charts files",
CONFIG_SECTION_DB, "delete obsolete charts files");
config_move(CONFIG_SECTION_GLOBAL, "delete orphan hosts files",
CONFIG_SECTION_DB, "delete orphan hosts files");
config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics",
CONFIG_SECTION_DB, "enable zero metrics");
}
static void get_netdata_configured_variables() {
@ -539,28 +591,40 @@ static void get_netdata_configured_variables() {
debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname);
// ------------------------------------------------------------------------
// get default database size
// get default database update frequency
default_rrd_history_entries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));
long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);
if(h != default_rrd_history_entries) {
config_set_number(CONFIG_SECTION_GLOBAL, "history", h);
default_rrd_history_entries = (int)h;
}
if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) {
error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES);
default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY);
default_rrd_update_every = UPDATE_EVERY;
config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
}
// ------------------------------------------------------------------------
// get default database update frequency
// get default memory mode for the database
default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_GLOBAL, "update every", UPDATE_EVERY);
if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX);
default_rrd_update_every = UPDATE_EVERY;
{
const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
default_rrd_memory_mode = rrd_memory_mode_id(mode);
if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) {
error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode));
config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
}
}
// ------------------------------------------------------------------------
// get default database size
if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {
default_rrd_history_entries = (int)config_get_number(
CONFIG_SECTION_DB, "retention",
align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));
long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);
if (h != default_rrd_history_entries) {
config_set_number(CONFIG_SECTION_DB, "retention", h);
default_rrd_history_entries = (int)h;
}
}
// ------------------------------------------------------------------------
@ -582,39 +646,38 @@ static void get_netdata_configured_variables() {
netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH];
}
// ------------------------------------------------------------------------
// get default memory mode for the database
default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
#ifdef ENABLE_DBENGINE
// ------------------------------------------------------------------------
// get default Database Engine page cache size in MiB
db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_GLOBAL, "page cache uses malloc", CONFIG_BOOLEAN_NO);
default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "page cache size", default_rrdeng_page_cache_mb);
db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "page cache with malloc", CONFIG_BOOLEAN_NO);
default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
config_set_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb);
}
// ------------------------------------------------------------------------
// get default Database Engine disk space quota in MiB
default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine disk space", default_rrdeng_disk_quota_mb);
default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb);
}
default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace());
default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace());
if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb);
}
#else
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use memory mode ram instead.");
default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");
default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
}
#endif
// ------------------------------------------------------------------------
@ -626,11 +689,12 @@ static void get_netdata_configured_variables() {
// get KSM settings
#ifdef MADV_MERGEABLE
enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm);
enable_ksm = config_get_boolean(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm);
#endif
// --------------------------------------------------------------------
// metric correlations
enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method)));
@ -1232,39 +1296,47 @@ int main(int argc, char **argv) {
// --------------------------------------------------------------------
// get log filenames and settings
log_init();
error_log_limit_unlimited();
// initialize the log files
open_all_log_files();
#ifdef ENABLE_DBENGINE
default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
if (default_rrdeng_page_fetch_timeout < 1) {
info("\"dbengine page fetch timeout\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
default_rrdeng_page_fetch_timeout = 1;
config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout);
}
default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
if (default_rrdeng_page_fetch_retries < 1) {
info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries);
default_rrdeng_page_fetch_retries = 1;
config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
}
#endif
get_system_timezone();
// --------------------------------------------------------------------
// get the certificate and start security
#ifdef ENABLE_HTTPS
security_init();
#endif
// --------------------------------------------------------------------
// This is the safest place to start the SILENCERS structure
set_silencers_filename();
health_initialize_global_silencers();
// --------------------------------------------------------------------
// Initialize ML configuration
ml_init();
// --------------------------------------------------------------------
@ -1272,9 +1344,11 @@ int main(int argc, char **argv) {
// block signals while initializing threads.
// this causes the threads to block signals.
signals_block();
// setup the signals we want to use
signals_init();
// setup threads configs
@ -1303,6 +1377,7 @@ int main(int argc, char **argv) {
if(web_server_mode != WEB_SERVER_MODE_NONE)
api_listen_sockets_setup();
}
#ifdef NETDATA_INTERNAL_CHECKS
@ -1386,7 +1461,7 @@ int main(int argc, char **argv) {
web_server_config_options();
netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_GLOBAL, "enable zero metrics", CONFIG_BOOLEAN_NO);
netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO);
set_late_global_environment();

251
daemon/unit_test.c
View file

@ -4,7 +4,7 @@
static int check_number_printing(void) {
struct {
calculated_number n;
NETDATA_DOUBLE n;
const char *correct;
} values[] = {
{ .n = 0, .correct = "0" },
@ -22,8 +22,8 @@ static int check_number_printing(void) {
char netdata[50], system[50];
int i, failed = 0;
for(i = 0; values[i].correct ; i++) {
print_calculated_number(netdata, values[i].n);
snprintfz(system, 49, "%0.12" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)values[i].n);
print_netdata_double(netdata, values[i].n);
snprintfz(system, 49, "%0.12" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)values[i].n);
int ok = 1;
if(strcmp(netdata, values[i].correct) != 0) {
@ -95,35 +95,36 @@ static int check_rrdcalc_comparisons(void) {
return 0;
}
int check_storage_number(calculated_number n, int debug) {
int check_storage_number(NETDATA_DOUBLE n, int debug) {
char buffer[100];
uint32_t flags = SN_DEFAULT_FLAGS;
storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s);
NETDATA_DOUBLE d = unpack_storage_number(s);
if(!does_storage_number_exist(s)) {
fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n);
fprintf(stderr, "Exists flags missing for number " NETDATA_DOUBLE_FORMAT "!\n", n);
return 5;
}
calculated_number ddiff = d - n;
calculated_number dcdiff = ddiff * 100.0 / n;
NETDATA_DOUBLE ddiff = d - n;
NETDATA_DOUBLE dcdiff = ddiff * 100.0 / n;
if(dcdiff < 0) dcdiff = -dcdiff;
size_t len = (size_t)print_calculated_number(buffer, d);
calculated_number p = str2ld(buffer, NULL);
calculated_number pdiff = n - p;
calculated_number pcdiff = pdiff * 100.0 / n;
size_t len = (size_t)print_netdata_double(buffer, d);
NETDATA_DOUBLE p = str2ndd(buffer, NULL);
NETDATA_DOUBLE pdiff = n - p;
NETDATA_DOUBLE pcdiff = pdiff * 100.0 / n;
if(pcdiff < 0) pcdiff = -pcdiff;
if(debug) {
fprintf(stderr,
CALCULATED_NUMBER_FORMAT " original\n"
CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n"
"%s printed after unpacked (%zu bytes)\n"
CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n",
NETDATA_DOUBLE_FORMAT
" original\n" NETDATA_DOUBLE_FORMAT " packed and unpacked, (stored as 0x%08X, diff " NETDATA_DOUBLE_FORMAT
", " NETDATA_DOUBLE_FORMAT "%%)\n"
"%s printed after unpacked (%zu bytes)\n" NETDATA_DOUBLE_FORMAT
" re-parsed from printed (diff " NETDATA_DOUBLE_FORMAT ", " NETDATA_DOUBLE_FORMAT "%%)\n\n",
n,
d, s, ddiff, dcdiff,
buffer, len,
@ -132,10 +133,11 @@ int check_storage_number(calculated_number n, int debug) {
if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer));
if(dcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT)
fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, dcdiff);
fprintf(stderr, "WARNING: packing number " NETDATA_DOUBLE_FORMAT " has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, dcdiff);
if(pcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT)
fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, pcdiff);
fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " NETDATA_DOUBLE_FORMAT
" has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, pcdiff);
}
if(len != strlen(buffer)) return 1;
@ -144,8 +146,8 @@ int check_storage_number(calculated_number n, int debug) {
return 0;
}
calculated_number storage_number_min(calculated_number n) {
calculated_number r = 1, last;
NETDATA_DOUBLE storage_number_min(NETDATA_DOUBLE n) {
NETDATA_DOUBLE r = 1, last;
do {
last = n;
@ -159,12 +161,12 @@ calculated_number storage_number_min(calculated_number n) {
void benchmark_storage_number(int loop, int multiplier) {
int i, j;
calculated_number n, d;
NETDATA_DOUBLE n, d;
storage_number s;
unsigned long long user, system, total, mine, their;
calculated_number storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
calculated_number storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW);
NETDATA_DOUBLE storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
NETDATA_DOUBLE storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW);
char buffer[100];
@ -174,25 +176,25 @@ void benchmark_storage_number(int loop, int multiplier) {
// ------------------------------------------------------------------------
fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(calculated_number));
fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(NETDATA_DOUBLE));
fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number));
mine = (calculated_number)sizeof(storage_number) * (calculated_number)loop;
their = (calculated_number)sizeof(calculated_number) * (calculated_number)loop;
mine = (NETDATA_DOUBLE)sizeof(storage_number) * (NETDATA_DOUBLE)loop;
their = (NETDATA_DOUBLE)sizeof(NETDATA_DOUBLE) * (NETDATA_DOUBLE)loop;
if(mine > their) {
fprintf(stderr, "\nNETDATA NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (LONG_DOUBLE)(mine / their));
fprintf(stderr, "\nNETDATA NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (NETDATA_DOUBLE)(mine / their));
}
else {
fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (LONG_DOUBLE)(their / mine));
fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (NETDATA_DOUBLE)(their / mine));
}
fprintf(stderr, "\nNETDATA FLOATING POINT\n");
fprintf(stderr, "MIN POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW));
fprintf(stderr, "MAX POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW));
fprintf(stderr, "MIN NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW));
fprintf(stderr, "MAX NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW));
fprintf(stderr, "Maximum accuracy loss accepted: " CALCULATED_NUMBER_FORMAT "%%\n\n\n", (calculated_number)ACCURACY_LOSS_ACCEPTED_PERCENT);
fprintf(stderr, "MIN POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW));
fprintf(stderr, "MAX POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW));
fprintf(stderr, "MIN NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW));
fprintf(stderr, "MAX NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW));
fprintf(stderr, "Maximum accuracy loss accepted: " NETDATA_DOUBLE_FORMAT "%%\n\n\n", (NETDATA_DOUBLE)ACCURACY_LOSS_ACCEPTED_PERCENT);
// ------------------------------------------------------------------------
@ -207,7 +209,7 @@ void benchmark_storage_number(int loop, int multiplier) {
n *= multiplier;
if(n > storage_number_positive_max) n = storage_number_positive_min;
print_calculated_number(buffer, n);
print_netdata_double(buffer, n);
}
}
@ -217,7 +219,8 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system;
mine = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0));
fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
// ------------------------------------------------------------------------
@ -231,7 +234,7 @@ void benchmark_storage_number(int loop, int multiplier) {
for(i = 0; i < loop ;i++) {
n *= multiplier;
if(n > storage_number_positive_max) n = storage_number_positive_min;
snprintfz(buffer, 100, CALCULATED_NUMBER_FORMAT, n);
snprintfz(buffer, 100, NETDATA_DOUBLE_FORMAT, n);
}
}
@ -241,13 +244,14 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system;
their = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0));
fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
if(mine > total) {
fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0));
fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0));
}
else {
fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0));
fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(their * 100.0 / mine - 100.0));
}
// ------------------------------------------------------------------------
@ -265,7 +269,7 @@ void benchmark_storage_number(int loop, int multiplier) {
s = pack_storage_number(n, SN_DEFAULT_FLAGS);
d = unpack_storage_number(s);
print_calculated_number(buffer, d);
print_netdata_double(buffer, d);
}
}
@ -275,13 +279,14 @@ void benchmark_storage_number(int loop, int multiplier) {
total = user + system;
mine = total;
fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0));
fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER
", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0));
if(mine > their) {
fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0));
fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0));
}
else {
fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0));
fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(their * 100.0 / mine - 100.0));
}
// ------------------------------------------------------------------------
@ -290,13 +295,13 @@ void benchmark_storage_number(int loop, int multiplier) {
static int check_storage_number_exists() {
uint32_t flags = SN_DEFAULT_FLAGS;
calculated_number n = 0.0;
NETDATA_DOUBLE n = 0.0;
storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s);
NETDATA_DOUBLE d = unpack_storage_number(s);
if(n != d) {
fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d);
fprintf(stderr, "Wrong number returned. Expected " NETDATA_DOUBLE_FORMAT ", returned " NETDATA_DOUBLE_FORMAT "!\n", n, d);
return 1;
}
@ -306,10 +311,10 @@ static int check_storage_number_exists() {
int unit_test_storage() {
if(check_storage_number_exists()) return 0;
calculated_number storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
calculated_number storage_number_negative_max = unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW);
NETDATA_DOUBLE storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW);
NETDATA_DOUBLE storage_number_negative_max = unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW);
calculated_number c, a = 0;
NETDATA_DOUBLE c, a = 0;
int i, j, g, r = 0;
for(g = -1; g <= 1 ; g++) {
@ -343,23 +348,26 @@ int unit_test_str2ld() {
int i;
for(i = 0; values[i] ; i++) {
char *e_mine = "hello", *e_sys = "world";
LONG_DOUBLE mine = str2ld(values[i], &e_mine);
LONG_DOUBLE sys = strtold(values[i], &e_sys);
NETDATA_DOUBLE mine = str2ndd(values[i], &e_mine);
NETDATA_DOUBLE sys = strtondd(values[i], &e_sys);
if(isnan(mine)) {
if(!isnan(sys)) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
return -1;
}
}
else if(isinf(mine)) {
if(!isinf(sys)) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys);
return -1;
}
}
else if(mine != sys && ABS(mine-sys) > 0.000001) {
fprintf(stderr, "Value '%s' is parsed as %" LONG_DOUBLE_MODIFIER ", but system believes it is %" LONG_DOUBLE_MODIFIER ", delta %" LONG_DOUBLE_MODIFIER ".\n", values[i], mine, sys, sys-mine);
fprintf(stderr, "Value '%s' is parsed as %" NETDATA_DOUBLE_MODIFIER
", but system believes it is %" NETDATA_DOUBLE_MODIFIER ", delta %" NETDATA_DOUBLE_MODIFIER ".\n", values[i], mine, sys, sys-mine);
return -1;
}
@ -368,7 +376,8 @@ int unit_test_str2ld() {
return -1;
}
fprintf(stderr, "str2ld() parsed value '%s' exactly the same way with strtold(), returned %" LONG_DOUBLE_MODIFIER " vs %" LONG_DOUBLE_MODIFIER "\n", values[i], mine, sys);
fprintf(stderr, "str2ndd() parsed value '%s' exactly the same way with strtold(), returned %" NETDATA_DOUBLE_MODIFIER
" vs %" NETDATA_DOUBLE_MODIFIER "\n", values[i], mine, sys);
}
return 0;
@ -461,10 +470,10 @@ struct test {
unsigned long feed_entries;
unsigned long result_entries;
struct feed_values *feed;
calculated_number *results;
NETDATA_DOUBLE *results;
collected_number *feed2;
calculated_number *results2;
NETDATA_DOUBLE *results2;
};
// --------------------------------------------------------------------------------------------------------------------
@ -484,7 +493,7 @@ struct feed_values test1_feed[] = {
{ 1000000, 100 },
};
calculated_number test1_results[] = {
NETDATA_DOUBLE test1_results[] = {
20, 30, 40, 50, 60, 70, 80, 90, 100
};
@ -520,7 +529,7 @@ struct feed_values test2_feed[] = {
{ 1000000, 100 },
};
calculated_number test2_results[] = {
NETDATA_DOUBLE test2_results[] = {
20, 30, 40, 50, 60, 70, 80, 90, 100
};
@ -555,7 +564,7 @@ struct feed_values test3_feed[] = {
{ 1000000, 100 },
};
calculated_number test3_results[] = {
NETDATA_DOUBLE test3_results[] = {
10, 10, 10, 10, 10, 10, 10, 10, 10
};
@ -590,7 +599,7 @@ struct feed_values test4_feed[] = {
{ 1000000, 100 },
};
calculated_number test4_results[] = {
NETDATA_DOUBLE test4_results[] = {
10, 10, 10, 10, 10, 10, 10, 10, 10
};
@ -625,7 +634,7 @@ struct feed_values test5_feed[] = {
{ 1000000, 0x00000000FFFFFFFFULL / 15 * 0 },
};
calculated_number test5_results[] = {
NETDATA_DOUBLE test5_results[] = {
0x00000000FFFFFFFFULL / 15 * 7,
0x00000000FFFFFFFFULL / 15 * 7,
0x00000000FFFFFFFFULL / 15,
@ -668,7 +677,7 @@ struct feed_values test5b_feed[] = {
{ 1000000, 0xFFFFFFFFFFFFFFFFULL / 15 * 0 },
};
calculated_number test5b_results[] = {
NETDATA_DOUBLE test5b_results[] = {
0xFFFFFFFFFFFFFFFFULL / 15 * 7,
0xFFFFFFFFFFFFFFFFULL / 15 * 7,
0xFFFFFFFFFFFFFFFFULL / 15,
@ -717,7 +726,7 @@ struct feed_values test6_feed[] = {
{ 250000, 16000 },
};
calculated_number test6_results[] = {
NETDATA_DOUBLE test6_results[] = {
4000, 4000, 4000, 4000
};
@ -752,7 +761,7 @@ struct feed_values test7_feed[] = {
{ 2000000, 10000 },
};
calculated_number test7_results[] = {
NETDATA_DOUBLE test7_results[] = {
500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500
};
@ -783,7 +792,7 @@ struct feed_values test8_feed[] = {
{ 2000000, 6000 },
};
calculated_number test8_results[] = {
NETDATA_DOUBLE test8_results[] = {
1250, 2000, 2250, 3000, 3250, 4000, 4250, 5000, 5250, 6000
};
@ -824,7 +833,7 @@ struct feed_values test9_feed[] = {
{ 250000, 16000 },
};
calculated_number test9_results[] = {
NETDATA_DOUBLE test9_results[] = {
4000, 8000, 12000, 16000
};
@ -859,7 +868,7 @@ struct feed_values test10_feed[] = {
{ 1000000, 6900 + 1000 },
};
calculated_number test10_results[] = {
NETDATA_DOUBLE test10_results[] = {
1000, 1000, 1000, 1000, 1000, 1000, 1000
};
@ -898,11 +907,11 @@ collected_number test11_feed2[] = {
10, 20, 30, 40, 50, 60, 70, 80, 90, 100
};
calculated_number test11_results[] = {
NETDATA_DOUBLE test11_results[] = {
50, 50, 50, 50, 50, 50, 50, 50, 50
};
calculated_number test11_results2[] = {
NETDATA_DOUBLE test11_results2[] = {
50, 50, 50, 50, 50, 50, 50, 50, 50
};
@ -941,11 +950,11 @@ collected_number test12_feed2[] = {
10*3, 20*3, 30*3, 40*3, 50*3, 60*3, 70*3, 80*3, 90*3, 100*3
};
calculated_number test12_results[] = {
NETDATA_DOUBLE test12_results[] = {
25, 25, 25, 25, 25, 25, 25, 25, 25
};
calculated_number test12_results2[] = {
NETDATA_DOUBLE test12_results2[] = {
75, 75, 75, 75, 75, 75, 75, 75, 75
};
@ -980,7 +989,7 @@ struct feed_values test13_feed[] = {
{ 1000000, 6900 + 1000 },
};
calculated_number test13_results[] = {
NETDATA_DOUBLE test13_results[] = {
83.3333300, 100, 100, 100, 100, 100, 100
};
@ -1015,7 +1024,7 @@ struct feed_values test14_feed[] = {
{ 29942000, 0x0153987f888982d0ULL },
};
calculated_number test14_results[] = {
NETDATA_DOUBLE test14_results[] = {
23.1383300, 21.8515600, 21.8804600, 21.7788000, 22.0112200, 22.4386100, 22.0906100, 21.9150800
};
@ -1047,7 +1056,7 @@ struct feed_values test14b_feed[] = {
{ 29942000, 13573000 + 29969000 + 29958000 + 30054000 + 34952000 + 25046000 + 29947000 + 30054000 + 29942000 },
};
calculated_number test14b_results[] = {
NETDATA_DOUBLE test14b_results[] = {
1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000
};
@ -1079,7 +1088,7 @@ struct feed_values test14c_feed[] = {
{ 30000000, 29000000 + 1000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 + 30000000 },
};
calculated_number test14c_results[] = {
NETDATA_DOUBLE test14c_results[] = {
1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000, 1000000
};
@ -1118,11 +1127,11 @@ collected_number test15_feed2[] = {
178825286, 178825286, 178825286, 178825286, 178825498, 178825498, 179165652, 179202964, 179203282, 179204130
};
calculated_number test15_results[] = {
NETDATA_DOUBLE test15_results[] = {
5857.4080000, 5898.4540000, 5891.6590000, 5806.3160000, 5914.2640000, 3202.2630000, 5589.6560000, 5822.5260000, 5911.7520000
};
calculated_number test15_results2[] = {
NETDATA_DOUBLE test15_results2[] = {
0.0000000, 0.0000000, 0.0024944, 1.6324779, 0.0212777, 2655.1890000, 290.5387000, 5.6733610, 6.5960220
};
@ -1173,12 +1182,13 @@ int run_test(struct test *test)
if(c) {
time_now += test->feed[c].microseconds;
fprintf(stderr, " > %s: feeding position %lu, after %0.3f seconds (%0.3f seconds from start), delta " CALCULATED_NUMBER_FORMAT ", rate " CALCULATED_NUMBER_FORMAT "\n",
fprintf(stderr, " > %s: feeding position %lu, after %0.3f seconds (%0.3f seconds from start), delta " NETDATA_DOUBLE_FORMAT
", rate " NETDATA_DOUBLE_FORMAT "\n",
test->name, c+1,
(float)test->feed[c].microseconds / 1000000.0,
(float)time_now / 1000000.0,
((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor,
(((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000);
((NETDATA_DOUBLE)test->feed[c].value - (NETDATA_DOUBLE)last) * (NETDATA_DOUBLE)test->multiplier / (NETDATA_DOUBLE)test->divisor,
(((NETDATA_DOUBLE)test->feed[c].value - (NETDATA_DOUBLE)last) * (NETDATA_DOUBLE)test->multiplier / (NETDATA_DOUBLE)test->divisor) / (NETDATA_DOUBLE)test->feed[c].microseconds * (NETDATA_DOUBLE)1000000);
// rrdset_next_usec_unfiltered(st, test->feed[c].microseconds);
st->usec_since_last_update = test->feed[c].microseconds;
@ -1216,10 +1226,11 @@ int run_test(struct test *test)
unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;
for(c = 0 ; c < max ; c++) {
calculated_number v = unpack_storage_number(rd->values[c]);
calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
int same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n",
NETDATA_DOUBLE v = unpack_storage_number(rd->db[c]);
NETDATA_DOUBLE n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd->name, c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###");
@ -1227,10 +1238,11 @@ int run_test(struct test *test)
if(!same) errors++;
if(rd2) {
v = unpack_storage_number(rd2->values[c]);
v = unpack_storage_number(rd2->db[c]);
n = test->results2[c];
same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n",
same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", %s\n",
test->name, rd2->name, c+1,
(int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
n, v, (same)?"OK":"### E R R O R ###");
@ -1470,14 +1482,14 @@ int unit_test(long delay, long shift)
int ret = 0;
storage_number sn;
calculated_number cn, v;
NETDATA_DOUBLE cn, v;
for(c = 0 ; c < st->counter ; c++) {
fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
for(rd = st->dimensions ; rd ; rd = rd->next) {
sn = rd->values[c];
sn = rd->db[c];
cn = unpack_storage_number(sn);
fprintf(stderr, "\t %s " CALCULATED_NUMBER_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn);
fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn);
if(rd == rdabs) v =
( oincrement
@ -1492,7 +1504,7 @@ int unit_test(long delay, long shift)
if(v == cn) fprintf(stderr, "passed.\n");
else {
fprintf(stderr, "ERROR! (expected " CALCULATED_NUMBER_FORMAT ")\n", v);
fprintf(stderr, "ERROR! (expected " NETDATA_DOUBLE_FORMAT ")\n", v);
ret = 1;
}
}
@ -1742,7 +1754,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
time_t time_now, time_retrieved;
int i, j, k, c, errors, update_every;
collected_number last;
calculated_number value, expected;
NETDATA_DOUBLE value, expected;
SN_FLAGS nflags;
struct rrddim_query_handle handle;
size_t value_errors = 0, time_errors = 0;
@ -1759,16 +1771,16 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
for (k = 0; k < QUERY_BATCH; ++k) {
last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
j * REGION_POINTS[current_region] + c + k;
expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
time_t end_time;
value = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) {
if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value);
value_errors++;
errors++;
@ -1806,7 +1818,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
int i, j, errors, value_errors = 0, time_errors = 0;
long c;
collected_number last;
calculated_number value, expected;
NETDATA_DOUBLE value, expected;
errors = 0;
long points = (time_end - time_start) / update_every;
@ -1825,18 +1837,18 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
// for each dimension
for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
calculated_number *cn = &r->v[ c * r->d ];
NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
value = cn[j];
assert(rd[i][j] == d);
last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c;
expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) {
if(value_errors < 10)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
value_errors++;
}
@ -1959,18 +1971,18 @@ int test_dbengine(void)
// for each dimension
for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
calculated_number *cn = &r->v[ c * r->d ];
calculated_number value = cn[j];
NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
NETDATA_DOUBLE value = cn[j];
assert(rd[i][j] == d);
collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset + 1;
calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
NETDATA_DOUBLE expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
uint8_t same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if(!same) {
if(!value_errors)
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", RRDR found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n",
fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
value_errors++;
}
@ -2167,7 +2179,7 @@ static void query_dbengine_chart(void *arg)
uint8_t same;
time_t time_now, time_retrieved;
collected_number generatedv;
calculated_number value, expected;
NETDATA_DOUBLE value, expected;
SN_FLAGS nflags;
struct rrddim_query_handle handle;
size_t value_errors = 0, time_errors = 0;
@ -2200,12 +2212,12 @@ static void query_dbengine_chart(void *arg)
++thread_info->queries_nr;
for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
generatedv = generate_dbengine_chart_value(i, j, time_now);
expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_DEFAULT_FLAGS));
expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
if (unlikely(rd->state->query_ops.is_finished(&handle))) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n",
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected);
++thread_info->errors;
}
@ -2213,10 +2225,10 @@ static void query_dbengine_chart(void *arg)
}
time_t end_time;
value = rd->state->query_ops.next_metric(&handle, &time_retrieved, &end_time, &nflags);
if (!calculated_number_isnumber(value)) {
if (!netdata_double_isnumber(value)) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n",
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found data gap, ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected);
++thread_info->errors;
}
@ -2224,13 +2236,12 @@ static void query_dbengine_chart(void *arg)
}
++thread_info->queried_metrics_nr;
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
same = (roundndd(value) == roundndd(expected)) ? 1 : 0;
if (!same) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
if(!value_errors)
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value "
CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT
", ### E R R O R ###\n",
fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
st->name, rd->name, (unsigned long) time_now, expected, value);
value_errors++;
thread_info->errors++;

41
database/README.md
View file

@ -13,7 +13,7 @@ Netdata is fully capable of long-term metrics storage, at per-second granularity
traditional database. There is some amount of RAM dedicated to data caching and indexing and the rest of the data
reside compressed on disk. The number of history entries is not fixed in this case, but depends on the configured
disk space and the effective compression ratio of the data stored. This is the **only mode** that supports changing
the data collection update frequency (`update_every`) **without losing** the previously stored metrics. For more
the data collection update frequency (`update every`) **without losing** the previously stored metrics. For more
details see [here](/database/engine/README.md).
2. `ram`, data are purely in memory. Data are never saved on disk. This mode uses `mmap()` and supports [KSM](#ksm).
@ -32,15 +32,12 @@ Netdata is fully capable of long-term metrics storage, at per-second granularity
6. `alloc`, like `ram` but it uses `calloc()` and does not support [KSM](#ksm). This mode is the fallback for all
others except `none`.
You can select the memory mode by editing `netdata.conf` and setting:
You can select the database mode by editing `netdata.conf` and setting:
```conf
[global]
[db]
# dbengine (default), ram, save (the default if dbengine not available), map (swap like), none, alloc
memory mode = dbengine
# the directory where data are saved
cache directory = /var/cache/netdata
mode = dbengine
```
## Running Netdata in embedded devices
@ -49,26 +46,23 @@ Embedded devices usually have very limited RAM resources available.
There are 2 settings for you to tweak:
1. `update every`, which controls the data collection frequency
2. `history`, which controls the size of the database in RAM (except for `memory mode = dbengine`)
1. `[db].update every`, which controls the data collection frequency
2. `[db].retention`, which controls the size of the database in memory (except for `[db].mode = dbengine`)
By default `update every = 1` and `history = 3600`. This gives you an hour of data with per second updates.
By default `[db].update every = 1` and `[db].retention = 3600`. This gives you an hour of data with per second updates.
If you set `update every = 2` and `history = 1800`, you will still have an hour of data, but collected once every 2
If you set `[db].update every = 2` and `[db].retention = 1800`, you will still have an hour of data, but collected once every 2
seconds. This will **cut in half** both CPU and RAM resources consumed by Netdata. Of course experiment a bit. On very
weak devices you might have to use `update every = 5` and `history = 720` (still 1 hour of data, but 1/5 of the CPU and
weak devices you might have to use `[db].update every = 5` and `[db].retention = 720` (still 1 hour of data, but 1/5 of the CPU and
RAM resources).
You can also disable [data collection plugins](/collectors/README.md) you don't need. Disabling such plugins will also free both
CPU and RAM resources.
## Running a dedicated central Netdata server
## Running a dedicated parent Netdata server
Netdata allows streaming data between Netdata nodes. This allows us to have a central Netdata server that will maintain
the entire database for all nodes, and will also run health checks/alarms for all nodes.
For this central Netdata, memory size can be a problem. Fortunately, Netdata supports several memory modes. **One
interesting option** for this setup is `memory mode = map`.
Netdata allows streaming data between Netdata nodes in real-time. This allows having one or more parent Netdata servers that will maintain
the entire database for all the nodes that connect to them (their children), and will also run health checks/alarms for all these nodes.
### map
@ -77,8 +71,7 @@ in memory, but the kernel automatically loads and saves memory pages from/to dis
**We suggest _not_ to use this mode on nodes that run other applications.** There will always be dirty memory to be
synced and this syncing process may influence the way other applications work. This mode however is useful when we need
a central Netdata server that would normally need huge amounts of memory. Using memory mode `map` we can overcome all
memory restrictions.
a parent Netdata server that would normally need huge amounts of memory.
There are a few kernel options that provide finer control on the way this syncing works. But before explaining them, a
brief introduction of how Netdata database works is needed.
@ -142,8 +135,8 @@ vm.dirty_ratio = 90
vm.dirty_writeback_centisecs = 0
```
There is another memory mode to help overcome the memory size problem. What is **most interesting for this setup** is
`memory mode = dbengine`.
There is another mode to help overcome the memory size problem. What is **most interesting for this setup** is
`[db].mode = dbengine`.
### dbengine
@ -154,12 +147,12 @@ configured disk space and the effective compression ratio of the data stored.
We suggest to use **this** mode on nodes that also run other applications. The Database Engine uses direct I/O to avoid
polluting the OS filesystem caches and does not generate excessive I/O traffic so as to create the minimum possible
interference with other applications. Using memory mode `dbengine` we can overcome most memory restrictions. For more
interference with other applications. Using mode `dbengine` we can overcome most memory restrictions. For more
details see [here](/database/engine/README.md).
## KSM
Netdata offers all its round robin database to kernel for deduplication (except for `memory mode = dbengine`).
Netdata offers all its in-memory database to kernel for deduplication.
In the past KSM has been criticized for consuming a lot of CPU resources. Although this is true when KSM is used for
deduplicating certain applications, it is not true with netdata, since the Netdata memory is written very infrequently

65
database/engine/README.md
View file

@ -7,63 +7,62 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/database/engine/
# Database engine
The Database Engine works like a traditional database. It dedicates a certain amount of RAM to data caching and
indexing, while the rest of the data resides compressed on disk. Unlike other [memory modes](/database/README.md), the
indexing, while the rest of the data resides compressed on disk. Unlike other [database modes](/database/README.md), the
amount of historical metrics stored is based on the amount of disk space you allocate and the effective compression
ratio, not a fixed number of metrics collected.
By using both RAM and disk space, the database engine allows for long-term storage of per-second metrics inside of the
Agent itself.
In addition, the database engine is the only memory mode that supports changing the data collection update frequency
(`update_every`) without losing the metrics your Agent already gathered and stored.
In addition, the dbengine is the only mode that supports changing the data collection update frequency
(`update every`) without losing the metrics your Agent already gathered and stored.
## Configuration
To use the database engine, open `netdata.conf` and set `memory mode` to `dbengine`.
To use the database engine, open `netdata.conf` and set `[db].mode` to `dbengine`.
```conf
[global]
memory mode = dbengine
[db]
mode = dbengine
```
To configure the database engine, look for the `page cache size` and `dbengine multihost disk space` settings in the
`[global]` section of your `netdata.conf`. The Agent ignores the `history` setting when using the database engine.
To configure the database engine, look for the `page cache size MB` and `dbengine multihost disk space MB` settings in the
`[db]` section of your `netdata.conf`. The Agent ignores the `[db].retention` setting when using the dbengine.
```conf
[global]
page cache size = 32
dbengine multihost disk space = 256
[db]
page cache size MB = 32
dbengine multihost disk space MB = 256
```
The above values are the default values for Page Cache size and DB engine disk space quota. Both numbers are
in **MiB**.
The above values are the default values for Page Cache size and DB engine disk space quota.
The `page cache size` option determines the amount of RAM in **MiB** dedicated to caching Netdata metric values. The
The `page cache size MB` option determines the amount of RAM dedicated to caching Netdata metric values. The
actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements)
section for details.
The `dbengine multihost disk space` option determines the amount of disk space in **MiB** that is dedicated to storing
The `dbengine multihost disk space MB` option determines the amount of disk space that is dedicated to storing
Netdata metric values and all related metadata describing them. You can use the [**database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
to correctly set `dbengine multihost disk space` based on your metrics retention policy. The calculator gives an
to correctly set `dbengine multihost disk space MB` based on your metrics retention policy. The calculator gives an
accurate estimate based on how many child nodes you have, how many metrics your Agent collects, and more.
### Legacy configuration
The deprecated `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing
The deprecated `dbengine disk space MB` option determines the amount of disk space that is dedicated to storing
Netdata metric values per legacy database engine instance (see [details on the legacy mode](#legacy-mode) below).
```conf
[global]
dbengine disk space = 256
[db]
dbengine disk space MB = 256
```
### Streaming metrics to the database engine
When using the multihost database engine, all parent and child nodes share the same `page cache size` and `dbengine
multihost disk space` in a single dbengine instance. The [**database engine
When using the multihost database engine, all parent and child nodes share the same `page cache size MB` and `dbengine
multihost disk space MB` in a single dbengine instance. The [**database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
helps you properly set `page cache size` and `dbengine multihost disk space` on your parent node to allocate enough
helps you properly set `page cache size MB` and `dbengine multihost disk space MB` on your parent node to allocate enough
resources based on your metrics retention policy and how many child nodes you have.
#### Legacy mode
@ -72,8 +71,8 @@ _For Netdata Agents earlier than v1.23.2_, the Agent on the parent node uses one
another instance for every child node it receives metrics from. If you had four streaming nodes, you would have five
instances in total (`1 parent + 4 child nodes = 5 instances`).
The Agent allocates resources for each instance separately using the `dbengine disk space` (**deprecated**) setting. If
`dbengine disk space`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which
The Agent allocates resources for each instance separately using the `dbengine disk space MB` (**deprecated**) setting. If
`dbengine disk space MB`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which
means the total disk space required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.
#### Backward compatibility
@ -88,18 +87,18 @@ Agent.
##### Information
For more information about setting `memory mode` on your nodes, in addition to other streaming configurations, see
For more information about setting `[db].mode` on your nodes, in addition to other streaming configurations, see
[streaming](/streaming/README.md).
### Memory requirements
Using memory mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the
Using database mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the
available memory.
There are explicit memory requirements **per** DB engine **instance**:
- The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what
the user configured with `page cache size`.
the user configured with `page cache size MB`.
- an additional `#pages-on-disk x 4096 x 0.03` bytes of RAM are allocated for metadata.
@ -132,7 +131,7 @@ Netdata allocates 25% of the available file descriptors to its Database Engine i
the file descriptors that are available to the Netdata service are accessible by dbengine instances. You should take
that into account when configuring your service or system-wide file descriptor limits. You can roughly estimate that the
Netdata service needs 2048 file descriptors for every 10 streaming child hosts when streaming is configured to use
`memory mode = dbengine`.
`[db].mode = dbengine`.
If for example one wants to allocate 65536 file descriptors to the Netdata service on a systemd system one needs to
override the Netdata service by running `sudo systemctl edit netdata` and creating a file with contents:
@ -166,7 +165,7 @@ You can apply the settings by running `sysctl -p` or by rebooting.
## Files
With the DB engine memory mode the metric data are stored in database files. These files are organized in pairs, the
With the DB engine mode the metric data are stored in database files. These files are organized in pairs, the
datafiles and their corresponding journalfiles, e.g.:
```sh
@ -232,10 +231,10 @@ so as to avoid all disk bottlenecks.
The reported numbers are the following:
| device | page cache | dataset | reads/sec | writes/sec |
| :----: | :--------: | ------: | --------: | ---------: |
| HDD | 64 MiB | 4.1 GiB | 813K | 18.0M |
| SSD | 64 MiB | 9.8 GiB | 1.7M | 43.0M |
| N/A | 16 GiB | 6.8 GiB | 118.2M | 30.2M |
|:------:|:----------:|--------:|----------:|-----------:|
| HDD | 64 MiB | 4.1 GiB | 813K | 18.0M |
| SSD | 64 MiB | 9.8 GiB | 1.7M | 43.0M |
| N/A | 16 GiB | 6.8 GiB | 118.2M | 30.2M |
where "reads/sec" is the number of metric data points being read from the database via its API per second and
"writes/sec" is the number of metric data points being written to the database per second.

14
database/engine/rrdengine.c
View file

@ -1074,16 +1074,12 @@ struct rrdeng_cmd rrdeng_deq_cmd(struct rrdengine_worker_config* wc)
static void load_configuration_dynamic(void)
{
unsigned read_num;
static int printed_error = 0;
read_num = (unsigned) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine extent pages",
MAX_PAGES_PER_EXTENT);
if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT) {
unsigned read_num = (unsigned)config_get_number(CONFIG_SECTION_DB, "dbengine pages per extent", MAX_PAGES_PER_EXTENT);
if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT)
pages_per_extent = read_num;
} else if (!printed_error) {
printed_error = 1;
error("Invalid dbengine extent pages %u given. Defaulting to %u.", read_num, pages_per_extent);
else {
error("Invalid dbengine pages per extent %u given. Using %u.", read_num, pages_per_extent);
config_set_number(CONFIG_SECTION_DB, "dbengine pages per extent", pages_per_extent);
}
}

5
database/engine/rrdengineapi.c
View file

@ -195,7 +195,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
handle->descr = NULL;
}
void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number n, SN_FLAGS flags)
void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE n, SN_FLAGS flags)
{
storage_number number = pack_storage_number(n, flags);
@ -603,7 +603,8 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
// Returns the metric and sets its timestamp into current_time
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
NETDATA_DOUBLE
rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
struct rrdeng_page_descr *descr = handle->descr;

4
database/engine/rrdengineapi.h
View file

@ -42,14 +42,14 @@ extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN +
extern void rrdeng_metric_init(RRDDIM *rd);
extern void rrdeng_store_metric_init(RRDDIM *rd);
extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);
extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
extern int rrdeng_store_metric_finalize(RRDDIM *rd);
extern unsigned
rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
time_t start_time, time_t end_time);
extern calculated_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern NETDATA_DOUBLE rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
extern void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
extern time_t rrdeng_metric_latest_time(RRDDIM *rd);

66
database/metric_correlations.c
View file

@ -9,7 +9,7 @@ int metric_correlations_version = 1;
METRIC_CORRELATIONS_METHOD default_metric_correlations_method = METRIC_CORRELATIONS_KS2;
typedef struct mc_stats {
calculated_number max_base_high_ratio;
NETDATA_DOUBLE max_base_high_ratio;
size_t db_points;
size_t result_points;
size_t db_queries;
@ -58,7 +58,7 @@ struct register_result {
const char *chart_id;
const char *context;
const char *dim_name;
calculated_number value;
NETDATA_DOUBLE value;
};
static void register_result_insert_callback(const char *name, void *value, void *data) {
@ -93,11 +93,11 @@ static void register_result_destroy(DICTIONARY *results) {
dictionary_destroy(results);
}
static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, calculated_number value, RESULT_FLAGS flags, MC_STATS *stats) {
if(!calculated_number_isnumber(value)) return;
static void register_result(DICTIONARY *results, RRDSET *st, RRDDIM *d, NETDATA_DOUBLE value, RESULT_FLAGS flags, MC_STATS *stats) {
if(!netdata_double_isnumber(value)) return;
// make it positive
calculated_number v = calculated_number_fabs(value);
NETDATA_DOUBLE v = fabsndd(value);
// no need to store zero scored values
if(v == 0.0) return;
@ -186,7 +186,7 @@ static size_t registered_results_to_json(DICTIONARY *results, BUFFER *wb,
chart_dims = 0;
}
if (chart_dims) buffer_sprintf(wb, ",\n");
buffer_sprintf(wb, "\t\t\t\t\"%s\": " CALCULATED_NUMBER_FORMAT, t->dim_name, t->value);
buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value);
chart_dims++;
total_dimensions++;
}
@ -240,14 +240,14 @@ int compare_diffs(const void *left, const void *right) {
return (lt > rt) - (lt < rt);
}
static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, calculated_number *arr, size_t size) {
calculated_number *last = &arr[size - 1];
static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, size_t size) {
NETDATA_DOUBLE *last = &arr[size - 1];
size_t added = 0;
while(last > arr) {
calculated_number second = *last--;
calculated_number first = *last;
*diffs++ = (DIFFS_NUMBERS)((first - second) * (calculated_number)DOUBLE_TO_INT_MULTIPLIER);
NETDATA_DOUBLE second = *last--;
NETDATA_DOUBLE first = *last;
*diffs++ = (DIFFS_NUMBERS)((first - second) * (NETDATA_DOUBLE)DOUBLE_TO_INT_MULTIPLIER);
added++;
}
@ -356,7 +356,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
return KSfbar((int)en, d);
}
static double kstwo(calculated_number baseline[], int baseline_points, calculated_number highlight[], int highlight_points, uint32_t base_shifts) {
static double kstwo(
NETDATA_DOUBLE baseline[], int baseline_points,
NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) {
// -1 in size, since the calculate_pairs_diffs() returns one less point
DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
@ -469,14 +471,14 @@ static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
// copy the baseline points of the dimension to a contiguous array
// there is no need to check for empty values, since empty are already zero
calculated_number baseline[base_points];
NETDATA_DOUBLE baseline[base_points];
for(int c = 0; c < base_points; c++)
baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
// copy the highlight points of the dimension to a contiguous array
// there is no need to check for empty values, since empty values are already zero
// https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
calculated_number highlight[high_points];
NETDATA_DOUBLE highlight[high_points];
for(int c = 0; c < high_points; c++)
highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
@ -537,7 +539,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
// dimensions, and we query a single dimension at a time.
stats->db_queries++;
calculated_number baseline_average = NAN;
NETDATA_DOUBLE baseline_average = NAN;
uint8_t base_anomaly_rate = 0;
value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
@ -547,13 +549,13 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points,
&value_is_null, &base_anomaly_rate, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(baseline_average)) {
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
// this means no data for the baseline window, but we may have data for the highlighted one - assume zero
baseline_average = 0.0;
}
stats->db_queries++;
calculated_number highlight_average = NAN;
NETDATA_DOUBLE highlight_average = NAN;
uint8_t high_anomaly_rate = 0;
value_is_null = 1;
ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
@ -563,7 +565,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points,
&value_is_null, &high_anomaly_rate, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_average)) {
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
// this means no data for the highlighted duration - so skip it
continue;
}
@ -574,11 +576,11 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
}
stats->db_queries++;
calculated_number highlight_countif = NAN;
NETDATA_DOUBLE highlight_countif = NAN;
value_is_null = 1;
char highlighted_countif_options[50 + 1];
snprintfz(highlighted_countif_options, 50, "%s" CALCULATED_NUMBER_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,
after, before,
@ -588,7 +590,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
&stats->db_points, &stats->result_points,
&value_is_null, NULL, 0);
if(ret != HTTP_RESP_OK || value_is_null || !calculated_number_isnumber(highlight_countif)) {
if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
info("MC: highlighted countif query failed, but highlighted average worked - strange...");
continue;
}
@ -599,7 +601,7 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0
RESULT_FLAGS flags;
calculated_number pcent = NAN;
NETDATA_DOUBLE pcent = NAN;
if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {
flags = RESULT_IS_BASE_HIGH_RATIO;
pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;
@ -615,15 +617,15 @@ static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
return correlated_dimensions;
}
int compare_calculated_numbers(const void *left, const void *right) {
calculated_number lt = *(calculated_number *)left;
calculated_number rt = *(calculated_number *)right;
int compare_netdata_doubles(const void *left, const void *right) {
NETDATA_DOUBLE lt = *(NETDATA_DOUBLE *)left;
NETDATA_DOUBLE rt = *(NETDATA_DOUBLE *)right;
// https://stackoverflow.com/a/3886497/1114110
return (lt > rt) - (lt < rt);
}
static inline int binary_search_bigger_than_calculated_number(const calculated_number arr[], int left, int size, calculated_number K) {
static inline int binary_search_bigger_than_netdata_double(const NETDATA_DOUBLE arr[], int left, int size, NETDATA_DOUBLE K) {
// binary search to find the index the smallest index
// of the first value in the array that is greater than K
@ -655,7 +657,7 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
stats->max_base_high_ratio = 1.0;
// create an array of the right size and copy all the values in it
calculated_number slots[dimensions];
NETDATA_DOUBLE slots[dimensions];
dimensions = 0;
dfe_start_read(results, t) {
if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME))
@ -666,10 +668,10 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
dfe_done(t);
// sort the array with the values of all dimensions
qsort(slots, dimensions, sizeof(calculated_number), compare_calculated_numbers);
qsort(slots, dimensions, sizeof(NETDATA_DOUBLE), compare_netdata_doubles);
// skip the duplicates in the sorted array
calculated_number last_value = NAN;
NETDATA_DOUBLE last_value = NAN;
size_t unique_values = 0;
for(size_t i = 0; i < dimensions ;i++) {
if(likely(slots[i] != last_value))
@ -681,11 +683,11 @@ static size_t spread_results_evenly(DICTIONARY *results, MC_STATS *stats) {
unique_values = dimensions;
// calculate the weight of each slot, using the number of unique values
calculated_number slot_weight = 1.0 / (calculated_number)unique_values;
NETDATA_DOUBLE slot_weight = 1.0 / (NETDATA_DOUBLE)unique_values;
dfe_start_read(results, t) {
int slot = binary_search_bigger_than_calculated_number(slots, 0, (int)unique_values, t->value);
calculated_number v = slot * slot_weight;
int slot = binary_search_bigger_than_netdata_double(slots, 0, (int)unique_values, t->value);
NETDATA_DOUBLE v = slot * slot_weight;
if(unlikely(v > 1.0)) v = 1.0;
v = 1.0 - v;
t->value = v;

11
database/ram/rrddim_mem.c
View file

@ -6,12 +6,12 @@
// RRDDIM legacy data collection functions
void rrddim_collect_init(RRDDIM *rd) {
rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT;
rd->db[rd->rrdset->current_entry] = SN_EMPTY_SLOT;
rd->state->handle = calloc(1, sizeof(struct mem_collect_handle));
}
void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags) {
void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags) {
(void)point_in_time;
rd->values[rd->rrdset->current_entry] = pack_storage_number(number, flags);
rd->db[rd->rrdset->current_entry] = pack_storage_number(number, flags);
}
int rrddim_collect_finalize(RRDDIM *rd) {
free((struct mem_collect_handle*)rd->state->handle);
@ -42,7 +42,8 @@ void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t st
// Returns the metric and sets its timestamp into current_time
// IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
// IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
NETDATA_DOUBLE
rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags) {
RRDDIM *rd = handle->rd;
struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
size_t entries = rd->rrdset->entries;
@ -65,7 +66,7 @@ calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, t
return NAN;
}
storage_number n = rd->values[slot++];
storage_number n = rd->db[slot++];
if(unlikely(slot >= entries)) slot = 0;
h->slot = slot;

5
database/ram/rrddim_mem.h
View file

@ -20,11 +20,12 @@ struct mem_query_handle {
};
extern void rrddim_collect_init(RRDDIM *rd);
extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);
extern void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
extern int rrddim_collect_finalize(RRDDIM *rd);
extern void rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
extern calculated_number rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern NETDATA_DOUBLE
rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
extern int rrddim_query_is_finished(struct rrddim_query_handle *handle);
extern void rrddim_query_finalize(struct rrddim_query_handle *handle);
extern time_t rrddim_query_latest_time(RRDDIM *rd);

94
database/rrd.h
View file

@ -74,9 +74,6 @@ extern time_t rrdset_free_obsolete_time;
#define RRD_ID_LENGTH_MAX 200
#define RRDSET_MAGIC "NETDATA RRD SET FILE V019"
#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V019"
typedef long long total_number;
#define TOTAL_NUMBER_FORMAT "%lld"
@ -237,52 +234,44 @@ struct rrddim {
// this is a pointer to the config structure
// since the config always has a higher priority
// (the user overwrites the name of the charts)
// DO NOT FREE THIS - IT IS ALLOCATED IN CONFIG
uint32_t hash; // a simple hash of the id, to speed up searching / indexing
// instead of strcmp() every item in the binary index
// we first compare the hashes
uint32_t hash_name; // a simple hash of the name
RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
RRDDIM_FLAGS flags; // configuration flags for the dimension
unsigned int updated:1; // 1 when the dimension has been updated since the last processing
unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
collected_number multiplier; // the multiplier of the collected values
collected_number divisor; // the divider of the collected values
uint32_t flags; // configuration flags for the dimension
// ------------------------------------------------------------------------
// members for temporary data we need for calculations
uint32_t hash; // a simple hash of the id, to speed up searching / indexing
// instead of strcmp() every item in the binary index
// we first compare the hashes
uint32_t hash_name; // a simple hash of the name
char *cache_filename; // the filename we load/save from/to this set
size_t collections_counter; // the number of times we added values to this rrdim
struct rrddim_volatile *state; // volatile state that is not persistently stored
size_t unused[8];
collected_number collected_value_max; // the absolute maximum of the collected value
unsigned int updated:1; // 1 when the dimension has been updated since the last processing
unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
struct timeval last_collected_time; // when was this dimension last updated
// this is actual date time we updated the last_collected_value
// THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
calculated_number calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
calculated_number last_calculated_value; // the last calculated value processed
struct rrddim_volatile *state; // volatile state that is not persistently stored
size_t collections_counter; // the number of times we added values to this rrdim
collected_number collected_value_max; // the absolute maximum of the collected value
calculated_number last_stored_value; // the last value as stored in the database (after interpolation)
NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
collected_number collected_value; // the current value, as collected - resets to 0 after being used
collected_number last_collected_value; // the last value that was collected, after being processed
// the *_volume members are used to calculate the accuracy of the rounding done by the
// storage number - they are printed to debug.log when debug is enabled for a set.
calculated_number collected_volume; // the sum of all collected values so far
calculated_number stored_volume; // the sum of all stored values so far
NETDATA_DOUBLE collected_volume; // the sum of all collected values so far
NETDATA_DOUBLE stored_volume; // the sum of all stored values so far
struct rrddim *next; // linking of dimensions within the same data set
struct rrdset *rrdset;
@ -296,18 +285,33 @@ struct rrddim {
int update_every; // every how many seconds is this updated
size_t memsize; // the memory allocated for this dimension
char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file
size_t memsize; // the memory allocated for this dimension (without RRDDIM)
struct rrddimvar *variables;
// ------------------------------------------------------------------------
// the values stored in this dimension, using our floating point numbers
storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER
void *rd_on_file; // pointer to the header written on disk
storage_number *db; // the array of values
};
// returns the RRDDIM cache filename, or NULL if it does not exist
extern const char *rrddim_cache_filename(RRDDIM *rd);
// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
extern void rrddim_memory_file_update(RRDDIM *rd);
// free the memory file structures for memory mode MAP and SAVE
extern void rrddim_memory_file_free(RRDDIM *rd);
extern bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
// return the v019 header size of RRDDIM files
extern size_t rrddim_memory_file_header_size(void);
extern void rrddim_memory_file_save(RRDDIM *rd);
// ----------------------------------------------------------------------------
// engine-specific iterator state for dimension data collection
typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
@ -332,7 +336,7 @@ struct rrddim_collect_ops {
void (*init)(RRDDIM *rd);
// run this to store each metric into the database
void (*store_metric)(RRDDIM *rd, usec_t point_in_time, calculated_number number, SN_FLAGS flags);
void (*store_metric)(RRDDIM *rd, usec_t point_in_time, NETDATA_DOUBLE number, SN_FLAGS flags);
// an finalization function to run after collection is over
// returns 1 if it's safe to delete the dimension
@ -345,7 +349,7 @@ struct rrddim_query_ops {
void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
// run this to load each metric number from the database
calculated_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags);
NETDATA_DOUBLE (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time, time_t *end_time, SN_FLAGS *flags);
// run this to test if the series of next_metric() database queries is finished
int (*is_finished)(struct rrddim_query_handle *handle);
@ -450,8 +454,6 @@ struct rrdset {
// since the config always has a higher priority
// (the user overwrites the name of the charts)
void *unused_ptr; // Unused field (previously it held the config section of the chart)
char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
char *family; // grouping sets under the same family
char *title; // title shown to user
@ -484,7 +486,6 @@ struct rrdset {
RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped
char *cache_dir; // the directory to store dimensions
char cache_filename[FILENAME_MAX+1]; // the filename to store this set
netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list
@ -502,7 +503,6 @@ struct rrdset {
uuid_t *chart_uuid; // Store the global GUID for this chart
// this object.
struct rrdset_volatile *state; // volatile state that is not persistently stored
size_t unused[3];
size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
@ -527,8 +527,8 @@ struct rrdset {
// ------------------------------------------------------------------------
// local variables
calculated_number green; // green threshold for this chart
calculated_number red; // red threshold for this chart
NETDATA_DOUBLE green; // green threshold for this chart
NETDATA_DOUBLE red; // red threshold for this chart
avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart
RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs)
@ -538,15 +538,13 @@ struct rrdset {
// members for checking the data when loading from disk
unsigned long memsize; // how much mem we have allocated for this (without dimensions)
char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic
void *st_on_file; // compatibility with V019 RRDSET files
// ------------------------------------------------------------------------
// the dimensions
avl_tree_lock dimensions_index; // the root of the dimensions index
RRDDIM *dimensions; // the actual data for every dimension
};
#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
@ -564,6 +562,12 @@ struct rrdset {
for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next)
extern void rrdset_memory_file_save(RRDSET *st);
extern void rrdset_memory_file_free(RRDSET *st);
extern void rrdset_memory_file_update(RRDSET *st);
extern const char *rrdset_cache_filename(RRDSET *st);
extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
// ----------------------------------------------------------------------------
// RRDHOST flags
// use this for configuration flags, not for state control
@ -630,8 +634,8 @@ struct alarm_entry {
char *units;
char *info;
calculated_number old_value;
calculated_number new_value;
NETDATA_DOUBLE old_value;
NETDATA_DOUBLE new_value;
char *old_value_string;
char *new_value_string;

10
database/rrdcalc.c
View file

@ -57,12 +57,14 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
}
if(!isnan(rc->green) && isnan(st->green)) {
debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " CALCULATED_NUMBER_FORMAT_AUTO " to " CALCULATED_NUMBER_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from " NETDATA_DOUBLE_FORMAT_AUTO
" to " NETDATA_DOUBLE_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
st->green = rc->green;
}
if(!isnan(rc->red) && isnan(st->red)) {
debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " CALCULATED_NUMBER_FORMAT_AUTO " to " CALCULATED_NUMBER_FORMAT_AUTO ".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from " NETDATA_DOUBLE_FORMAT_AUTO " to " NETDATA_DOUBLE_FORMAT_AUTO
".", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
st->red = rc->red;
}
@ -445,7 +447,9 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
}
debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
(rc->chart)?rc->chart:"NOCHART",
rc->name,
(rc->exec)?rc->exec:"DEFAULT",

8
database/rrdcalc.h
View file

@ -63,8 +63,8 @@ struct rrdcalc {
int update_every; // update frequency for the alarm
// the red and green threshold of this alarm (to be set to the chart)
calculated_number green;
calculated_number red;
NETDATA_DOUBLE green;
NETDATA_DOUBLE red;
// ------------------------------------------------------------------------
// database lookup settings
@ -112,8 +112,8 @@ struct rrdcalc {
RRDCALC_STATUS old_status; // the old status of the alarm
RRDCALC_STATUS status; // the current status of the alarm
calculated_number value; // the current value of the alarm
calculated_number old_value; // the previous value of the alarm
NETDATA_DOUBLE value; // the current value of the alarm
NETDATA_DOUBLE old_value; // the previous value of the alarm
uint32_t rrdcalc_flags; // check RRDCALC_FLAG_*

4
database/rrdcalctemplate.h
View file

@ -42,8 +42,8 @@ struct rrdcalctemplate {
int update_every; // update frequency for the alarm
// the red and green threshold of this alarm (to be set to the chart)
calculated_number green;
calculated_number red;
NETDATA_DOUBLE green;
NETDATA_DOUBLE red;
// ------------------------------------------------------------------------
// database lookup settings

349
database/rrddim.c
View file

@ -197,112 +197,15 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
char filename[FILENAME_MAX + 1];
char fullfilename[FILENAME_MAX + 1];
unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number));
debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
rrdset_strncpyz_name(filename, id, FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
memory_mode == RRD_MEMORY_MODE_RAM) {
rd = (RRDDIM *)netdata_mmap(
(memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
1);
if(likely(rd)) {
// we have a file mapped for rd
memset(&rd->avl, 0, sizeof(avl_t));
rd->id = NULL;
rd->name = NULL;
rd->cache_filename = NULL;
rd->variables = NULL;
rd->next = NULL;
rd->rrdset = NULL;
rd->exposed = 0;
struct timeval now;
now_realtime_timeval(&now);
if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(rd, 0, size);
}
else {
int reset = 0;
if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
info("Initializing file %s.", fullfilename);
memset(rd, 0, size);
reset = 1;
}
else if(rd->memsize != size) {
error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd->memsize);
memset(rd, 0, size);
reset = 1;
}
else if(rd->update_every != st->update_every) {
error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd->update_every);
memset(rd, 0, size);
reset = 1;
}
else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) {
info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every);
memset(rd, 0, size);
reset = 1;
}
if(!reset) {
if(rd->algorithm != algorithm) {
info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, rrd_algorithm_name(rd->algorithm));
}
if(rd->multiplier != multiplier) {
info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, multiplier, rd->multiplier);
}
if(rd->divisor != divisor) {
info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, divisor, rd->divisor);
}
}
}
// make sure we have the right memory mode
// even if we cleared the memory
rd->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!rd)) {
// if we didn't manage to get a mmap'd dimension, just create one
rd = callocz(1, size);
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
else
rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
rd->memsize = size;
strcpy(rd->magic, RRDDIMENSION_MAGIC);
rd = callocz(1, sizeof(RRDDIM));
rd->id = strdupz(id);
rd->hash = simple_hash(rd->id);
rd->cache_filename = strdupz(fullfilename);
rd->name = (name && *name)?strdupz(name):strdupz(rd->id);
rd->hash_name = simple_hash(rd->name);
rd->algorithm = algorithm;
rd->multiplier = multiplier;
rd->divisor = divisor;
if(!rd->divisor) rd->divisor = 1;
@ -311,23 +214,38 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
rd->collections_counter = 1;
else
rd->collections_counter = 0;
rd->updated = 0;
rd->flags = 0x00000000;
rd->calculated_value = 0;
rd->last_calculated_value = 0;
rd->collected_value = 0;
rd->last_collected_value = 0;
rd->collected_value_max = 0;
rd->collected_volume = 0;
rd->stored_volume = 0;
rd->last_stored_value = 0;
rd->last_collected_time.tv_sec = 0;
rd->last_collected_time.tv_usec = 0;
rd->rrdset = st;
if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) {
if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name, rd->name);
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
if(memory_mode == RRD_MEMORY_MODE_RAM) {
size_t entries = st->entries;
if(!entries) entries = 5;
rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
if(!rd->db) {
info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", st->name, rd->name);
memory_mode = RRD_MEMORY_MODE_ALLOC;
}
else rd->memsize = entries * sizeof(storage_number);
}
if(memory_mode == RRD_MEMORY_MODE_ALLOC || memory_mode == RRD_MEMORY_MODE_NONE) {
size_t entries = st->entries;
if(entries < 5) entries = 5;
rd->db = callocz(entries, sizeof(storage_number));
rd->memsize = entries * sizeof(storage_number);
}
rd->rrd_memory_mode = memory_mode;
rd->state = callocz(1, sizeof(*rd->state));
#ifdef ENABLE_ACLK
rd->state->aclk_live_status = -1;
@ -432,30 +350,21 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
// aclk_send_dimension_update(rd);
//#endif
RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
switch(rrd_memory_mode) {
case RRD_MEMORY_MODE_SAVE:
case RRD_MEMORY_MODE_MAP:
case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
freez((void *)rd->id);
freez((void *)rd->name);
freez(rd->cache_filename);
freez(rd->state);
munmap(rd, rd->memsize);
break;
freez((void *)rd->id);
freez((void *)rd->name);
freez(rd->state);
case RRD_MEMORY_MODE_ALLOC:
case RRD_MEMORY_MODE_NONE:
case RRD_MEMORY_MODE_DBENGINE:
debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
freez((void *)rd->id);
freez((void *)rd->name);
freez(rd->cache_filename);
freez(rd->state);
freez(rd);
break;
// this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
rrddim_memory_file_free(rd);
if(rd->db) {
if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
munmap(rd->db, rd->memsize);
else
freez(rd->db);
}
freez(rd);
}
@ -529,7 +438,7 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM
collected_number v = (value >= 0) ? value : -value;
if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v;
// fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " CALCULATED_NUMBER_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (calculated_number)((value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor * 1000000.0 / (calculated_number)st->usec_since_last_update));
// fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " NETDATA_DOUBLE_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (NETDATA_DOUBLE)((value - rd->last_collected_value) * (NETDATA_DOUBLE)rd->multiplier / (NETDATA_DOUBLE)rd->divisor * 1000000.0 / (NETDATA_DOUBLE)st->usec_since_last_update));
return rd->last_collected_value;
}
@ -544,3 +453,171 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
return rrddim_set_by_pointer(st, rd, value);
}
// ----------------------------------------------------------------------------
// compatibility layer for RRDDIM files v019
#define RRDDIMENSION_MAGIC_V019 "NETDATA RRD DIMENSION FILE V019"
struct avl_element_v019 {
void *avl_link[2];
signed char avl_balance;
};
struct rrddim_map_save_v019 {
struct avl_element_v019 avl; // ignored
void *id; // ignored
void *name; // ignored
uint32_t algorithm; // print warning on mismatch - update on load
uint32_t rrd_memory_mode; // ignored
long long multiplier; // print warning on mismatch - update on load
long long divisor; // print warning on mismatch - update on load
uint32_t flags; // ignored
uint32_t hash; // ignored
uint32_t hash_name; // ignored
void *cache_filename; // ignored - we use it to keep the filename to save back
size_t collections_counter; // ignored
void *state; // ignored
size_t unused[8]; // ignored
long long collected_value_max; // ignored
unsigned int updated:1; // ignored
unsigned int exposed:1; // ignored
struct timeval last_collected_time; // check to reset all - ignored after load
long double calculated_value; // ignored
long double last_calculated_value; // ignored
long double last_stored_value; // ignored
long long collected_value; // ignored
long long last_collected_value; // ignored
long double collected_volume; // ignored
long double stored_volume; // ignored
void *next; // ignored
void *rrdset; // ignored
long entries; // check to reset all - update on load
int update_every; // check to reset all - update on load
size_t memsize; // check to reset all - update on load
char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
void *variables; // ignored
storage_number values[]; // the array of values
};
size_t rrddim_memory_file_header_size(void) {
return sizeof(struct rrddim_map_save_v019);
}
void rrddim_memory_file_update(RRDDIM *rd) {
if(!rd->rd_on_file) return;
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
}
void rrddim_memory_file_free(RRDDIM *rd) {
if(!rd->rd_on_file) return;
// needed for memory mode map, to save the latest state
rrddim_memory_file_update(rd);
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
freez(rd_on_file->cache_filename);
munmap(rd_on_file, rd_on_file->memsize);
// remove the pointers from the RRDDIM
rd->rd_on_file = NULL;
rd->db = NULL;
}
const char *rrddim_cache_filename(RRDDIM *rd) {
if(!rd->rd_on_file) return NULL;
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
return rd_on_file->cache_filename;
}
void rrddim_memory_file_save(RRDDIM *rd) {
if(!rd->rd_on_file) return;
rrddim_memory_file_update(rd);
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
}
bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) {
if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
return false;
struct rrddim_map_save_v019 *rd_on_file = NULL;
unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->entries * sizeof(storage_number));
char filename[FILENAME_MAX + 1];
char fullfilename[FILENAME_MAX + 1];
rrdset_strncpyz_name(filename, rd->id, FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
if(unlikely(!rd_on_file)) return false;
struct timeval now;
now_realtime_timeval(&now);
int reset = 0;
if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
info("Initializing file %s.", fullfilename);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(rd_on_file->memsize != size) {
error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd_on_file->memsize);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(rd_on_file->update_every != st->update_every) {
error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
memset(rd_on_file, 0, size);
reset = 1;
}
else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
memset(rd_on_file, 0, size);
reset = 1;
}
if(!reset) {
if(rd_on_file->algorithm != rd->algorithm)
info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
if(rd_on_file->multiplier != rd->multiplier)
info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->multiplier, rd_on_file->multiplier);
if(rd_on_file->divisor != rd->divisor)
info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->divisor, rd_on_file->divisor);
}
// zero the entire header
memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
// set the important fields
strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
rd_on_file->algorithm = rd->algorithm;
rd_on_file->multiplier = rd->multiplier;
rd_on_file->divisor = rd->divisor;
rd_on_file->entries = st->entries;
rd_on_file->update_every = rd->update_every;
rd_on_file->memsize = size;
rd_on_file->rrd_memory_mode = memory_mode;
rd_on_file->cache_filename = strdupz(fullfilename);
rd->db = &rd_on_file->values[0];
rd->rd_on_file = rd_on_file;
rd->memsize = size;
rrddim_memory_file_update(rd);
return true;
}

16
database/rrdhost.c
View file

@ -210,10 +210,10 @@ RRDHOST *rrdhost_create(const char *hostname,
avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);
avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1))
if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1))
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost)
if(config_get_boolean(CONFIG_SECTION_DB, "delete orphan hosts files", 1) && !is_localhost)
rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST);
host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
@ -689,17 +689,21 @@ restart_after_removal:
// RRDHOST global / startup initialization
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time);
rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
// Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
// cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
// https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
if (rrdset_free_obsolete_time < 10) {
rrdset_free_obsolete_time = 10;
info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds. A lower delay can potentially cause a segmentation fault.");
info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
}
gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above);
if (gap_when_lost_iterations_above < 1)
gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
if (gap_when_lost_iterations_above < 1) {
gap_when_lost_iterations_above = 1;
config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
}
if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) {
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

616
database/rrdset.c
View file

@ -80,17 +80,9 @@ static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name, ui
tmp.name = name;
tmp.hash_name = (hash)?hash:simple_hash(tmp.name);
// fprintf(stderr, "SEARCHING: %s\n", name);
result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname)));
if(result) {
RRDSET *st = rrdset_from_avlname(result);
if(strcmp(st->magic, RRDSET_MAGIC) != 0)
error("Search for RRDSET %s returned an invalid RRDSET %s (name %s)", name, st->id, st->name);
if(result) return rrdset_from_avlname(result);
// fprintf(stderr, "FOUND: %s\n", name);
return rrdset_from_avlname(result);
}
// fprintf(stderr, "NOT FOUND: %s\n", name);
return NULL;
}
@ -294,20 +286,27 @@ void rrdset_reset(RRDSET *st) {
// RRDSET - helpers for rrdset_create()
inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
if(unlikely(entries < 5)) entries = 5;
if(unlikely(entries > RRD_HISTORY_ENTRIES_MAX)) entries = RRD_HISTORY_ENTRIES_MAX;
if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
if(mode == RRD_MEMORY_MODE_NONE) return 5;
if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_ALLOC))
return entries;
if(entries < 5) entries = 5;
if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
long page = (size_t)sysconf(_SC_PAGESIZE);
long size = sizeof(RRDDIM) + entries * sizeof(storage_number);
if(unlikely(size % page)) {
size -= (size % page);
size += page;
if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE || mode == RRD_MEMORY_MODE_RAM) {
long header_size = 0;
long n = (size - sizeof(RRDDIM)) / sizeof(storage_number);
return n;
if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE)
header_size = (long)rrddim_memory_file_header_size();
long page = (long)sysconf(_SC_PAGESIZE);
long size = (long)(header_size + entries * sizeof(storage_number));
if (unlikely(size % page)) {
size -= (size % page);
size += page;
long n = (long)((size - header_size) / sizeof(storage_number));
return n;
}
}
return entries;
@ -405,40 +404,18 @@ void rrdset_free(RRDSET *st) {
freez(st->state);
freez(st->chart_uuid);
switch(st->rrd_memory_mode) {
case RRD_MEMORY_MODE_SAVE:
case RRD_MEMORY_MODE_MAP:
case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
munmap(st, st->memsize);
break;
case RRD_MEMORY_MODE_ALLOC:
case RRD_MEMORY_MODE_NONE:
case RRD_MEMORY_MODE_DBENGINE:
freez(st);
break;
}
rrdset_memory_file_free(st);
freez(st);
}
void rrdset_save(RRDSET *st) {
rrdset_check_rdlock(st);
// info("Saving chart '%s' ('%s')", st->id, st->name);
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename);
memory_file_save(st->cache_filename, st, st->memsize);
}
rrdset_memory_file_save(st);
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) {
debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
memory_file_save(rd->cache_filename, rd, rd->memsize);
}
}
rrddim_foreach_read(rd, st)
rrddim_memory_file_save(rd);
}
void rrdset_delete(RRDSET *st) {
@ -448,17 +425,23 @@ void rrdset_delete(RRDSET *st) {
info("Deleting chart '%s' ('%s') from disk...", st->id, st->name);
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
info("Deleting chart header file '%s'.", st->cache_filename);
if(unlikely(unlink(st->cache_filename) == -1))
error("Cannot delete chart header file '%s'", st->cache_filename);
const char *cache_filename = rrdset_cache_filename(st);
if(cache_filename) {
info("Deleting chart header file '%s'.", cache_filename);
if (unlikely(unlink(cache_filename) == -1))
error("Cannot delete chart header file '%s'", cache_filename);
}
else
error("Cannot find the cache filename of chart '%s'", st->id);
}
rrddim_foreach_read(rd, st) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
info("Deleting dimension file '%s'.", rd->cache_filename);
if(unlikely(unlink(rd->cache_filename) == -1))
error("Cannot delete dimension file '%s'", rd->cache_filename);
}
const char *cache_filename = rrddim_cache_filename(rd);
if(!cache_filename) continue;
info("Deleting dimension file '%s'.", cache_filename);
if(unlikely(unlink(cache_filename) == -1))
error("Cannot delete dimension file '%s'", cache_filename);
}
recursively_delete_dir(st->cache_dir, "left-over chart");
@ -473,11 +456,11 @@ void rrdset_delete_obsolete_dimensions(RRDSET *st) {
rrddim_foreach_read(rd, st) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
info("Deleting dimension file '%s'.", rd->cache_filename);
if(unlikely(unlink(rd->cache_filename) == -1))
error("Cannot delete dimension file '%s'", rd->cache_filename);
}
const char *cache_filename = rrddim_cache_filename(rd);
if(!cache_filename) continue;
info("Deleting dimension file '%s'.", cache_filename);
if(unlikely(unlink(cache_filename) == -1))
error("Cannot delete dimension file '%s'", cache_filename);
}
}
}
@ -699,8 +682,6 @@ RRDSET *rrdset_create_custom(
return st;
}
char fullfilename[FILENAME_MAX + 1];
// ------------------------------------------------------------------------
// get the options from the config, we need to create it
@ -708,126 +689,37 @@ RRDSET *rrdset_create_custom(
if (memory_mode != RRD_MEMORY_MODE_DBENGINE)
entries = align_entries_to_pagesize(memory_mode, history_entries);
unsigned long size = sizeof(RRDSET);
char *cache_dir = rrdset_cache_dir(host, fullid);
time_t now = now_realtime_sec();
// ------------------------------------------------------------------------
// load it or allocate it
debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
memory_mode == RRD_MEMORY_MODE_RAM) {
st = (RRDSET *)netdata_mmap(
(memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
0);
if(st) {
memset(&st->avl, 0, sizeof(avl_t));
memset(&st->avlname, 0, sizeof(avl_t));
memset(&st->rrdvar_root_index, 0, sizeof(avl_tree_lock));
memset(&st->dimensions_index, 0, sizeof(avl_tree_lock));
memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t));
st->name = NULL;
st->type = NULL;
st->family = NULL;
st->title = NULL;
st->units = NULL;
st->context = NULL;
st->cache_dir = NULL;
st->plugin_name = NULL;
st->module_name = NULL;
st->dimensions = NULL;
st->rrdfamily = NULL;
st->rrdhost = NULL;
st->next = NULL;
st->variables = NULL;
st->alarms = NULL;
st->flags = 0x00000000;
st->exporting_flags = NULL;
if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(st, 0, size);
}
else {
if(strcmp(st->magic, RRDSET_MAGIC) != 0) {
info("Initializing file %s.", fullfilename);
memset(st, 0, size);
}
else if(strcmp(st->id, fullid) != 0) {
error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
// munmap(st, size);
// st = NULL;
memset(st, 0, size);
}
else if(st->memsize != size || st->entries != entries) {
error("File %s does not have the desired size. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if(st->update_every != update_every) {
error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if((now - st->last_updated.tv_sec) > update_every * entries) {
info("File %s is too old. Clearing it.", fullfilename);
memset(st, 0, size);
}
else if(st->last_updated.tv_sec > now + update_every) {
error("File %s refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st->last_updated.tv_sec - now));
st->last_updated.tv_sec = now;
}
// make sure the database is aligned
if(st->last_updated.tv_sec) {
st->update_every = update_every;
last_updated_time_align(st);
}
}
// make sure we have the right memory mode
// even if we cleared the memory
st->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!st)) {
st = callocz(1, size);
if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
st->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
else
st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
st->rrdhost = host;
st->memsize = size;
st->entries = entries;
st->update_every = update_every;
if(st->current_entry >= st->entries) st->current_entry = 0;
strcpy(st->cache_filename, fullfilename);
strcpy(st->magic, RRDSET_MAGIC);
st = callocz(1, sizeof(RRDSET));
st->state = callocz(1, sizeof(*st->state));
strcpy(st->id, fullid);
st->hash = simple_hash(st->id);
st->rrdhost = host;
st->cache_dir = cache_dir;
st->entries = entries;
st->update_every = update_every;
st->chart_type = chart_type;
st->type = strdupz(type);
if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP) {
if(!rrdset_memory_load_or_create_map_save(st, memory_mode)) {
info("Failed to use memory mode %s for chart '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name);
memory_mode = RRD_MEMORY_MODE_RAM;
}
}
st->rrd_memory_mode = memory_mode;
st->state = callocz(1, sizeof(*st->state));
st->family = family ? strdupz(family) : strdupz(st->type);
st->plugin_name = plugin?strdupz(plugin):NULL;
st->module_name = module?strdupz(module):NULL;
st->chart_type = chart_type;
st->type = strdupz(type);
st->family = family ? strdupz(family) : strdupz(st->type);
json_fix_string(st->family);
st->state->is_ar_chart = strcmp(st->id, ML_ANOMALY_RATES_CHART_ID) == 0;
@ -848,16 +740,8 @@ RRDSET *rrdset_create_custom(
st->green = NAN;
st->red = NAN;
st->last_collected_time.tv_sec = 0;
st->last_collected_time.tv_usec = 0;
st->counter_done = 0;
st->rrddim_page_alignment = 0;
st->gap_when_lost_iterations_above = (int) (gap_when_lost_iterations_above + 2);
st->last_accessed_time = 0;
st->upstream_resync_time = 0;
avl_init_lock(&st->dimensions_index, rrddim_compare);
avl_init_lock(&st->rrdvar_root_index, rrdvar_compare);
@ -963,7 +847,8 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
if(unlikely(since_last_usec < 0)) {
// oops! the database is in the future
#ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" LONG_DOUBLE_MODIFIER " secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (LONG_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the future (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif
st->last_collected_time.tv_sec = now.tv_sec - st->update_every;
@ -982,7 +867,8 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) {
// oops! the database is too far behind
#ifdef NETDATA_INTERNAL_CHECKS
info("RRD database for chart '%s' on host '%s' is %0.5" LONG_DOUBLE_MODIFIER " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (LONG_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER
" secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
#endif
microseconds = (usec_t)since_last_usec;
@ -1039,7 +925,7 @@ static inline usec_t rrdset_init_last_collected_time(RRDSET *st) {
usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "initialized last collected time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_collect_ut / USEC_PER_SEC);
rrdset_debug(st, "initialized last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
#endif
return last_collect_ut;
@ -1052,7 +938,7 @@ static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "updated last collected time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_collect_ut / USEC_PER_SEC);
rrdset_debug(st, "updated last collected time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_collect_ut / USEC_PER_SEC);
#endif
return last_collect_ut;
@ -1071,7 +957,7 @@ static inline usec_t rrdset_init_last_updated_time(RRDSET *st) {
usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "initialized last updated time to %0.3" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)last_updated_ut / USEC_PER_SEC);
rrdset_debug(st, "initialized last updated time to %0.3" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)last_updated_ut / USEC_PER_SEC);
#endif
return last_updated_ut;
@ -1109,8 +995,8 @@ static inline size_t rrdset_done_interpolate(
#ifdef NETDATA_INTERNAL_CHECKS
if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
rrdset_debug(st, "last_stored_ut = %0.3" LONG_DOUBLE_MODIFIER " (last updated time)", (LONG_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" LONG_DOUBLE_MODIFIER " (next interpolation point)", (LONG_DOUBLE)next_store_ut/USEC_PER_SEC);
rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
#endif
last_ut = next_store_ut;
@ -1119,20 +1005,19 @@ static inline size_t rrdset_done_interpolate(
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue;
calculated_number new_value;
NETDATA_DOUBLE new_value;
switch(rd->algorithm) {
case RRD_ALGORITHM_INCREMENTAL:
new_value = (calculated_number)
new_value = (NETDATA_DOUBLE)
( rd->calculated_value
* (calculated_number)(next_store_ut - last_collect_ut)
/ (calculated_number)(now_collect_ut - last_collect_ut)
* (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
/ (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
);
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC2 INC "
CALCULATED_NUMBER_FORMAT " = "
CALCULATED_NUMBER_FORMAT
rrdset_debug(st, "%s: CALC2 INC " NETDATA_DOUBLE_FORMAT " = "
NETDATA_DOUBLE_FORMAT
" * (%llu - %llu)"
" / (%llu - %llu)"
, rd->name
@ -1146,18 +1031,18 @@ static inline size_t rrdset_done_interpolate(
rd->calculated_value -= new_value;
new_value += rd->last_calculated_value;
rd->last_calculated_value = 0;
new_value /= (calculated_number)st->update_every;
new_value /= (NETDATA_DOUBLE)st->update_every;
if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " NETDATA_DOUBLE_FORMAT " - EXTRAPOLATING",
rd->name
, (calculated_number)(next_store_ut - last_stored_ut)
, (NETDATA_DOUBLE)(next_store_ut - last_stored_ut)
);
#endif
new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut);
new_value = new_value * (NETDATA_DOUBLE)(st->update_every * USEC_PER_SEC) / (NETDATA_DOUBLE)(next_store_ut - last_stored_ut);
}
break;
@ -1176,21 +1061,19 @@ static inline size_t rrdset_done_interpolate(
// we have missed an update
// interpolate in the middle values
new_value = (calculated_number)
new_value = (NETDATA_DOUBLE)
( ( (rd->calculated_value - rd->last_calculated_value)
* (calculated_number)(next_store_ut - last_collect_ut)
/ (calculated_number)(now_collect_ut - last_collect_ut)
* (NETDATA_DOUBLE)(next_store_ut - last_collect_ut)
/ (NETDATA_DOUBLE)(now_collect_ut - last_collect_ut)
)
+ rd->last_calculated_value
);
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC2 DEF "
CALCULATED_NUMBER_FORMAT " = ((("
"(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
"(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
" * %llu"
" / %llu) + " CALCULATED_NUMBER_FORMAT
, rd->name
" / %llu) + " NETDATA_DOUBLE_FORMAT, rd->name
, new_value
, rd->calculated_value, rd->last_calculated_value
, (next_store_ut - first_ut)
@ -1271,7 +1154,7 @@ static inline void rrdset_done_fill_the_gap(RRDSET *st) {
long current_entry = st->current_entry;
for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) {
rd->values[current_entry] = SN_EMPTY_SLOT;
rd->db[current_entry] = SN_EMPTY_SLOT;
current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1;
#ifdef NETDATA_INTERNAL_CHECKS
@ -1332,7 +1215,8 @@ void rrdset_done(RRDSET *st) {
// check if the chart has a long time to be updated
if(unlikely(st->usec_since_last_update > st->entries * update_every_ut &&
st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && st->rrd_memory_mode != RRD_MEMORY_MODE_NONE)) {
info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" LONG_DOUBLE_MODIFIER " secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (LONG_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
info("host '%s', chart %s: took too long to be updated (counter #%zu, update #%zu, %0.3" NETDATA_DOUBLE_MODIFIER
" secs). Resetting it.", st->rrdhost->hostname, st->name, st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);
rrdset_reset(st);
st->usec_since_last_update = update_every_ut;
store_this_entry = 0;
@ -1459,6 +1343,7 @@ void rrdset_done(RRDSET *st) {
#endif
}
}
after_first_database_work:
st->counter_done++;
@ -1469,10 +1354,10 @@ after_first_database_work:
}
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "last_collect_ut = %0.3" LONG_DOUBLE_MODIFIER " (last collection time)", (LONG_DOUBLE)last_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "now_collect_ut = %0.3" LONG_DOUBLE_MODIFIER " (current collection time)", (LONG_DOUBLE)now_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "last_stored_ut = %0.3" LONG_DOUBLE_MODIFIER " (last updated time)", (LONG_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" LONG_DOUBLE_MODIFIER " (next interpolation point)", (LONG_DOUBLE)next_store_ut/USEC_PER_SEC);
rrdset_debug(st, "last_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last collection time)", (NETDATA_DOUBLE)last_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "now_collect_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (current collection time)", (NETDATA_DOUBLE)now_collect_ut/USEC_PER_SEC);
rrdset_debug(st, "last_stored_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (last updated time)", (NETDATA_DOUBLE)last_stored_ut/USEC_PER_SEC);
rrdset_debug(st, "next_store_ut = %0.3" NETDATA_DOUBLE_MODIFIER " (next interpolation point)", (NETDATA_DOUBLE)next_store_ut/USEC_PER_SEC);
#endif
// calculate totals and count the dimensions
@ -1481,7 +1366,9 @@ after_first_database_work:
rrddim_foreach_read(rd, st) {
if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
continue;
dimensions++;
if(likely(rd->updated))
st->collected_total += rd->collected_value;
}
@ -1509,9 +1396,8 @@ after_first_database_work:
rrdset_debug(st, "%s: START "
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT
, rd->name
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
@ -1521,21 +1407,19 @@ after_first_database_work:
switch(rd->algorithm) {
case RRD_ALGORITHM_ABSOLUTE:
rd->calculated_value = (calculated_number)rd->collected_value
* (calculated_number)rd->multiplier
/ (calculated_number)rd->divisor;
rd->calculated_value = (NETDATA_DOUBLE)rd->collected_value
* (NETDATA_DOUBLE)rd->multiplier
/ (NETDATA_DOUBLE)rd->divisor;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN "
CALCULATED_NUMBER_FORMAT " = "
rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " NETDATA_DOUBLE_FORMAT " = "
COLLECTED_NUMBER_FORMAT
" * " CALCULATED_NUMBER_FORMAT
" / " CALCULATED_NUMBER_FORMAT
, rd->name
" * " NETDATA_DOUBLE_FORMAT
" / " NETDATA_DOUBLE_FORMAT, rd->name
, rd->calculated_value
, rd->collected_value
, (calculated_number)rd->multiplier
, (calculated_number)rd->divisor
, (NETDATA_DOUBLE)rd->multiplier
, (NETDATA_DOUBLE)rd->divisor
);
#endif
@ -1548,13 +1432,12 @@ after_first_database_work:
// the percentage of the current value
// over the total of all dimensions
rd->calculated_value =
(calculated_number)100
* (calculated_number)rd->collected_value
/ (calculated_number)st->collected_total;
(NETDATA_DOUBLE)100
* (NETDATA_DOUBLE)rd->collected_value
/ (NETDATA_DOUBLE)st->collected_total;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC PCENT-ROW "
CALCULATED_NUMBER_FORMAT " = 100"
rrdset_debug(st, "%s: CALC PCENT-ROW " NETDATA_DOUBLE_FORMAT " = 100"
" * " COLLECTED_NUMBER_FORMAT
" / " COLLECTED_NUMBER_FORMAT
, rd->name
@ -1605,34 +1488,32 @@ after_first_database_work:
// TODO: remember recent history of rates and compare with current rate to reduce this chance.
if (delta < max_acceptable_rate) {
rd->calculated_value +=
(calculated_number) delta
* (calculated_number) rd->multiplier
/ (calculated_number) rd->divisor;
(NETDATA_DOUBLE) delta
* (NETDATA_DOUBLE) rd->multiplier
/ (NETDATA_DOUBLE) rd->divisor;
} else {
// This is a reset. Any overflow with a rate greater than MAX_INCREMENTAL_PERCENT_RATE will also
// be detected as a reset instead.
rd->calculated_value += (calculated_number)0;
rd->calculated_value += (NETDATA_DOUBLE)0;
}
}
else {
rd->calculated_value +=
(calculated_number) (rd->collected_value - rd->last_collected_value)
* (calculated_number) rd->multiplier
/ (calculated_number) rd->divisor;
(NETDATA_DOUBLE) (rd->collected_value - rd->last_collected_value)
* (NETDATA_DOUBLE) rd->multiplier
/ (NETDATA_DOUBLE) rd->divisor;
}
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC INC PRE "
CALCULATED_NUMBER_FORMAT " = ("
rrdset_debug(st, "%s: CALC INC PRE " NETDATA_DOUBLE_FORMAT " = ("
COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
")"
" * " CALCULATED_NUMBER_FORMAT
" / " CALCULATED_NUMBER_FORMAT
, rd->name
" * " NETDATA_DOUBLE_FORMAT
" / " NETDATA_DOUBLE_FORMAT, rd->name
, rd->calculated_value
, rd->collected_value, rd->last_collected_value
, (calculated_number)rd->multiplier
, (calculated_number)rd->divisor
, (NETDATA_DOUBLE)rd->multiplier
, (NETDATA_DOUBLE)rd->divisor
);
#endif
@ -1665,13 +1546,12 @@ after_first_database_work:
rd->calculated_value = 0;
else
rd->calculated_value =
(calculated_number)100
* (calculated_number)(rd->collected_value - rd->last_collected_value)
/ (calculated_number)(st->collected_total - st->last_collected_total);
(NETDATA_DOUBLE)100
* (NETDATA_DOUBLE)(rd->collected_value - rd->last_collected_value)
/ (NETDATA_DOUBLE)(st->collected_total - st->last_collected_total);
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC PCENT-DIFF "
CALCULATED_NUMBER_FORMAT " = 100"
rrdset_debug(st, "%s: CALC PCENT-DIFF " NETDATA_DOUBLE_FORMAT " = 100"
" * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
" / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
, rd->name
@ -1689,8 +1569,7 @@ after_first_database_work:
rd->calculated_value = 0;
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: CALC "
CALCULATED_NUMBER_FORMAT " = 0"
rrdset_debug(st, "%s: CALC " NETDATA_DOUBLE_FORMAT " = 0"
, rd->name
, rd->calculated_value
);
@ -1703,9 +1582,8 @@ after_first_database_work:
rrdset_debug(st, "%s: PHASE2 "
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT
, rd->name
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
@ -1765,7 +1643,8 @@ after_second_database_work:
case RRD_ALGORITHM_INCREMENTAL:
if(unlikely(!first_entry)) {
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
#endif
rd->last_calculated_value += rd->calculated_value;
@ -1781,7 +1660,8 @@ after_second_database_work:
case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
#ifdef NETDATA_INTERNAL_CHECKS
rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
rrdset_debug(st, "%s: setting last_calculated_value (old: " NETDATA_DOUBLE_FORMAT
") to last_calculated_value (new: " NETDATA_DOUBLE_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
#endif
rd->last_calculated_value = rd->calculated_value;
@ -1796,9 +1676,8 @@ after_second_database_work:
rrdset_debug(st, "%s: END "
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT
, rd->name
" last_calculated_value = " NETDATA_DOUBLE_FORMAT
" calculated_value = " NETDATA_DOUBLE_FORMAT, rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
@ -1811,15 +1690,24 @@ after_second_database_work:
// ALL DONE ABOUT THE DATA UPDATE
// --------------------------------------------------------------------
// find if there are any obsolete dimensions
time_t now = now_realtime_sec();
if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
// update the memory mapped files with the latest values
rrdset_memory_file_update(st);
rrddim_foreach_read(rd, st) {
rrddim_memory_file_update(rd);
}
}
// find if there are any obsolete dimensions
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
rrddim_foreach_read(rd, st)
if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)))
break;
if(unlikely(rd)) {
time_t now = now_realtime_sec();
RRDDIM *last;
// there is a dimension to free
// upgrade our read lock to a write lock
@ -1831,10 +1719,11 @@ after_second_database_work:
&& (rd->last_collected_time.tv_sec + rrdset_free_obsolete_time < now))) {
info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
info("Deleting dimension file '%s'.", rd->cache_filename);
if(unlikely(unlink(rd->cache_filename) == -1))
error("Cannot delete dimension file '%s'", rd->cache_filename);
const char *cache_filename = rrddim_cache_filename(rd);
if(cache_filename) {
info("Deleting dimension file '%s'.", cache_filename);
if (unlikely(unlink(cache_filename) == -1))
error("Cannot delete dimension file '%s'", cache_filename);
}
#ifdef ENABLE_DBENGINE
@ -1885,3 +1774,208 @@ after_second_database_work:
netdata_thread_enable_cancelability();
}
// ----------------------------------------------------------------------------
// compatibility layer for RRDSET files v019
#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
#define RRD_ID_LENGTH_MAX_V019 200
struct avl_element_v019 {
void *avl_link[2];
signed char avl_balance;
};
struct avl_tree_type_v019 {
void *root;
int (*compar)(void *a, void *b);
};
struct avl_tree_lock_v019 {
struct avl_tree_type_v019 avl_tree;
pthread_rwlock_t rwlock;
};
struct rrdset_map_save_v019 {
struct avl_element_v019 avl; // ignored
struct avl_element_v019 avlname; // ignored
char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
void *name; // ignored
void *unused_ptr; // ignored
void *type; // ignored
void *family; // ignored
void *title; // ignored
void *units; // ignored
void *context; // ignored
uint32_t hash_context; // ignored
uint32_t chart_type; // ignored
int update_every; // check to reset all - update on load
long entries; // check to reset all - update on load
long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
uint32_t flags; // ignored
void *exporting_flags; // ignored
int gap_when_lost_iterations_above; // ignored
long priority; // ignored
uint32_t rrd_memory_mode; // ignored
void *cache_dir; // ignored
char cache_filename[FILENAME_MAX+1]; // ignored - update on load
pthread_rwlock_t rrdset_rwlock; // ignored
size_t counter; // NEEDS TO BE UPDATED - maintained on load
size_t counter_done; // ignored
union { //
time_t last_accessed_time; // ignored
time_t last_entry_t; // ignored
}; //
time_t upstream_resync_time; // ignored
void *plugin_name; // ignored
void *module_name; // ignored
void *chart_uuid; // ignored
void *state; // ignored
size_t unused[3]; // ignored
size_t rrddim_page_alignment; // ignored
uint32_t hash; // ignored
uint32_t hash_name; // ignored
usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
struct timeval last_collected_time; // ignored
long long collected_total; // NEEDS TO BE UPDATED - maintained on load
long long last_collected_total; // NEEDS TO BE UPDATED - maintained on load
void *rrdfamily; // ignored
void *rrdhost; // ignored
void *next; // ignored
long double green; // ignored
long double red; // ignored
struct avl_tree_lock_v019 rrdvar_root_index; // ignored
void *variables; // ignored
void *alarms; // ignored
unsigned long memsize; // check to reset all - update on load
char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
struct avl_tree_lock_v019 dimensions_index; // ignored
void *dimensions; // ignored
};
void rrdset_memory_file_update(RRDSET *st) {
if(!st->st_on_file) return;
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
st_on_file->current_entry = st->current_entry;
st_on_file->counter = st->counter;
st_on_file->usec_since_last_update = st->usec_since_last_update;
st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
st_on_file->collected_total = st->collected_total;
st_on_file->last_collected_total = st->last_collected_total;
}
const char *rrdset_cache_filename(RRDSET *st) {
if(!st->st_on_file) return NULL;
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
return st_on_file->cache_filename;
}
void rrdset_memory_file_free(RRDSET *st) {
if(!st->st_on_file) return;
// needed for memory mode map, to save the latest state
rrdset_memory_file_update(st);
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
munmap(st_on_file, st_on_file->memsize);
// remove the pointers from the RRDDIM
st->st_on_file = NULL;
}
void rrdset_memory_file_save(RRDSET *st) {
if(!st->st_on_file) return;
rrdset_memory_file_update(st);
struct rrdset_map_save_v019 *st_on_file = st->st_on_file;
if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
memory_file_save(st_on_file->cache_filename, st->st_on_file, st_on_file->memsize);
}
bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {
if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
return false;
char fullfilename[FILENAME_MAX + 1];
snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", st->cache_dir);
unsigned long size = sizeof(struct rrdset_map_save_v019);
struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
fullfilename, size,
((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
0);
if(!st_on_file) return false;
time_t now = now_realtime_sec();
if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
info("Initializing file '%s'.", fullfilename);
memset(st_on_file, 0, size);
}
else if(strncmp(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019) != 0) {
error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, st->id);
memset(st_on_file, 0, size);
}
else if(st_on_file->memsize != size || st_on_file->entries != st->entries) {
error("File '%s' does not have the desired size. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if(st_on_file->update_every != st->update_every) {
error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if((now - st_on_file->last_updated.tv_sec) > st->update_every * st->entries) {
info("File '%s' is too old. Clearing it.", fullfilename);
memset(st_on_file, 0, size);
}
else if(st_on_file->last_updated.tv_sec > now + st->update_every) {
error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now));
st_on_file->last_updated.tv_sec = now;
}
if(st_on_file->current_entry >= st_on_file->entries)
st_on_file->current_entry = 0;
// make sure the database is aligned
bool align_last_updated = false;
if(st_on_file->last_updated.tv_sec) {
st_on_file->update_every = st->update_every;
align_last_updated = true;
}
// copy the useful values to st
st->current_entry = st_on_file->current_entry;
st->counter = st_on_file->counter;
st->usec_since_last_update = st_on_file->usec_since_last_update;
st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
st->collected_total = st_on_file->collected_total;
st->last_collected_total = st_on_file->last_collected_total;
// link it to st
st->st_on_file = st_on_file;
// clear everything
memset(st_on_file, 0, size);
// set the values we need
strncpyz(st_on_file->id, st->id, RRD_ID_LENGTH_MAX_V019 + 1);
strcpy(st_on_file->cache_filename, fullfilename);
strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
st_on_file->memsize = size;
st_on_file->entries = st->entries;
st_on_file->update_every = st->update_every;
st_on_file->rrd_memory_mode = memory_mode;
if(align_last_updated)
last_updated_time_align(st);
// copy the useful values back to st_on_file
rrdset_memory_file_update(st);
return true;
}

9
database/rrdsetvar.c
View file

@ -163,7 +163,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
// not found, allocate one
calculated_number *v = mallocz(sizeof(calculated_number));
NETDATA_DOUBLE *v = mallocz(sizeof(NETDATA_DOUBLE));
*v = NAN;
rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);
@ -173,12 +173,13 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
return rs;
}
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) {
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, NETDATA_DOUBLE value) {
if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) {
error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " NETDATA_DOUBLE_FORMAT
" but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
}
else {
calculated_number *v = rs->value;
NETDATA_DOUBLE *v = rs->value;
if(*v != value) {
*v = value;

2
database/rrdsetvar.h
View file

@ -35,7 +35,7 @@ struct rrdsetvar {
};
extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name);
extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, calculated_number value);
extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, NETDATA_DOUBLE value);
extern void rrdsetvar_rename_all(RRDSET *st);
extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options);

20
database/rrdvar.c
View file

@ -133,7 +133,7 @@ inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)
}
static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) {
calculated_number *v = callocz(1, sizeof(calculated_number));
NETDATA_DOUBLE *v = callocz(1, sizeof(NETDATA_DOUBLE));
*v = NAN;
RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);
@ -158,11 +158,11 @@ RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) {
return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name);
}
void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value) {
void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value) {
if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED))
error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value);
error("requested to set variable '%s' to value " NETDATA_DOUBLE_FORMAT " but the variable is not a custom one.", rv->name, value);
else {
calculated_number *v = rv->value;
NETDATA_DOUBLE *v = rv->value;
if(*v != value) {
*v = value;
@ -181,10 +181,10 @@ int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR * /*rv*
// ----------------------------------------------------------------------------
// RRDVAR lookup
calculated_number rrdvar2number(RRDVAR *rv) {
NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) {
switch(rv->type) {
case RRDVAR_TYPE_CALCULATED: {
calculated_number *n = (calculated_number *)rv->value;
NETDATA_DOUBLE *n = (NETDATA_DOUBLE *)rv->value;
return *n;
}
@ -209,12 +209,12 @@ calculated_number rrdvar2number(RRDVAR *rv) {
}
default:
error("I don't know how to convert RRDVAR type %u to calculated_number", rv->type);
error("I don't know how to convert RRDVAR type %u to NETDATA_DOUBLE", rv->type);
return NAN;
}
}
int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {
int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result) {
RRDSET *st = rc->rrdset;
if(!st) return 0;
@ -254,13 +254,13 @@ struct variable2json_helper {
static int single_variable2json(void *entry, void *data) {
struct variable2json_helper *helper = (struct variable2json_helper *)data;
RRDVAR *rv = (RRDVAR *)entry;
calculated_number value = rrdvar2number(rv);
NETDATA_DOUBLE value = rrdvar2number(rv);
if (helper->options == RRDVAR_OPTION_DEFAULT || rv->options & helper->options) {
if(unlikely(isnan(value) || isinf(value)))
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name);
else
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" LONG_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (LONG_DOUBLE)value);
buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5" NETDATA_DOUBLE_MODIFIER, helper->counter?",":"", rv->name, (NETDATA_DOUBLE)value);
helper->counter++;
}

4
database/rrdvar.h
View file

@ -52,13 +52,13 @@ extern int rrdvar_fix_name(char *variable);
#include "rrd.h"
extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);
extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value);
extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, NETDATA_DOUBLE value);
extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);
extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock);
extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data);
extern calculated_number rrdvar2number(RRDVAR *rv);
extern NETDATA_DOUBLE rrdvar2number(RRDVAR *rv);
extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);

8
database/sqlite/sqlite_aclk_alert.c
View file

@ -336,8 +336,8 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
strdupz((char *)format_value_and_unit(
old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
alarm_log.value = (calculated_number) sqlite3_column_double(res, 23);
alarm_log.old_value = (calculated_number) sqlite3_column_double(res, 24);
alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log.rendered_info = sqlite3_column_type(res, 18) == SQLITE_NULL ?
@ -868,8 +868,8 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
alarm_log->value_string = strdupz(ae->new_value_string);
alarm_log->old_value_string = strdupz(ae->old_value_string);
alarm_log->value = (!isnan(ae->new_value)) ? (calculated_number)ae->new_value : 0;
alarm_log->old_value = (!isnan(ae->old_value)) ? (calculated_number)ae->old_value : 0;
alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0;
alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0;
alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
alarm_log->rendered_info = ae->info ? strdupz(ae->info) : strdupz((char *)"");

4
database/sqlite/sqlite_health.c
View file

@ -751,8 +751,8 @@ void sql_health_alarm_log_load(RRDHOST *host) {
ae->old_status = (RRDCALC_STATUS)sqlite3_column_int(res, 23);
ae->delay = (int) sqlite3_column_int(res, 24);
ae->new_value = (calculated_number) sqlite3_column_double(res, 25);
ae->old_value = (calculated_number) sqlite3_column_double(res, 26);
ae->new_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 25);
ae->old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 26);
ae->last_repeat = last_repeat;

54
docs/guides/longer-metrics-storage.md
View file

@ -25,14 +25,14 @@ available disk space for long-term metrics storage. This feature of the database
larger dataset than your system's available RAM.
The database engine is currently the default method of storing metrics, but if you're not sure which database you're
using, check out your `netdata.conf` file and look for the `memory mode` setting:
using, check out your `netdata.conf` file and look for the `[db].mode` setting:
```conf
[global]
memory mode = dbengine
[db]
mode = dbengine
```
If `memory mode` is set to anything but `dbengine`, change it and restart Netdata using the standard command for
If `[db].mode` is set to anything but `dbengine`, change it and restart Netdata using the standard command for
restarting services on your system. You're now using the database engine!
What makes the database engine efficient? While it's structured like a traditional database, the database engine splits
@ -43,22 +43,22 @@ When the Netdata dashboard queries for historical metrics, the database engine w
return relevant metrics for visualization in charts.
Now, given that the database engine uses _both_ RAM and disk, there are two other settings to consider: `page cache
size` and `dbengine multihost disk space`.
size MB` and `dbengine multihost disk space MB`.
```conf
[global]
page cache size = 32
dbengine multihost disk space = 256
[db]
page cache size MB = 32
dbengine multihost disk space MB = 256
```
`page cache size` sets the maximum amount of RAM (in MiB) the database engine will use for caching and indexing.
`dbengine multihost disk space` sets the maximum disk space (again, in MiB) the database engine will use for storing
compressed metrics. The default settings retain about two day's worth of metrics on a system collecting 2,000 metrics
`[db].page cache size MB` sets the maximum amount of RAM the database engine will use for caching and indexing.
`[db].dbengine multihost disk space MB` sets the maximum disk space the database engine will use for storing
compressed metrics. The default settings retain about four day's worth of metrics on a system collecting 2,000 metrics
every second.
[**See our database engine
calculator**](/docs/store/change-metrics-storage.md#calculate-the-system-resources-ram-disk-space-needed-to-store-metrics)
to help you correctly set `dbengine multihost disk space` based on your needs. The calculator gives an accurate estimate
to help you correctly set `[db].dbengine multihost disk space MB` based on your needs. The calculator gives an accurate estimate
based on how many child nodes you have, how many metrics your Agent collects, and more.
With the database engine active, you can back up your `/var/cache/netdata/dbengine/` folder to another location for
@ -72,26 +72,26 @@ aren't ready to make the move.
In previous versions, Netdata used a round-robin database to store 1 hour of per-second metrics.
To see if you're still using this database, or if you would like to switch to it, open your `netdata.conf` file and see
if `memory mode` option is set to `save`.
if `[db].mode` option is set to `save`.
```conf
[global]
memory mode = save
[db]
mode = save
```
If `memory mode` is set to `save`, then you're using the round-robin database. If so, the `history` option is set to
If `[db].mode` is set to `save`, then you're using the round-robin database. If so, the `[db].retention` option is set to
`3600`, which is the equivalent to 3,600 seconds, or one hour.
To increase your historical metrics, you can increase `history` to the number of seconds you'd like to store:
To increase your historical metrics, you can increase `[db].retention` to the number of seconds you'd like to store:
```conf
[global]
[db]
# 2 hours = 2 * 60 * 60 = 7200 seconds
history = 7200
retention = 7200
# 4 hours = 4 * 60 * 60 = 14440 seconds
history = 14440
retention = 14440
# 24 hours = 24 * 60 * 60 = 86400 seconds
history = 86400
retention = 86400
```
And so on.
@ -105,20 +105,10 @@ dashboard and look at the bottom-right corner of the interface. You'll find a se
On this desktop system, using a Ryzen 5 1600 and 16GB of RAM, the round-robin databases uses 25 MB of RAM to store just
over an hour's worth of data for nearly 2,000 metrics.
To increase the `history` option, you need to edit your `netdata.conf` file and increase the `history` setting. In most
installations, you'll find it at `/etc/netdata/netdata.conf`, but some operating systems place it at
`/opt/netdata/etc/netdata/netdata.conf`.
Use `/etc/netdata/edit-config netdata.conf`, or your favorite text editor, to replace `3600` with the number of seconds
you'd like to store.
You should base this number on two things: How much history you need for your use case, and how much RAM you're willing
to dedicate to Netdata.
> Take care when you change the `history` option on production systems. Netdata is configured to stop its process if
> your system starts running out of RAM, but you can never be too careful. Out of memory situations are very bad.
How much RAM will a longer history use? Let's use a little math.
How much RAM will a longer retention use? Let's use a little math.
The round-robin database needs 4 bytes for every value Netdata collects. If Netdata collects metrics every second,
that's 4 bytes, per second, per metric.

3
exporting/exporting_engine.h
View file

@ -269,7 +269,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st);
extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options);
calculated_number exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE
exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);

4
exporting/graphite/graphite.c
View file

@ -173,14 +173,14 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
RRD_ID_LENGTH_MAX);
time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
buffer_sprintf(
instance->buffer,
"%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
"%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
chart_name,

4
exporting/json/json.c
View file

@ -224,7 +224,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
RRDHOST *host = st->rrdhost;
time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
@ -265,7 +265,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
"\"id\":\"%s\","
"\"name\":\"%s\","
"\"value\":" CALCULATED_NUMBER_FORMAT ","
"\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"timestamp\": %llu}",

8
exporting/opentsdb/opentsdb.c
View file

@ -225,14 +225,14 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
RRD_ID_LENGTH_MAX);
time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
buffer_sprintf(
instance->buffer,
"put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n",
"put %s.%s.%s %llu " NETDATA_DOUBLE_FORMAT " host=%s%s%s%s\n",
instance->config.prefix,
chart_name,
dimension_name,
@ -368,7 +368,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
RRD_ID_LENGTH_MAX);
time_t last_t;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
@ -381,7 +381,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
"{"
"\"metric\":\"%s.%s.%s\","
"\"timestamp\":%llu,"
"\"value\":"CALCULATED_NUMBER_FORMAT","
"\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"tags\":{"
"\"host\":\"%s%s%s\"%s"
"}"

10
exporting/process_data.c
View file

@ -64,7 +64,7 @@ int mark_scheduled_instances(struct engine *engine)
* @param last_timestamp the timestamp that should be reported to the exporting connector instance.
* @return Returns the value, calculated over the given period.
*/
calculated_number exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp)
@ -123,15 +123,15 @@ calculated_number exporting_calculate_value_from_stored_data(
*last_timestamp = before;
size_t counter = 0;
calculated_number sum = 0;
calculated_number value;
NETDATA_DOUBLE sum = 0;
NETDATA_DOUBLE value;
for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
time_t curr_t, end_t;
SN_FLAGS flags;
value = rd->state->query_ops.next_metric(&handle, &curr_t, &end_t, &flags);
if (unlikely(!calculated_number_isnumber(value))) {
if (unlikely(!netdata_double_isnumber(value))) {
// not collected
continue;
}
@ -156,7 +156,7 @@ calculated_number exporting_calculate_value_from_stored_data(
if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM))
return sum;
return sum / (calculated_number)counter;
return sum / (NETDATA_DOUBLE)counter;
}
/**

18
exporting/prometheus/prometheus.c
View file

@ -362,7 +362,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
}
}
calculated_number value = rrdvar2number(rv);
NETDATA_DOUBLE value = rrdvar2number(rv);
if (isnan(value) || isinf(value)) {
if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
buffer_sprintf(
@ -383,7 +383,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf(
opts->wb,
"%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
"%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
opts->prefix,
opts->name,
label_pre,
@ -394,7 +394,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
else
buffer_sprintf(
opts->wb,
"%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n",
"%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT "\n",
opts->prefix,
opts->name,
label_pre,
@ -483,9 +483,9 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
if (prometheus_collector)
buffer_sprintf(
wb,
CALCULATED_NUMBER_FORMAT,
(calculated_number)p->rd->last_collected_value * (calculated_number)p->rd->multiplier /
(calculated_number)p->rd->divisor);
NETDATA_DOUBLE_FORMAT,
(NETDATA_DOUBLE)p->rd->last_collected_value * (NETDATA_DOUBLE)p->rd->multiplier /
(NETDATA_DOUBLE)p->rd->divisor);
else
buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value);
@ -732,7 +732,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
time_t first_time = instance->after;
time_t last_time = instance->before;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@ -764,7 +764,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf(
wb,
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
" %llu\n",
prefix,
context,
@ -779,7 +779,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
else
buffer_sprintf(
wb,
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
"%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
"\n",
prefix,
context,

2
exporting/prometheus/remote_write/remote_write.c
View file

@ -294,7 +294,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
// we need average or sum of the data
time_t last_t = instance->before;
calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)

6
exporting/tests/exporting_doubles.c
View file

@ -52,11 +52,11 @@ int __wrap_mark_scheduled_instances(struct engine *engine)
return mock_type(int);
}
calculated_number __real_exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
calculated_number __wrap_exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp)
@ -67,7 +67,7 @@ calculated_number __wrap_exporting_calculate_value_from_stored_data(
*last_timestamp = 15052;
function_called();
return mock_type(calculated_number);
return mock_type(NETDATA_DOUBLE);
}
int __real_prepare_buffers(struct engine *engine);

6
exporting/tests/netdata_doubles.c
View file

@ -177,7 +177,7 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
return RRD_MEMORY_MODE_NONE_NAME;
}
calculated_number rrdvar2number(RRDVAR *rv)
NETDATA_DOUBLE rrdvar2number(RRDVAR *rv)
{
(void)rv;
return 0;
@ -230,7 +230,7 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
return mock_type(int);
}
calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags)
{
(void)handle;
(void)start_time;
@ -239,7 +239,7 @@ calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *ha
function_called();
return mock_type(calculated_number);
return mock_type(NETDATA_DOUBLE);
}
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)

6
exporting/tests/test_exporting_engine.h
View file

@ -61,7 +61,7 @@ time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
calculated_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
NETDATA_DOUBLE __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *start_time, time_t *end_time, SN_FLAGS *flags);
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
// -----------------------------------------------------------------------
@ -88,11 +88,11 @@ int __wrap_init_connectors(struct engine *engine);
int __real_mark_scheduled_instances(struct engine *engine);
int __wrap_mark_scheduled_instances(struct engine *engine);
calculated_number __real_exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
calculated_number __wrap_exporting_calculate_value_from_stored_data(
NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);

17
health/health.c
View file

@ -238,7 +238,7 @@ void health_reload(void) {
// ----------------------------------------------------------------------------
// health main thread and friends
static inline RRDCALC_STATUS rrdcalc_value2status(calculated_number n) {
static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) {
if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
if(n) return RRDCALC_STATUS_RAISED;
return RRDCALC_STATUS_CLEAR;
@ -346,7 +346,9 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'",
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '" NETDATA_DOUBLE_FORMAT_ZERO
"' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d' '%s' '%s' '%s' '%s' '%s'",
exec,
recipient,
host->registry_hostname,
@ -411,7 +413,7 @@ static inline void health_alarm_wait_for_execution(ALARM_ENTRY *ae) {
}
static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
debug(D_HEALTH, "Health alarm '%s.%s' = " CALCULATED_NUMBER_FORMAT_AUTO " - changed status from %s to %s",
debug(D_HEALTH, "Health alarm '%s.%s' = " NETDATA_DOUBLE_FORMAT_AUTO " - changed status from %s to %s",
ae->chart?ae->chart:"NOCHART", ae->name,
ae->new_value,
rrdcalc_status2string(ae->old_status),
@ -892,8 +894,7 @@ void *health_main(void *ptr) {
} else
rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value "
CALCULATED_NUMBER_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rc->value
);
}
@ -917,7 +918,7 @@ void *health_main(void *ptr) {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
CALCULATED_NUMBER_FORMAT
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name,
rc->calculation->parsed_as, rc->calculation->result,
buffer_tostring(rc->calculation->error_msg), rc->source
@ -966,7 +967,7 @@ void *health_main(void *ptr) {
} else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
CALCULATED_NUMBER_FORMAT
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source
);
@ -992,7 +993,7 @@ void *health_main(void *ptr) {
} else {
rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
CALCULATED_NUMBER_FORMAT
NETDATA_DOUBLE_FORMAT
": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART",
rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg),
rc->source

6
health/health.h
View file

@ -35,7 +35,7 @@ extern void health_init(void);
extern void health_reload(void);
extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result);
extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, NETDATA_DOUBLE *result);
extern void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);
extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
extern void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);
@ -63,8 +63,8 @@ extern ALARM_ENTRY* health_create_alarm_entry(
const char *exec,
const char *recipient,
time_t duration,
calculated_number old_value,
calculated_number new_value,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
const char *source,

16
health/health_config.c
View file

@ -54,7 +54,9 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
rc->id = rrdcalc_get_unique_id(host, rc->chart, rc->name, &rc->next_event_id);
debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
rc->chart?rc->chart:"NOCHART",
rc->name,
rc->id,
@ -141,7 +143,9 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
}
}
debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " CALCULATED_NUMBER_FORMAT_AUTO ", red " CALCULATED_NUMBER_FORMAT_AUTO ", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green " NETDATA_DOUBLE_FORMAT_AUTO
", red " NETDATA_DOUBLE_FORMAT_AUTO
", lookup: group %d, after %d, before %d, options %u, dimensions '%s', for each dimension '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f, warn_repeat_every %u, crit_repeat_every %u",
rt->name,
(rt->context)?rt->context:"NONE",
(rt->exec)?rt->exec:"DEFAULT",
@ -848,7 +852,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
alert_cfg->green = strdupz(value);
char *e;
rc->green = str2ld(value, &e);
rc->green = str2ndd(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rc->name, key, e);
@ -857,7 +861,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
alert_cfg->red = strdupz(value);
char *e;
rc->red = str2ld(value, &e);
rc->red = str2ndd(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rc->name, key, e);
@ -1097,7 +1101,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
alert_cfg->green = strdupz(value);
char *e;
rt->green = str2ld(value, &e);
rt->green = str2ndd(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rt->name, key, e);
@ -1106,7 +1110,7 @@ static int health_readfile(const char *filename, void *data) {
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
alert_cfg->red = strdupz(value);
char *e;
rt->red = str2ld(value, &e);
rt->red = str2ndd(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, filename, rt->name, key, e);

6
health/health_log.c
View file

@ -98,7 +98,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
"\t%08x\t%08x\t%08x"
"\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
"\t%d\t%d\t%d\t%d"
"\t" CALCULATED_NUMBER_FORMAT_AUTO "\t" CALCULATED_NUMBER_FORMAT_AUTO
"\t" NETDATA_DOUBLE_FORMAT_AUTO "\t" NETDATA_DOUBLE_FORMAT_AUTO
"\t%016"PRIx64""
"\t%s\t%s\t%s"
"\n"
@ -457,8 +457,8 @@ inline ALARM_ENTRY* health_create_alarm_entry(
const char *exec,
const char *recipient,
time_t duration,
calculated_number old_value,
calculated_number new_value,
NETDATA_DOUBLE old_value,
NETDATA_DOUBLE new_value,
RRDCALC_STATUS old_status,
RRDCALC_STATUS new_status,
const char *source,

8
libnetdata/buffer/buffer.c
View file

@ -65,9 +65,9 @@ void buffer_char_replace(BUFFER *wb, char from, char to)
}
// This trick seems to give an 80% speed increase in 32bit systems
// print_calculated_number_llu_r() will just print the digits up to the
// print_number_llu_r() will just print the digits up to the
// point the remaining value fits in 32 bits, and then calls
// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic.
// print_number_lu_r() to print the rest with 32 bit arithmetic.
inline char *print_number_lu_r(char *str, unsigned long uvalue) {
char *wstr = str;
@ -299,7 +299,7 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
}
void buffer_rrd_value(BUFFER *wb, calculated_number value)
void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value)
{
buffer_need_bytes(wb, 50);
@ -308,7 +308,7 @@ void buffer_rrd_value(BUFFER *wb, calculated_number value)
return;
}
else
wb->len += print_calculated_number(&wb->buffer[wb->len], value);
wb->len += print_netdata_double(&wb->buffer[wb->len], value);
// terminate it
buffer_need_bytes(wb, 1);

2
libnetdata/buffer/buffer.h
View file

@ -56,7 +56,7 @@ extern void buffer_reset(BUFFER *wb);
extern void buffer_strcat(BUFFER *wb, const char *txt);
extern void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len);
extern void buffer_rrd_value(BUFFER *wb, calculated_number value);
extern void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value);
extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);

2
libnetdata/clocks/clocks.c
View file

@ -393,7 +393,7 @@ static inline collected_number read_proc_uptime(char *filename) {
return 0;
}
return (collected_number)(strtold(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0);
return (collected_number)(strtondd(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0);
}
inline collected_number uptime_msec(char *filename){

47
libnetdata/config/appconfig.c
View file

@ -462,15 +462,15 @@ long long appconfig_get_number(struct config *root, const char *section, const c
return strtoll(s, NULL, 0);
}
LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value)
NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value)
{
char buffer[100], *s;
sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value);
sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value);
s = appconfig_get(root, section, name, buffer);
if(!s) return value;
return str2ld(s, NULL);
return str2ndd(s, NULL);
}
static inline int appconfig_test_boolean_value(char *s) {
@ -588,10 +588,10 @@ long long appconfig_set_number(struct config *root, const char *section, const c
return value;
}
LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value)
NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value)
{
char buffer[100];
sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value);
sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value);
appconfig_set(root, section, name, buffer);
@ -831,26 +831,27 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
"#\n"
"\n# global netdata configuration\n");
for(i = 0; i <= 15 ;i++) {
for(i = 0; i <= 16 ;i++) {
appconfig_wrlock(root);
for(co = root->first_section; co ; co = co->next) {
if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)) pri = 0;
else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 1;
else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 2;
else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 3;
else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 4;
else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 5;
else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 6;
else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 7;
else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 8;
else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 9;
// by default, new sections will get pri = 10 (set at the end, below)
else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 11;
else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 12;
else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 13;
else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 14;
else if(!strncmp(co->name, "plugin:", 7)) pri = 15; // << change the loop too if you change this
else pri = 10; // this is used for any new (currently unknown) sections
else if(!strcmp(co->name, CONFIG_SECTION_DB)) pri = 1;
else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 2;
else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 3;
else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 4;
else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 5;
else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 6;
else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 7;
else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 8;
else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 9;
else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 10;
// by default, new sections will get pri = 11 (set at the end, below)
else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 12;
else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 13;
else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 14;
else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 15;
else if(!strncmp(co->name, "plugin:", 7)) pri = 16; // << change the loop too if you change this
else pri = 11; // this is used for any new (currently unknown) sections
if(i == pri) {
int loaded = 0;
@ -916,7 +917,7 @@ int config_parse_duration(const char* string, int* result) {
if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback;
char *e = NULL;
calculated_number n = str2ld(string, &e);
NETDATA_DOUBLE n = str2ndd(string, &e);
if(e && *e) {
switch (*e) {
case 'Y':

6
libnetdata/config/appconfig.h
View file

@ -100,6 +100,8 @@
#define CONFIG_SECTION_HOST_LABEL "host labels"
#define EXPORTING_CONF "exporting.conf"
#define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics"
#define CONFIG_SECTION_DB "db"
// these are used to limit the configuration names and values lengths
// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length)
@ -168,7 +170,7 @@ extern void config_section_unlock(struct section *co);
extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value);
extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value);
extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value);
extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value);
extern NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);
extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value);
extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value);
@ -177,7 +179,7 @@ extern int appconfig_get_duration(struct config *root, const char *section, cons
extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value);
extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value);
extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value);
extern LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value);
extern NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value);
extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_exists(struct config *root, const char *section, const char *name);

112
libnetdata/eval/eval.c
View file

@ -9,7 +9,7 @@ typedef struct eval_value {
int type;
union {
calculated_number number;
NETDATA_DOUBLE number;
EVAL_VARIABLE *variable;
struct eval_node *expression;
};
@ -54,16 +54,16 @@ typedef struct eval_node {
static inline void eval_node_free(EVAL_NODE *op);
static inline EVAL_NODE *parse_full_expression(const char **string, int *error);
static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error);
static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error);
static inline void print_parsed_as_constant(BUFFER *out, calculated_number n);
static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n);
// ----------------------------------------------------------------------------
// evaluation of expressions
static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0;
calculated_number n;
NETDATA_DOUBLE n;
if(unlikely(this_hash == 0)) {
this_hash = simple_hash("this");
@ -179,8 +179,8 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL
return NAN;
}
static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) {
calculated_number n;
static inline NETDATA_DOUBLE eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) {
NETDATA_DOUBLE n;
switch(v->type) {
case EVAL_VALUE_EXPRESSION:
@ -204,101 +204,101 @@ static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v,
return n;
}
static inline int is_true(calculated_number n) {
static inline int is_true(NETDATA_DOUBLE n) {
if(isnan(n)) return 0;
if(isinf(n)) return 1;
if(n == 0) return 0;
return 1;
}
calculated_number eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error));
}
calculated_number eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error));
}
calculated_number eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isgreaterequal(n1, n2);
}
calculated_number eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return islessequal(n1, n2);
}
calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) && isnan(n2)) return 1;
if(isinf(n1) && isinf(n2)) return 1;
if(isnan(n1) || isnan(n2)) return 0;
if(isinf(n1) || isinf(n2)) return 0;
return calculated_number_equal(n1, n2);
return considered_equal_ndd(n1, n2);
}
calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return !eval_equal(exp, op, error);
}
calculated_number eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isless(n1, n2);
}
calculated_number eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
return isgreater(n1, n2);
}
calculated_number eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 + n2;
}
calculated_number eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 - n2;
}
calculated_number eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 * n2;
}
calculated_number eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
calculated_number n2 = eval_value(exp, &op->ops[1], error);
NETDATA_DOUBLE eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
if(isnan(n1) || isnan(n2)) return NAN;
if(isinf(n1) || isinf(n2)) return INFINITY;
return n1 / n2;
}
calculated_number eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return eval_value(exp, &op->ops[0], error);
}
calculated_number eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return !is_true(eval_value(exp, &op->ops[0], error));
}
calculated_number eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
return eval_value(exp, &op->ops[0], error);
}
calculated_number eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
if(isnan(n1)) return NAN;
if(isinf(n1)) return INFINITY;
return -n1;
}
calculated_number eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
calculated_number n1 = eval_value(exp, &op->ops[0], error);
NETDATA_DOUBLE eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
if(isnan(n1)) return NAN;
if(isinf(n1)) return INFINITY;
return ABS(n1);
}
calculated_number eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
NETDATA_DOUBLE eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
if(is_true(eval_value(exp, &op->ops[0], error)))
return eval_value(exp, &op->ops[1], error);
else
@ -310,7 +310,7 @@ static struct operator {
char precedence;
char parameters;
char isfunction;
calculated_number (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
NETDATA_DOUBLE (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
} operators[256] = {
// this is a random access array
// we always access it with a known EVAL_OPERATOR_X
@ -341,13 +341,13 @@ static struct operator {
#define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence)
static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
if(unlikely(op->count != operators[op->operator].parameters)) {
*error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
return 0;
}
calculated_number n = operators[op->operator].eval(exp, op, error);
NETDATA_DOUBLE n = operators[op->operator].eval(exp, op, error);
return n;
}
@ -360,7 +360,7 @@ static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *
buffer_sprintf(out, "${%s}", v->name);
}
static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) {
static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) {
if(unlikely(isnan(n))) {
buffer_strcat(out, "nan");
return;
@ -372,7 +372,7 @@ static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) {
}
char b[100+1], *s;
snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n);
snprintfz(b, 100, NETDATA_DOUBLE_FORMAT, n);
s = &b[strlen(b) - 1];
while(s > b && *s == '0') {
@ -737,9 +737,9 @@ static inline int parse_variable(const char **string, char *buffer, size_t len)
return 0;
}
static inline int parse_constant(const char **string, calculated_number *number) {
static inline int parse_constant(const char **string, NETDATA_DOUBLE *number) {
char *end = NULL;
calculated_number n = str2ld(*string, &end);
NETDATA_DOUBLE n = str2ndd(*string, &end);
if(unlikely(!end || *string == end)) {
*number = 0;
return 0;
@ -845,7 +845,7 @@ static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE
op->ops[pos].expression = value;
}
static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, calculated_number value) {
static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, NETDATA_DOUBLE value) {
if(pos >= op->count)
fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
@ -911,7 +911,7 @@ static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **stri
static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) {
char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1];
EVAL_NODE *op1 = NULL;
calculated_number number;
NETDATA_DOUBLE number;
*error = EVAL_ERROR_OK;

6
libnetdata/eval/eval.h
View file

@ -28,11 +28,11 @@ typedef struct eval_expression {
const char *parsed_as;
RRDCALC_STATUS *status;
calculated_number *myself;
NETDATA_DOUBLE *myself;
time_t *after;
time_t *before;
calculated_number result;
NETDATA_DOUBLE result;
int error;
BUFFER *error_msg;
@ -83,6 +83,6 @@ extern const char *expression_strerror(int error);
// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg)
extern int expression_evaluate(EVAL_EXPRESSION *expression);
extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result);
extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result);
#endif //NETDATA_EVAL_H

71
libnetdata/inlined.h
View file

@ -151,77 +151,6 @@ static inline long long str2ll(const char *s, char **endptr) {
return n;
}
static inline long double str2ld(const char *s, char **endptr) {
int negative = 0;
const char *start = s;
unsigned long long integer_part = 0;
unsigned long decimal_part = 0;
size_t decimal_digits = 0;
switch(*s) {
case '-':
s++;
negative = 1;
break;
case '+':
s++;
break;
case 'n':
if(s[1] == 'a' && s[2] == 'n') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
break;
case 'i':
if(s[1] == 'n' && s[2] == 'f') {
if(endptr) *endptr = (char *)&s[3];
return INFINITY;
}
break;
default:
break;
}
while (*s >= '0' && *s <= '9') {
integer_part = (integer_part * 10) + (*s - '0');
s++;
}
if(unlikely(*s == '.')) {
decimal_part = 0;
s++;
while (*s >= '0' && *s <= '9') {
decimal_part = (decimal_part * 10) + (*s - '0');
s++;
decimal_digits++;
}
}
if(unlikely(*s == 'e' || *s == 'E'))
return strtold(start, endptr);
if(unlikely(endptr))
*endptr = (char *)s;
if(unlikely(negative)) {
if(unlikely(decimal_digits))
return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits));
else
return -((long double)integer_part);
}
else {
if(unlikely(decimal_digits))
return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits);
else
return (long double)integer_part;
}
}
static inline char *strncpyz(char *dst, const char *src, size_t n) {
char *p = dst;

4
libnetdata/json/json.c
View file

@ -111,7 +111,7 @@ int json_callback_print(JSON_ENTRY *e)
break;
case JSON_NUMBER:
sprintf(txt,"%Lf", e->data.number);
sprintf(txt, NETDATA_DOUBLE_FORMAT_AUTO, e->data.number);
buffer_strcat(wb,txt);
break;
@ -168,7 +168,7 @@ static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t valu
e->type = JSON_NUMBER;
memcpy(e->name, key, len);
e->name[len] = 0;
e->data.number = value;
e->data.number = (NETDATA_DOUBLE)value;
}
/**

10
libnetdata/json/json.h
View file

@ -31,12 +31,12 @@ typedef struct json_entry {
char name[JSON_NAME_LEN + 1];
char fullname[JSON_FULLNAME_LEN + 1];
union {
char *string; // type == JSON_STRING
long double number; // type == JSON_NUMBER
int boolean; // type == JSON_BOOLEAN
size_t items; // type == JSON_ARRAY
char *string; // type == JSON_STRING
NETDATA_DOUBLE number; // type == JSON_NUMBER
int boolean; // type == JSON_BOOLEAN
size_t items; // type == JSON_ARRAY
} data;
size_t pos; // the position of this item in its parent
size_t pos; // the position of this item in its parent
char *original_string;

2
libnetdata/required_dummies.h
View file

@ -23,7 +23,7 @@ void signals_unblock(void){};
void signals_reset(void){};
// callback required by eval()
int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result)
int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result)
{
(void)variable;
(void)hash;

180
libnetdata/statistical/statistical.c
View file

@ -2,28 +2,28 @@
#include "../libnetdata.h"
LONG_DOUBLE default_single_exponential_smoothing_alpha = 0.1;
NETDATA_DOUBLE default_single_exponential_smoothing_alpha = 0.1;
void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg) {
const LONG_DOUBLE *value, *end = &series[entries];
void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg) {
const NETDATA_DOUBLE *value, *end = &series[entries];
fprintf(stderr, "%s of %zu entries [ ", msg, entries);
for(value = series; value < end ;value++) {
if(value != series) fprintf(stderr, ", ");
fprintf(stderr, "%" LONG_DOUBLE_MODIFIER, *value);
fprintf(stderr, "%" NETDATA_DOUBLE_MODIFIER, *value);
}
fprintf(stderr, " ] results in " CALCULATED_NUMBER_FORMAT "\n", result);
fprintf(stderr, " ] results in " NETDATA_DOUBLE_FORMAT "\n", result);
}
// --------------------------------------------------------------------------------------------------------------------
inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count) {
const LONG_DOUBLE *value, *end = &series[entries];
LONG_DOUBLE sum = 0;
inline NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count) {
const NETDATA_DOUBLE *value, *end = &series[entries];
NETDATA_DOUBLE sum = 0;
size_t c = 0;
for(value = series; value < end ; value++) {
if(calculated_number_isnumber(*value)) {
if(netdata_double_isnumber(*value)) {
sum += *value;
c++;
}
@ -35,42 +35,42 @@ inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size
return sum;
}
inline LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries) {
inline NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries) {
return sum_and_count(series, entries, NULL);
}
inline LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries) {
inline NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries) {
size_t count = 0;
LONG_DOUBLE sum = sum_and_count(series, entries, &count);
NETDATA_DOUBLE sum = sum_and_count(series, entries, &count);
if(unlikely(!count)) return NAN;
return sum / (LONG_DOUBLE)count;
return sum / (NETDATA_DOUBLE)count;
}
// --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period) {
NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period) {
if(unlikely(period <= 0))
return 0.0;
size_t i, count;
LONG_DOUBLE sum = 0, avg = 0;
LONG_DOUBLE p[period];
NETDATA_DOUBLE sum = 0, avg = 0;
NETDATA_DOUBLE p[period];
for(count = 0; count < period ; count++)
p[count] = 0.0;
for(i = 0, count = 0; i < entries; i++) {
LONG_DOUBLE value = series[i];
if(unlikely(!calculated_number_isnumber(value))) continue;
NETDATA_DOUBLE value = series[i];
if(unlikely(!netdata_double_isnumber(value))) continue;
if(unlikely(count < period)) {
sum += value;
avg = (count == period - 1) ? sum / (LONG_DOUBLE)period : 0;
avg = (count == period - 1) ? sum / (NETDATA_DOUBLE)period : 0;
}
else {
sum = sum - p[count % period] + value;
avg = sum / (LONG_DOUBLE)period;
avg = sum / (NETDATA_DOUBLE)period;
}
p[count % period] = value;
@ -83,8 +83,8 @@ LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t per
// --------------------------------------------------------------------------------------------------------------------
static int qsort_compare(const void *a, const void *b) {
LONG_DOUBLE *p1 = (LONG_DOUBLE *)a, *p2 = (LONG_DOUBLE *)b;
LONG_DOUBLE n1 = *p1, n2 = *p2;
NETDATA_DOUBLE *p1 = (NETDATA_DOUBLE *)a, *p2 = (NETDATA_DOUBLE *)b;
NETDATA_DOUBLE n1 = *p1, n2 = *p2;
if(unlikely(isnan(n1) || isnan(n2))) {
if(isnan(n1) && !isnan(n2)) return -1;
@ -102,22 +102,22 @@ static int qsort_compare(const void *a, const void *b) {
return 0;
}
inline void sort_series(LONG_DOUBLE *series, size_t entries) {
qsort(series, entries, sizeof(LONG_DOUBLE), qsort_compare);
inline void sort_series(NETDATA_DOUBLE *series, size_t entries) {
qsort(series, entries, sizeof(NETDATA_DOUBLE), qsort_compare);
}
inline LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries) {
LONG_DOUBLE *copy = mallocz(sizeof(LONG_DOUBLE) * entries);
memcpy(copy, series, sizeof(LONG_DOUBLE) * entries);
inline NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries) {
NETDATA_DOUBLE *copy = mallocz(sizeof(NETDATA_DOUBLE) * entries);
memcpy(copy, series, sizeof(NETDATA_DOUBLE) * entries);
return copy;
}
LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) {
NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0];
if(unlikely(entries == 2)) return (series[0] + series[1]) / 2;
LONG_DOUBLE average;
NETDATA_DOUBLE average;
if(entries % 2 == 0) {
size_t m = entries / 2;
average = (series[m] + series[m + 1]) / 2;
@ -129,17 +129,17 @@ LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) {
return average;
}
LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) {
NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0];
if(unlikely(entries == 2))
return (series[0] + series[1]) / 2;
LONG_DOUBLE *copy = copy_series(series, entries);
NETDATA_DOUBLE *copy = copy_series(series, entries);
sort_series(copy, entries);
LONG_DOUBLE avg = median_on_sorted_series(copy, entries);
NETDATA_DOUBLE avg = median_on_sorted_series(copy, entries);
freez(copy);
return avg;
@ -147,18 +147,18 @@ LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) {
// --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period) {
NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period) {
if(entries <= period)
return median(series, entries);
LONG_DOUBLE *data = copy_series(series, entries);
NETDATA_DOUBLE *data = copy_series(series, entries);
size_t i;
for(i = period; i < entries; i++) {
data[i - period] = median(&series[i - period], period);
}
LONG_DOUBLE avg = median(data, entries - period);
NETDATA_DOUBLE avg = median(data, entries - period);
freez(data);
return avg;
}
@ -166,17 +166,17 @@ LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t peri
// --------------------------------------------------------------------------------------------------------------------
// http://stackoverflow.com/a/15150143/4525767
LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) {
LONG_DOUBLE median = 0.0f;
LONG_DOUBLE average = 0.0f;
NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries) {
NETDATA_DOUBLE median = 0.0f;
NETDATA_DOUBLE average = 0.0f;
size_t i;
for(i = 0; i < entries ; i++) {
LONG_DOUBLE value = series[i];
if(unlikely(!calculated_number_isnumber(value))) continue;
NETDATA_DOUBLE value = series[i];
if(unlikely(!netdata_double_isnumber(value))) continue;
average += ( value - average ) * 0.1f; // rough running average.
median += copysignl( average * 0.01, value - median );
median += copysignndd( average * 0.01, value - median );
}
return median;
@ -184,16 +184,16 @@ LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) {
// --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) {
NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries) {
if(unlikely(entries == 0)) return NAN;
if(unlikely(entries == 1)) return series[0];
const LONG_DOUBLE *value, *end = &series[entries];
const NETDATA_DOUBLE *value, *end = &series[entries];
size_t count;
LONG_DOUBLE sum;
NETDATA_DOUBLE sum;
for(count = 0, sum = 0, value = series ; value < end ;value++) {
if(likely(calculated_number_isnumber(*value))) {
if(likely(netdata_double_isnumber(*value))) {
count++;
sum += *value;
}
@ -202,55 +202,55 @@ LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) {
if(unlikely(count == 0)) return NAN;
if(unlikely(count == 1)) return sum;
LONG_DOUBLE average = sum / (LONG_DOUBLE)count;
NETDATA_DOUBLE average = sum / (NETDATA_DOUBLE)count;
for(count = 0, sum = 0, value = series ; value < end ;value++) {
if(calculated_number_isnumber(*value)) {
if(netdata_double_isnumber(*value)) {
count++;
sum += powl(*value - average, 2);
sum += powndd(*value - average, 2);
}
}
if(unlikely(count == 0)) return NAN;
if(unlikely(count == 1)) return average;
LONG_DOUBLE variance = sum / (LONG_DOUBLE)(count); // remove -1 from count to have a population stddev
LONG_DOUBLE stddev = sqrtl(variance);
NETDATA_DOUBLE variance = sum / (NETDATA_DOUBLE)(count); // remove -1 from count to have a population stddev
NETDATA_DOUBLE stddev = sqrtndd(variance);
return stddev;
}
// --------------------------------------------------------------------------------------------------------------------
LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) {
NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) {
if(unlikely(entries == 0))
return NAN;
if(unlikely(isnan(alpha)))
alpha = default_single_exponential_smoothing_alpha;
const LONG_DOUBLE *value = series, *end = &series[entries];
LONG_DOUBLE level = (1.0 - alpha) * (*value);
const NETDATA_DOUBLE *value = series, *end = &series[entries];
NETDATA_DOUBLE level = (1.0 - alpha) * (*value);
for(value++ ; value < end; value++) {
if(likely(calculated_number_isnumber(*value)))
if(likely(netdata_double_isnumber(*value)))
level = alpha * (*value) + (1.0 - alpha) * level;
}
return level;
}
LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) {
NETDATA_DOUBLE single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) {
if(unlikely(entries == 0))
return NAN;
if(unlikely(isnan(alpha)))
alpha = default_single_exponential_smoothing_alpha;
const LONG_DOUBLE *value = &series[entries -1];
LONG_DOUBLE level = (1.0 - alpha) * (*value);
const NETDATA_DOUBLE *value = &series[entries -1];
NETDATA_DOUBLE level = (1.0 - alpha) * (*value);
for(value++ ; value >= series; value--) {
if(likely(calculated_number_isnumber(*value)))
if(likely(netdata_double_isnumber(*value)))
level = alpha * (*value) + (1.0 - alpha) * level;
}
@ -260,11 +260,14 @@ LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size
// --------------------------------------------------------------------------------------------------------------------
// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/
LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast) {
NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE *forecast) {
if(unlikely(entries == 0))
return NAN;
LONG_DOUBLE level, trend;
NETDATA_DOUBLE level, trend;
if(unlikely(isnan(alpha)))
alpha = 0.3;
@ -279,11 +282,10 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri
else
trend = 0;
const LONG_DOUBLE *value = series;
const NETDATA_DOUBLE *value = series;
for(value++ ; value >= series; value--) {
if(likely(calculated_number_isnumber(*value))) {
LONG_DOUBLE last_level = level;
if(likely(netdata_double_isnumber(*value))) {
NETDATA_DOUBLE last_level = level;
level = alpha * *value + (1.0 - alpha) * (level + trend);
trend = beta * (level - last_level) + (1.0 - beta) * trend;
@ -320,24 +322,26 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri
* s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p]
*/
static int __HoltWinters(
const LONG_DOUBLE *series,
const NETDATA_DOUBLE *series,
int entries, // start_time + h
LONG_DOUBLE alpha, // alpha parameter of Holt-Winters Filter.
LONG_DOUBLE beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
LONG_DOUBLE gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
NETDATA_DOUBLE alpha, // alpha parameter of Holt-Winters Filter.
NETDATA_DOUBLE
beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
NETDATA_DOUBLE
gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
const int *seasonal,
const int *period,
const LONG_DOUBLE *a, // Start value for level (a[0]).
const LONG_DOUBLE *b, // Start value for trend (b[0]).
LONG_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
const NETDATA_DOUBLE *a, // Start value for level (a[0]).
const NETDATA_DOUBLE *b, // Start value for trend (b[0]).
NETDATA_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
/* return values */
LONG_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing
LONG_DOUBLE *level, // Estimated values for the level component (size entries - t + 2)
LONG_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2)
LONG_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2)
NETDATA_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing
NETDATA_DOUBLE *level, // Estimated values for the level component (size entries - t + 2)
NETDATA_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2)
NETDATA_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2)
)
{
if(unlikely(entries < 4))
@ -345,13 +349,13 @@ static int __HoltWinters(
int start_time = 2;
LONG_DOUBLE res = 0, xhat = 0, stmp = 0;
NETDATA_DOUBLE res = 0, xhat = 0, stmp = 0;
int i, i0, s0;
/* copy start values to the beginning of the vectors */
level[0] = *a;
if(beta > 0) trend[0] = *b;
if(gamma > 0) memcpy(season, s, *period * sizeof(LONG_DOUBLE));
if(gamma > 0) memcpy(season, s, *period * sizeof(NETDATA_DOUBLE));
for(i = start_time - 1; i < entries; i++) {
/* indices for period i */
@ -397,7 +401,11 @@ static int __HoltWinters(
return 1;
}
LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast) {
NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE gamma,
NETDATA_DOUBLE *forecast) {
if(unlikely(isnan(alpha)))
alpha = 0.3;
@ -409,15 +417,15 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a
int seasonal = 0;
int period = 0;
LONG_DOUBLE a0 = series[0];
LONG_DOUBLE b0 = 0;
LONG_DOUBLE s[] = {};
NETDATA_DOUBLE a0 = series[0];
NETDATA_DOUBLE b0 = 0;
NETDATA_DOUBLE s[] = {};
LONG_DOUBLE errors = 0.0;
NETDATA_DOUBLE errors = 0.0;
size_t nb_computations = entries;
LONG_DOUBLE *estimated_level = callocz(nb_computations, sizeof(LONG_DOUBLE));
LONG_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(LONG_DOUBLE));
LONG_DOUBLE *estimated_season = callocz(nb_computations, sizeof(LONG_DOUBLE));
NETDATA_DOUBLE *estimated_level = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
NETDATA_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
NETDATA_DOUBLE *estimated_season = callocz(nb_computations, sizeof(NETDATA_DOUBLE));
int ret = __HoltWinters(
series,
@ -436,7 +444,7 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a
estimated_season
);
LONG_DOUBLE value = estimated_level[nb_computations - 1];
NETDATA_DOUBLE value = estimated_level[nb_computations - 1];
if(forecast)
*forecast = 0.0;

40
libnetdata/statistical/statistical.h
View file

@ -5,22 +5,30 @@
#include "../libnetdata.h"
extern void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg);
extern void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg);
extern LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period);
extern LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period);
extern LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha);
extern LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha);
extern LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast);
extern LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast);
extern LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count);
extern LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries);
extern LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries);
extern void sort_series(LONG_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period);
extern NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period);
extern NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha);
extern NETDATA_DOUBLE
single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha);
extern NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE *forecast);
extern NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries,
NETDATA_DOUBLE alpha,
NETDATA_DOUBLE beta,
NETDATA_DOUBLE gamma,
NETDATA_DOUBLE *forecast);
extern NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count);
extern NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries);
extern NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries);
extern void sort_series(NETDATA_DOUBLE *series, size_t entries);
#endif //NETDATA_STATISTICAL_H

34
libnetdata/storage_number/storage_number.c
View file

@ -2,7 +2,7 @@
#include "../libnetdata.h"
storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
// bit 32 = sign 0:positive, 1:negative
// bit 31 = 0:divide, 1:multiply
// bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
@ -19,7 +19,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
goto RET_SN;
int m = 0;
calculated_number n = value, factor = 10;
NETDATA_DOUBLE n = value, factor = 10;
// if the value is negative
// add the sign bit and make it positive
@ -36,7 +36,7 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// make its integer part fit in 0x00ffffff
// by dividing it by 10 up to 7 times
// and increasing the multiplier
while(m < 7 && n > (calculated_number)0x00ffffff) {
while(m < 7 && n > (NETDATA_DOUBLE)0x00ffffff) {
n /= factor;
m++;
}
@ -46,9 +46,9 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// so we add a multiplier to unpack it
r += (1 << 30) + (m << 27); // the multiplier m
if(n > (calculated_number)0x00ffffff) {
if(n > (NETDATA_DOUBLE)0x00ffffff) {
#ifdef NETDATA_INTERNAL_CHECKS
error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value);
#endif
r += 0x00ffffff;
goto RET_SN;
@ -60,12 +60,12 @@ storage_number pack_storage_number(calculated_number value, SN_FLAGS flags) {
// while the value is below 0x0019999e we can
// multiply it by 10, up to 7 times, increasing
// the multiplier
while(m < 7 && n < (calculated_number)0x0019999e) {
while(m < 7 && n < (NETDATA_DOUBLE)0x0019999e) {
n *= 10;
m++;
}
if (unlikely(n > (calculated_number) (0x00ffffff))) {
if (unlikely(n > (NETDATA_DOUBLE) (0x00ffffff))) {
n /= 10;
m--;
}
@ -90,7 +90,7 @@ RET_SN:
}
// Lookup table to make storage number unpacking efficient.
calculated_number unpack_storage_number_lut10x[4 * 8];
NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
__attribute__((constructor)) void initialize_lut(void) {
// The lookup table is partitioned in 4 subtables based on the
@ -107,7 +107,7 @@ __attribute__((constructor)) void initialize_lut(void) {
}
/*
int print_calculated_number(char *str, calculated_number value)
int print_netdata_double(char *str, NETDATA_DOUBLE value)
{
char *wstr = str;
@ -117,9 +117,9 @@ int print_calculated_number(char *str, calculated_number value)
#ifdef STORAGE_WITH_MATH
// without llrintl() there are rounding problems
// for example 0.9 becomes 0.89
unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000);
unsigned long long uvalue = (unsigned long long int) llrintl(value * (NETDATA_DOUBLE)100000);
#else
unsigned long long uvalue = value * (calculated_number)100000;
unsigned long long uvalue = value * (NETDATA_DOUBLE)100000;
#endif
wstr = print_number_llu_r_smart(str, uvalue);
@ -163,8 +163,8 @@ int print_calculated_number(char *str, calculated_number value)
}
*/
int print_calculated_number(char *str, calculated_number value) {
// info("printing number " CALCULATED_NUMBER_FORMAT, value);
int print_netdata_double(char *str, NETDATA_DOUBLE value) {
// info("printing number " NETDATA_DOUBLE_FORMAT, value);
char integral_str[50], fractional_str[50];
char *wstr = str;
@ -174,22 +174,22 @@ int print_calculated_number(char *str, calculated_number value) {
value = -value;
}
calculated_number integral, fractional;
NETDATA_DOUBLE integral, fractional;
#ifdef STORAGE_WITH_MATH
fractional = calculated_number_modf(value, &integral) * 10000000.0;
fractional = modfndd(value, &integral) * 10000000.0;
#else
fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
#endif
unsigned long long integral_int = (unsigned long long)integral;
unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional);
unsigned long long fractional_int = (unsigned long long)llrintndd(fractional);
if(unlikely(fractional_int >= 10000000)) {
integral_int += 1;
fractional_int -= 10000000;
}
// info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
// info("integral " NETDATA_DOUBLE_FORMAT " (%llu), fractional " NETDATA_DOUBLE_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
char *istre;
if(unlikely(integral_int == 0)) {

163
libnetdata/storage_number/storage_number.h
View file

@ -6,68 +6,60 @@
#include <math.h>
#include "../libnetdata.h"
#ifdef NETDATA_WITHOUT_LONG_DOUBLE
#ifdef NETDATA_WITH_LONG_DOUBLE
#define powl pow
#define modfl modf
#define llrintl llrint
#define roundl round
#define sqrtl sqrt
#define copysignl copysign
#define strtold strtod
typedef long double NETDATA_DOUBLE;
#define NETDATA_DOUBLE_FORMAT "%0.7Lf"
#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0Lf"
#define NETDATA_DOUBLE_FORMAT_AUTO "%Lf"
#define NETDATA_DOUBLE_MODIFIER "Lf"
typedef double calculated_number;
#define CALCULATED_NUMBER_FORMAT "%0.7f"
#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f"
#define CALCULATED_NUMBER_FORMAT_AUTO "%f"
#define NETDATA_DOUBLE_MAX LDBL_MAX
#define LONG_DOUBLE_MODIFIER "f"
typedef double LONG_DOUBLE;
#define strtondd(s, endptr) strtold(s, endptr)
#define powndd(x, y) powl(x, y)
#define llrintndd(x) llrintl(x)
#define roundndd(x) roundl(x)
#define sqrtndd(x) sqrtl(x)
#define copysignndd(x, y) copysignl(x, y)
#define modfndd(x, y) modfl(x, y)
#define fabsndd(x) fabsl(x)
#define CALCULATED_NUMBER_MAX DBL_MAX
#else // NETDATA_WITH_LONG_DOUBLE
#else // NETDATA_WITHOUT_LONG_DOUBLE
typedef double NETDATA_DOUBLE;
#define NETDATA_DOUBLE_FORMAT "%0.7f"
#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0f"
#define NETDATA_DOUBLE_FORMAT_AUTO "%f"
#define NETDATA_DOUBLE_MODIFIER "f"
typedef long double calculated_number;
#define CALCULATED_NUMBER_FORMAT "%0.7Lf"
#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf"
#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf"
#define NETDATA_DOUBLE_MAX DBL_MAX
#define LONG_DOUBLE_MODIFIER "Lf"
typedef long double LONG_DOUBLE;
#define strtondd(s, endptr) strtod(s, endptr)
#define powndd(x, y) pow(x, y)
#define llrintndd(x) llrint(x)
#define roundndd(x) round(x)
#define sqrtndd(x) sqrt(x)
#define copysignndd(x, y) copysign(x, y)
#define modfndd(x, y) modf(x, y)
#define fabsndd(x) fabs(x)
#define CALCULATED_NUMBER_MAX LDBL_MAX
#endif // NETDATA_WITHOUT_LONG_DOUBLE
//typedef long long calculated_number;
//#define CALCULATED_NUMBER_FORMAT "%lld"
#endif // NETDATA_WITH_LONG_DOUBLE
typedef long long collected_number;
#define COLLECTED_NUMBER_FORMAT "%lld"
/*
typedef long double collected_number;
#define COLLECTED_NUMBER_FORMAT "%0.7Lf"
*/
#define calculated_number_modf(x, y) modfl(x, y)
#define calculated_number_llrint(x) llrintl(x)
#define calculated_number_round(x) roundl(x)
#define calculated_number_fabs(x) fabsl(x)
#define calculated_number_pow(x, y) powl(x, y)
#define calculated_number_epsilon (calculated_number)0.0000001
#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)
#define epsilonndd (NETDATA_DOUBLE)0.0000001
#define considered_equal_ndd(a, b) (fabsndd((a) - (b)) < epsilonndd)
#if defined(HAVE_ISFINITE) || defined(isfinite)
// The isfinite() macro shall determine whether its argument has a
// finite value (zero, subnormal, or normal, and not infinite or NaN).
#define calculated_number_isnumber(a) (isfinite(a))
#define netdata_double_isnumber(a) (isfinite(a))
#elif defined(HAVE_FINITE) || defined(finite)
#define calculated_number_isnumber(a) (finite(a))
#define netdata_double_isnumber(a) (finite(a))
#else
#define calculated_number_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
#define netdata_double_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE)
#endif
typedef uint32_t storage_number;
@ -95,10 +87,10 @@ typedef enum {
#define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)
#define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)
storage_number pack_storage_number(calculated_number value, SN_FLAGS flags);
static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const));
storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags);
static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attribute__((const));
int print_calculated_number(char *str, calculated_number value);
int print_netdata_double(char *str, NETDATA_DOUBLE value);
// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE
#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
@ -115,8 +107,8 @@ int print_calculated_number(char *str, calculated_number value);
#define MAX_INCREMENTAL_PERCENT_RATE 10
static inline calculated_number unpack_storage_number(storage_number value) {
extern calculated_number unpack_storage_number_lut10x[4 * 8];
static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) {
extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8];
if(unlikely(value == SN_EMPTY_SLOT))
return NAN;
@ -145,11 +137,82 @@ static inline calculated_number unpack_storage_number(storage_number value) {
// bit 24 to bit 1 = the value, so remove all other bits
value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
calculated_number n = value;
NETDATA_DOUBLE n = value;
// fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n);
return sign * unpack_storage_number_lut10x[(factor * 16) + (exp * 8) + mul] * n;
}
static inline NETDATA_DOUBLE str2ndd(const char *s, char **endptr) {
int negative = 0;
const char *start = s;
unsigned long long integer_part = 0;
unsigned long decimal_part = 0;
size_t decimal_digits = 0;
switch(*s) {
case '-':
s++;
negative = 1;
break;
case '+':
s++;
break;
case 'n':
if(s[1] == 'a' && s[2] == 'n') {
if(endptr) *endptr = (char *)&s[3];
return NAN;
}
break;
case 'i':
if(s[1] == 'n' && s[2] == 'f') {
if(endptr) *endptr = (char *)&s[3];
return INFINITY;
}
break;
default:
break;
}
while (*s >= '0' && *s <= '9') {
integer_part = (integer_part * 10) + (*s - '0');
s++;
}
if(unlikely(*s == '.')) {
decimal_part = 0;
s++;
while (*s >= '0' && *s <= '9') {
decimal_part = (decimal_part * 10) + (*s - '0');
s++;
decimal_digits++;
}
}
if(unlikely(*s == 'e' || *s == 'E'))
return strtondd(start, endptr);
if(unlikely(endptr))
*endptr = (char *)s;
if(unlikely(negative)) {
if(unlikely(decimal_digits))
return -((NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits));
else
return -((NETDATA_DOUBLE)integer_part);
}
else {
if(unlikely(decimal_digits))
return (NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits);
else
return (NETDATA_DOUBLE)integer_part;
}
}
#endif /* NETDATA_STORAGE_NUMBER_H */

20
libnetdata/storage_number/tests/test_storage_number.c
View file

@ -11,34 +11,34 @@ static void test_number_printing(void **state)
char value[50];
print_calculated_number(value, 0);
print_netdata_double(value, 0);
assert_string_equal(value, "0");
print_calculated_number(value, 0.0000001);
print_netdata_double(value, 0.0000001);
assert_string_equal(value, "0.0000001");
print_calculated_number(value, 0.00000009);
print_netdata_double(value, 0.00000009);
assert_string_equal(value, "0.0000001");
print_calculated_number(value, 0.000000001);
print_netdata_double(value, 0.000000001);
assert_string_equal(value, "0");
print_calculated_number(value, 99.99999999999999999);
print_netdata_double(value, 99.99999999999999999);
assert_string_equal(value, "100");
print_calculated_number(value, -99.99999999999999999);
print_netdata_double(value, -99.99999999999999999);
assert_string_equal(value, "-100");
print_calculated_number(value, 123.4567890123456789);
print_netdata_double(value, 123.4567890123456789);
assert_string_equal(value, "123.456789");
print_calculated_number(value, 9999.9999999);
print_netdata_double(value, 9999.9999999);
assert_string_equal(value, "9999.9999999");
print_calculated_number(value, -9999.9999999);
print_netdata_double(value, -9999.9999999);
assert_string_equal(value, "-9999.9999999");
print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
print_netdata_double(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
assert_string_equal(value, "16.77722");
}

4
libnetdata/tests/test_str2ld.c
View file

@ -24,8 +24,8 @@ static void test_str2ld(void **state)
for (int i = 0; values[i]; i++) {
char *e_mine = "hello", *e_sys = "world";
LONG_DOUBLE mine = str2ld(values[i], &e_mine);
LONG_DOUBLE sys = strtold(values[i], &e_sys);
NETDATA_DOUBLE mine = str2ndd(values[i], &e_mine);
NETDATA_DOUBLE sys = strtondd(values[i], &e_sys);
if (isnan(mine))
assert_true(isnan(sys));

2
ml/Dimension.cc
View file

@ -42,7 +42,7 @@ TrainableDimension::getCalculatedNumbers() {
auto P = Q.nextMetric();
CalculatedNumber Value = P.second;
if (calculated_number_isnumber(Value)) {
if (netdata_double_isnumber(Value)) {
CNs[Idx] = Value;
LastValue = CNs[Idx];
CollectedValues++;

4
ml/ml.cc
View file

@ -186,10 +186,10 @@ void ml_process_rrdr(RRDR *R, int MaxAnomalyRates) {
if (MaxAnomalyRates < 1 || MaxAnomalyRates >= R->d)
return;
calculated_number *CNs = R->v;
NETDATA_DOUBLE *CNs = R->v;
RRDR_DIMENSION_FLAGS *DimFlags = R->od;
std::vector<std::pair<calculated_number, int>> V;
std::vector<std::pair<NETDATA_DOUBLE, int>> V;
V.reserve(R->d);
for (int Idx = 0; Idx != R->d; Idx++)

2
parser/parser.h
View file

@ -28,7 +28,7 @@ typedef struct pluginsd_action {
PARSER_RC (*flush_action)(void *user, RRDSET *st);
PARSER_RC (*disable_action)(void *user);
PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, NETDATA_DOUBLE value);
PARSER_RC (*label_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
PARSER_RC (*clabel_action)(void *user, char *key, char *value, RRDLABEL_SRC source);

13
streaming/rrdpush.c
View file

@ -75,7 +75,8 @@ int rrdpush_init() {
default_rrdpush_destination = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "destination", "");
default_rrdpush_api_key = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "api key", "");
default_rrdpush_send_charts_matching = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "send charts matching", "*");
rrdhost_free_orphan_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds", rrdhost_free_orphan_time);
rrdhost_free_orphan_time = config_get_number(CONFIG_SECTION_DB, "cleanup orphan hosts after secs", rrdhost_free_orphan_time);
#ifdef ENABLE_COMPRESSION
default_compression_enabled = (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
"enable compression", default_compression_enabled);
@ -97,10 +98,10 @@ int rrdpush_init() {
}
}
char *invalid_certificate = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", "no");
bool invalid_certificate = appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", CONFIG_BOOLEAN_NO);
if ( !strcmp(invalid_certificate,"yes")){
if (netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE){
if(invalid_certificate == CONFIG_BOOLEAN_YES){
if(netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE){
info("Netdata is configured to accept invalid SSL certificate.");
netdata_validate_server = NETDATA_SSL_INVALID_CERTIFICATE;
}
@ -272,11 +273,11 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
RRDSETVAR *rs;
for(rs = st->variables; rs ;rs = rs->next) {
if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
calculated_number *value = (calculated_number *) rs->value;
NETDATA_DOUBLE *value = (NETDATA_DOUBLE *) rs->value;
buffer_sprintf(
host->sender->build
, "VARIABLE CHART %s = " CALCULATED_NUMBER_FORMAT "\n"
, "VARIABLE CHART %s = " NETDATA_DOUBLE_FORMAT "\n"
, rs->variable
, *value
);

6
streaming/sender.c
View file

@ -89,16 +89,16 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) {
}
static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) {
calculated_number *value = (calculated_number *)rv->value;
NETDATA_DOUBLE *value = (NETDATA_DOUBLE *)rv->value;
buffer_sprintf(
host->sender->build
, "VARIABLE HOST %s = " CALCULATED_NUMBER_FORMAT "\n"
, "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n"
, rv->name
, *value
);
debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " CALCULATED_NUMBER_FORMAT, rv->name, *value);
debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rv->name, *value);
}
void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {

10
tests/profile/test-eval.c
View file

@ -58,9 +58,9 @@ void print_node(EVAL_NODE *op, int level) {
while(i--) print_value(&op->ops[i], level + 1);
}
calculated_number evaluate(EVAL_NODE *op, int depth);
NETDATA_DOUBLE evaluate(EVAL_NODE *op, int depth);
calculated_number evaluate_value(EVAL_VALUE *v, int depth) {
NETDATA_DOUBLE evaluate_value(EVAL_VALUE *v, int depth) {
switch(v->type) {
case EVAL_VALUE_NUMBER:
return v->number;
@ -80,8 +80,8 @@ void print_depth(int depth) {
while(depth--) printf(" ");
}
calculated_number evaluate(EVAL_NODE *op, int depth) {
calculated_number n1, n2, r;
NETDATA_DOUBLE evaluate(EVAL_NODE *op, int depth) {
NETDATA_DOUBLE n1, n2, r;
switch(op->operator) {
case EVAL_OPERATOR_SIGN_PLUS:
@ -249,7 +249,7 @@ void print_expression(EVAL_NODE *op, const char *failed_at, int error) {
evaluate(op, 0);
int error;
calculated_number ret = expression_evaluate(op, &error);
NETDATA_DOUBLE ret = expression_evaluate(op, &error);
printf("\ninternal evaluator:\nSTATUS: %d, RESULT = %Lf\n", error, ret);
expression_free(op);

39
web/api/badges/web_buffer_svg.c
View file

@ -249,7 +249,8 @@ cleanup:
return len - i;
}
static inline char *format_value_with_precision_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision) {
static inline char *format_value_with_precision_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision) {
if(unlikely(isnan(value) || isinf(value)))
value = 0.0;
@ -260,23 +261,23 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz
if(precision < 0) {
int len, lstop = 0, trim_zeros = 1;
calculated_number abs = value;
NETDATA_DOUBLE abs = value;
if(isless(value, 0)) {
lstop = 1;
abs = calculated_number_fabs(value);
abs = fabsndd(value);
}
if(isgreaterequal(abs, 1000)) {
len = snprintfz(value_string, value_string_len, "%0.0" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
len = snprintfz(value_string, value_string_len, "%0.0" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
trim_zeros = 0;
}
else if(isgreaterequal(abs, 10)) len = snprintfz(value_string, value_string_len, "%0.1" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 1)) len = snprintfz(value_string, value_string_len, "%0.2" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 0.1)) len = snprintfz(value_string, value_string_len, "%0.2" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 0.01)) len = snprintfz(value_string, value_string_len, "%0.4" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 0.001)) len = snprintfz(value_string, value_string_len, "%0.5" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 0.0001)) len = snprintfz(value_string, value_string_len, "%0.6" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else len = snprintfz(value_string, value_string_len, "%0.7" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE) value);
else if(isgreaterequal(abs, 10)) len = snprintfz(value_string, value_string_len, "%0.1" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 1)) len = snprintfz(value_string, value_string_len, "%0.2" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.1)) len = snprintfz(value_string, value_string_len, "%0.2" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.01)) len = snprintfz(value_string, value_string_len, "%0.4" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.001)) len = snprintfz(value_string, value_string_len, "%0.5" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else if(isgreaterequal(abs, 0.0001)) len = snprintfz(value_string, value_string_len, "%0.6" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
else len = snprintfz(value_string, value_string_len, "%0.7" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE) value);
if(unlikely(trim_zeros)) {
int l;
@ -303,7 +304,7 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz
}
else {
if(precision > 50) precision = 50;
snprintfz(value_string, value_string_len, "%0.*" LONG_DOUBLE_MODIFIER "%s%s", precision, (LONG_DOUBLE) value, separator, units);
snprintfz(value_string, value_string_len, "%0.*" NETDATA_DOUBLE_MODIFIER "%s%s", precision, (NETDATA_DOUBLE) value, separator, units);
}
return value_string;
@ -359,7 +360,8 @@ static struct units_formatter {
{ NULL, 0, UNITS_FORMAT_NONE }
};
inline char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision) {
inline char *format_value_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision) {
static int max = -1;
int i;
@ -555,7 +557,7 @@ typedef enum color_comparison {
COLOR_COMPARE_GREATEREQUAL,
} BADGE_COLOR_COMPARISON;
static inline void calc_colorz(const char *color, char *final, size_t len, calculated_number value) {
static inline void calc_colorz(const char *color, char *final, size_t len, NETDATA_DOUBLE value) {
if(isnan(value) || isinf(value))
value = NAN;
@ -642,7 +644,7 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu
*dc = '\0';
if(dv) {
*dv = '\0';
calculated_number v;
NETDATA_DOUBLE v;
if(!*value_buffer || !strcmp(value_buffer, "null")) {
v = NAN;
@ -732,7 +734,8 @@ static const char *parse_color_argument(const char *arg, const char *def)
return color_map(arg, def);
}
void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val) {
void buffer_svg(BUFFER *wb, const char *label,
NETDATA_DOUBLE value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val) {
char value_color_buffer[COLOR_STRING_SIZE + 1]
, value_string[VALUE_STRING_SIZE + 1]
, label_escaped[LABEL_STRING_SIZE + 1]
@ -750,7 +753,7 @@ void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const ch
value_color = (isnan(value) || isinf(value))?"999":"4c1";
calc_colorz(value_color, value_color_buffer, COLOR_STRING_SIZE, value);
format_value_and_unit(value_string, VALUE_STRING_SIZE, (options & RRDR_OPTION_DISPLAY_ABS)?calculated_number_fabs(value):value, units, precision);
format_value_and_unit(value_string, VALUE_STRING_SIZE, (options & RRDR_OPTION_DISPLAY_ABS)? fabsndd(value):value, units, precision);
if(fixed_width_lbl <= 0 || fixed_width_val <= 0) {
label_width = verdana11_width(label, font_size) + (BADGE_HORIZONTAL_PADDING * 2);
@ -1098,7 +1101,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
else {
time_t latest_timestamp = 0;
int value_is_null = 1;
calculated_number n = NAN;
NETDATA_DOUBLE n = NAN;
ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
// if the collected value is too old, don't calculate its value

6
web/api/badges/web_buffer_svg.h
View file

@ -6,8 +6,10 @@
#include "libnetdata/libnetdata.h"
#include "web/server/web_client.h"
extern void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val);
extern char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision);
extern void buffer_svg(BUFFER *wb, const char *label,
NETDATA_DOUBLE value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val);
extern char *format_value_and_unit(char *value_string, size_t value_string_len,
NETDATA_DOUBLE value, const char *units, int precision);
extern int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url);

20
web/api/exporters/shell/allmetrics_shell.c
View file

@ -33,7 +33,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
if (filter && !simple_pattern_matches(filter, st->name))
continue;
calculated_number total = 0.0;
NETDATA_DOUBLE total = 0.0;
char chart[SHELL_ELEMENT_MAX + 1];
shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
@ -48,21 +48,21 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
char dimension[SHELL_ELEMENT_MAX + 1];
shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
calculated_number n = rd->last_stored_value;
NETDATA_DOUBLE n = rd->last_stored_value;
if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units);
else {
if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
n = calculated_number_round(n);
n = roundndd(n);
if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
buffer_sprintf(wb, "NETDATA_%s_%s=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units);
buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, dimension, n, st->units);
}
}
}
total = calculated_number_round(total);
buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, total, st->units);
total = roundndd(total);
buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, total, st->units);
rrdset_unlock(st);
}
}
@ -79,13 +79,13 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
char alarm[SHELL_ELEMENT_MAX + 1];
shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
calculated_number n = rc->value;
NETDATA_DOUBLE n = rc->value;
if(isnan(n) || isinf(n))
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units);
else {
n = calculated_number_round(n);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" CALCULATED_NUMBER_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units);
n = roundndd(n);
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\" # %s\n", chart, alarm, n, rc->units);
}
buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
@ -154,7 +154,7 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
if(isnan(rd->last_stored_value))
buffer_strcat(wb, "null");
else
buffer_sprintf(wb, CALCULATED_NUMBER_FORMAT, rd->last_stored_value);
buffer_sprintf(wb, NETDATA_DOUBLE_FORMAT, rd->last_stored_value);
buffer_strcat(wb, "\n\t\t\t}");

10
web/api/formatters/csv/csv.c
View file

@ -63,9 +63,9 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
}
// for each line in the array
calculated_number total = 1;
NETDATA_DOUBLE total = 1;
for(i = start; i != end ;i += step) {
calculated_number *cn = &r->v[ i * r->d ];
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
buffer_strcat(wb, betweenlines);
@ -75,7 +75,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
// print the timestamp of the line
buffer_rrd_value(wb, (calculated_number)now);
buffer_rrd_value(wb, (NETDATA_DOUBLE)now);
// in ms
if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
}
@ -90,7 +90,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
calculated_number n = cn[c];
NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;
@ -109,7 +109,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
buffer_strcat(wb, separator);
calculated_number n = cn[c];
NETDATA_DOUBLE n = cn[c];
if(co[c] & RRDR_VALUE_EMPTY) {
if(options & RRDR_OPTION_NULL2ZERO)

14
web/api/formatters/json/json.c
View file

@ -158,9 +158,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
);
// for each line in the array
calculated_number total = 1;
NETDATA_DOUBLE total = 1;
for(i = start; i != end ;i += step) {
calculated_number *cn = &r->v[ i * r->d ];
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
uint8_t *ar = &r->ar[ i * r->d ];
@ -210,7 +210,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
buffer_fast_strcat(wb, object_rows_time, object_rows_time_len);
buffer_rrd_value(wb, (calculated_number)r->t[i]);
buffer_rrd_value(wb, (NETDATA_DOUBLE)r->t[i]);
// in ms
if(unlikely(options & RRDR_OPTION_MILLISECONDS))
@ -223,9 +223,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
calculated_number n;
NETDATA_DOUBLE n;
if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
else
n = cn[c];
@ -244,9 +244,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
calculated_number n;
NETDATA_DOUBLE n;
if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
n = (calculated_number)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
n = (NETDATA_DOUBLE)ar[c] / 2.0; // rrdr stores anomaly rates 0 - 200
else
n = cn[c];

12
web/api/formatters/json_wrapper.c
View file

@ -257,7 +257,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", ");
i++;
calculated_number value = rd->last_stored_value;
NETDATA_DOUBLE value = rd->last_stored_value;
if (NAN == value)
buffer_strcat(wb, "null");
else
@ -282,13 +282,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
i = 0;
if(rows) {
calculated_number total = 1;
NETDATA_DOUBLE total = 1;
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c];
NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;
@ -306,9 +306,9 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
if(i) buffer_strcat(wb, ", ");
i++;
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c];
NETDATA_DOUBLE n = cn[c];
if(co[c] & RRDR_VALUE_EMPTY) {
if(options & RRDR_OPTION_NULL2ZERO)

4
web/api/formatters/rrd2json.c
View file

@ -86,7 +86,7 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis
(*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
rrddim_foreach_read(rd1, st) {
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize);
RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
rd->id = onewayalloc_strdupz(owa, rd1->id);
rd->name = onewayalloc_strdupz(owa, rd1->name);
rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
@ -152,7 +152,7 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) {
int rrdset2value_api_v1(
RRDSET *st
, BUFFER *wb
, calculated_number *n
, NETDATA_DOUBLE *n
, const char *dimensions
, long points
, long long after

6
web/api/formatters/rrd2json.h
View file

@ -67,8 +67,8 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
extern int rrdset2anything_api_v1(
ONEWAYALLOC *owa
, RRDSET *st
,
QUERY_PARAMS *query_params, BUFFER *dimensions
, QUERY_PARAMS *query_params
, BUFFER *dimensions
, uint32_t format
, long points
, long long after
@ -83,7 +83,7 @@ extern int rrdset2anything_api_v1(
extern int rrdset2value_api_v1(
RRDSET *st
, BUFFER *wb
, calculated_number *n
, NETDATA_DOUBLE *n
, const char *dimensions
, long points
, long long after

4
web/api/formatters/rrdset2json.c
View file

@ -85,14 +85,14 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
"\t\t\t\"dimensions\": {\n",
st->update_every);
unsigned long memory = st->memsize;
unsigned long memory = sizeof(RRDSET) + st->memsize;
size_t dimensions = 0;
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue;
memory += rd->memsize;
memory += sizeof(RRDDIM) + rd->memsize;
if (dimensions)
buffer_strcat(wb, ",\n\t\t\t\t\"");

2
web/api/formatters/ssv/ssv.c
View file

@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con
// for each line in the array
for(i = start; i != end ;i += step) {
int all_values_are_null = 0;
calculated_number v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);
NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);
if(likely(i != start)) {
if(r->min > v) r->min = v;

13
web/api/formatters/value/value.c
View file

@ -3,28 +3,29 @@
#include "value.h"
inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
inline NETDATA_DOUBLE
rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd) {
if (r->st_needs_lock)
rrdset_check_rdlock(r->st);
long c;
RRDDIM *d;
calculated_number *cn = &r->v[ i * r->d ];
NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
uint8_t *ar = &r->ar[ i * r->d ];
calculated_number sum = 0, min = 0, max = 0, v;
NETDATA_DOUBLE sum = 0, min = 0, max = 0, v;
int all_null = 1, init = 1;
calculated_number total = 1;
NETDATA_DOUBLE total = 1;
size_t total_anomaly_rate = 0;
int set_min_max = 0;
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
calculated_number n = cn[c];
NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;
@ -41,7 +42,7 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
calculated_number n = cn[c];
NETDATA_DOUBLE n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;

3
web/api/formatters/value/value.h
View file

@ -5,6 +5,7 @@
#include "../rrd2json.h"
extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd);
extern NETDATA_DOUBLE
rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, uint8_t *anomaly_rate, RRDDIM *temp_rd);
#endif //NETDATA_API_FORMATTER_VALUE_H

8
web/api/queries/average/average.c
View file

@ -6,7 +6,7 @@
// average
struct grouping_average {
calculated_number sum;
NETDATA_DOUBLE sum;
size_t count;
};
@ -27,16 +27,16 @@ void grouping_free_average(RRDR *r) {
r->internal.grouping_data = NULL;
}
void grouping_add_average(RRDR *r, calculated_number value) {
void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
g->sum += value;
g->count++;
}
calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data;
calculated_number value;
NETDATA_DOUBLE value;
if(unlikely(!g->count)) {
value = 0.0;

4
web/api/queries/average/average.h
View file

@ -9,7 +9,7 @@
extern void grouping_create_average(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_average(RRDR *r);
extern void grouping_free_average(RRDR *r);
extern void grouping_add_average(RRDR *r, calculated_number value);
extern calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_AVERAGE_H

26
web/api/queries/countif/countif.c
View file

@ -6,33 +6,33 @@
// countif
struct grouping_countif {
size_t (*comparison)(calculated_number, calculated_number);
calculated_number target;
size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE);
NETDATA_DOUBLE target;
size_t count;
size_t matched;
};
static size_t countif_equal(calculated_number v, calculated_number target) {
static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v == target);
}
static size_t countif_notequal(calculated_number v, calculated_number target) {
static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v != target);
}
static size_t countif_less(calculated_number v, calculated_number target) {
static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v < target);
}
static size_t countif_lessequal(calculated_number v, calculated_number target) {
static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v <= target);
}
static size_t countif_greater(calculated_number v, calculated_number target) {
static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v > target);
}
static size_t countif_greaterequal(calculated_number v, calculated_number target) {
static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) {
return (v >= target);
}
@ -89,7 +89,7 @@ void grouping_create_countif(RRDR *r, const char *options __maybe_unused) {
// skip everything up to the first digit
while(isspace(*options)) options++;
g->target = str2ld(options, NULL);
g->target = str2ndd(options, NULL);
}
else {
g->target = 0.0;
@ -110,23 +110,23 @@ void grouping_free_countif(RRDR *r) {
r->internal.grouping_data = NULL;
}
void grouping_add_countif(RRDR *r, calculated_number value) {
void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
g->matched += g->comparison(value, g->target);
g->count++;
}
calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data;
calculated_number value;
NETDATA_DOUBLE value;
if(unlikely(!g->count)) {
value = 0.0;
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
}
else {
value = (calculated_number)g->matched * 100 / (calculated_number)g->count;
value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count;
}
g->matched = 0;

4
web/api/queries/countif/countif.h
View file

@ -9,7 +9,7 @@
extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_countif(RRDR *r);
extern void grouping_free_countif(RRDR *r);
extern void grouping_add_countif(RRDR *r, calculated_number value);
extern calculated_number grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_COUNTIF_H

24
web/api/queries/des/des.c
View file

@ -8,13 +8,13 @@
// single exponential smoothing
struct grouping_des {
calculated_number alpha;
calculated_number alpha_other;
calculated_number beta;
calculated_number beta_other;
NETDATA_DOUBLE alpha;
NETDATA_DOUBLE alpha_other;
NETDATA_DOUBLE beta;
NETDATA_DOUBLE beta_other;
calculated_number level;
calculated_number trend;
NETDATA_DOUBLE level;
NETDATA_DOUBLE trend;
size_t count;
};
@ -31,10 +31,10 @@ void grouping_init_des(void) {
}
}
static inline calculated_number window(RRDR *r, struct grouping_des *g) {
static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) {
(void)g;
calculated_number points;
NETDATA_DOUBLE points;
if(r->group == 1) {
// provide a running DES
points = r->internal.points_wanted;
@ -96,7 +96,7 @@ void grouping_free_des(RRDR *r) {
r->internal.grouping_data = NULL;
}
void grouping_add_des(RRDR *r, calculated_number value) {
void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
if(likely(g->count > 0)) {
@ -109,7 +109,7 @@ void grouping_add_des(RRDR *r, calculated_number value) {
}
// for the values, except the first
calculated_number last_level = g->level;
NETDATA_DOUBLE last_level = g->level;
g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
}
@ -123,10 +123,10 @@ void grouping_add_des(RRDR *r, calculated_number value) {
//fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
}
calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
if(unlikely(!g->count || !calculated_number_isnumber(g->level))) {
if(unlikely(!g->count || !netdata_double_isnumber(g->level))) {
*rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
return 0.0;
}

4
web/api/queries/des/des.h
View file

@ -11,7 +11,7 @@ extern void grouping_init_des(void);
extern void grouping_create_des(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_des(RRDR *r);
extern void grouping_free_des(RRDR *r);
extern void grouping_add_des(RRDR *r, calculated_number value);
extern calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_DES_H

10
web/api/queries/incremental_sum/incremental_sum.c
View file

@ -6,8 +6,8 @@
// incremental sum
struct grouping_incremental_sum {
calculated_number first;
calculated_number last;
NETDATA_DOUBLE first;
NETDATA_DOUBLE last;
size_t count;
};
@ -29,7 +29,7 @@ void grouping_free_incremental_sum(RRDR *r) {
r->internal.grouping_data = NULL;
}
void grouping_add_incremental_sum(RRDR *r, calculated_number value) {
void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
if(unlikely(!g->count)) {
@ -42,10 +42,10 @@ void grouping_add_incremental_sum(RRDR *r, calculated_number value) {
}
}
calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data;
calculated_number value;
NETDATA_DOUBLE value;
if(unlikely(!g->count)) {
value = 0.0;

4
web/api/queries/incremental_sum/incremental_sum.h
View file

@ -9,7 +9,7 @@
extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
extern void grouping_reset_incremental_sum(RRDR *r);
extern void grouping_free_incremental_sum(RRDR *r);
extern void grouping_add_incremental_sum(RRDR *r, calculated_number value);
extern calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H

10
web/api/queries/max/max.c
View file

@ -6,7 +6,7 @@
// max
struct grouping_max {
calculated_number max;
NETDATA_DOUBLE max;
size_t count;
};
@ -27,19 +27,19 @@ void grouping_free_max(RRDR *r) {
r->internal.grouping_data = NULL;
}
void grouping_add_max(RRDR *r, calculated_number value) {
void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) {
struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) {
if(!g->count || fabsndd(value) > fabsndd(g->max)) {
g->max = value;
g->count++;
}
}
calculated_number grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data;
calculated_number value;
NETDATA_DOUBLE value;
if(unlikely(!g->count)) {
value = 0.0;

Some files were not shown because too many files have changed in this diff Show more