archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-28 22:52:30 +00:00
Code Issues Projects Releases Wiki Activity

Implemented multihost database ()

Browse source 
* Hard code a node for non-legacy multidb test
Skip dbengine initialization for new incoming children
Add code to switch to multidb ctx when accessing the dbengine

* When a non-legacy streaming connection is detected, use the multidb metadata log context

* Clear the superblock memory to avoid random data written in the metadata log

* Activate the host detection during compaction
Activate the host detection during metadata log chart updates
Keep the host in the user object during replay of the HOST command

* Add defaults for health / rrdpush on HOST metadata replay
Check for legacy status on host creation by checking is_archived and if not conclusive, call is_legacy_child()

Use defaults from the stream.conf

* Count hosts only if not archived
When host switches from archived to active update rrd_hosts_available
Remove archived hosts from charts and info

* Change parameter from "multidb disk space" to "dbengine multihost disk space"
Remove unused variables
Fix compilation error when dbengine is disabled
Fix condition for machine_guid directory creation under cache_dir

* Enable multidb disk space file creation.

* Stop deleting dimensions when rotating archived metrics if the dimension is active in a different database engine.

* Fix old bug in the code that confused obsolete hosts with orphan hosts.

* Do not delete multi-host DB host files.

* Discard dbengine state when a legacy memory mode instantiates to avoid inconsistencies.

* Identify metadata that collide with non-dbengine memory mode hosts and ignore them.

* Handle non-dbengine localhost with dbengine archived charts in localhost and streaming.

* Ignore archived hosts in streaming.

* Add documentation before merging to master.

Co-authored-by: Markos Fountoulakis <markos.fountoulakis.senior@gmail.com>
This commit is contained in:
Stelios Fragkakis 2020-07-28 15:04:39 +03:00 • committed by GitHub
parent 3d4314bc4d
commit eda12f579f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 425 additions and 215 deletions

8
collectors/plugins.d/pluginsd_parser.c
View file

@ -274,6 +274,10 @@ PARSER_RC pluginsd_end(char **words, void *user, PLUGINSD_ACTION *plugins_actio
PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION *plugins_action)
{
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
if (unlikely(!host)) {
debug(D_PLUGINSD, "Ignoring chart belonging to missing or ignored host.");
return PARSER_RC_OK;
}
char *type = words[1];
char *name = words[2];
@ -371,6 +375,10 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION *plugins
RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
if (unlikely(!host)) {
debug(D_PLUGINSD, "Ignoring dimension belonging to missing or ignored host.");
return PARSER_RC_OK;
}
if (unlikely(!id)) {
error(

15
daemon/main.c
View file

@ -41,7 +41,13 @@ void netdata_cleanup_and_exit(int ret) {
// free the database
info("EXIT: freeing database memory...");
#ifdef ENABLE_DBENGINE
rrdeng_prepare_exit(&multidb_ctx);
#endif
rrdhost_free_all();
#ifdef ENABLE_DBENGINE
rrdeng_exit(&multidb_ctx);
#endif
}
// unlink the pid
@ -568,10 +574,10 @@ static void get_netdata_configured_variables() {
default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
}
default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "multidb disk space", compute_multidb_diskspace());
default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace());
if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb);
default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
}
#endif
@ -1456,7 +1462,8 @@ int main(int argc, char **argv) {
// Load host labels
reload_host_labels();
#ifdef ENABLE_DBENGINE
metalog_commit_update_host(localhost);
if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
metalog_commit_update_host(localhost);
#endif
// ------------------------------------------------------------------------

63
database/engine/README.md
View file

@ -32,35 +32,64 @@ section of your `netdata.conf`. The Agent ignores the `history` setting when usi
```conf
[global]
page cache size = 32
dbengine disk space = 256
dbengine multihost disk space = 256
```
The above values are the default and minimum values for Page Cache size and DB engine disk space quota. Both numbers are
The above values are the default values for Page Cache size and DB engine disk space quota. Both numbers are
in **MiB**.
The `page cache size` option determines the amount of RAM in **MiB** dedicated to caching Netdata metric values. The
actual page cache size will be slightly larger than this figure—see the [memory requirements](#memory-requirements)
section for details.
The `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing Netdata
metric values and all related metadata describing them.
The `dbengine multihost disk space` option determines the amount of disk space in **MiB** that is dedicated to storing
Netdata metric values and all related metadata describing them.
### Legacy configuration
The deprecated `dbengine disk space` option determines the amount of disk space in **MiB** that is dedicated to storing
Netdata metric values per legacy database engine instance (see [below](#Streaming-metrics-to-the-database-engine)).
```conf
[global]
dbengine disk space = 256
```
Use the [**database engine calculator**](https://learn.netdata.cloud/docs/agent/database/calculator) to correctly set
`dbengine disk space` based on your needs. The calculator gives an accurate estimate based on how many child nodes
you have, how many metrics your Agent collects, and more.
`dbengine disk space`(**deprecated**) based on your needs. The calculator gives an accurate estimate based on how many
child nodes you have, how many metrics your Agent collects, and more.
### Streaming metrics to the database engine
When streaming metrics, the Agent on the parent node creates one instance of the database engine for itself, and another
instance for every child node it receives metrics from. If you have four streaming nodes, you will have five instances
in total (`1 parent + 4 child nodes = 5 instances`).
##### Legacy mode
The Agent allocates resources for each instance separately using the `dbengine disk space` setting. If `dbengine disk
space` is set to the default `256`, each instance is given 256 MiB in disk space, which means the total disk space
required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.
When streaming metrics, the Agent on the parent node used to create one instance (legacy, version <= 1.23.2) of the
database engine for itself, and another instance for every child node it receives metrics from. If you had four
streaming nodes, you would have five instances in total (`1 parent + 4 child nodes = 5 instances`).
The Agent allocated resources for each instance separately using the `dbengine disk space`(**deprecated**) setting. If
`dbengine disk space`(**deprecated**) is set to the default `256`, each instance is given 256 MiB in disk space, which
means the total disk space required to store all instances is, roughly, `256 MiB * 1 parent * 4 child nodes = 1280 MiB`.
See the [database engine calculator](https://learn.netdata.cloud/docs/agent/database/calculator) to help you correctly
set `dbengine disk space` and undertand the toal disk space required based on your streaming setup.
set `dbengine disk space`(**deprecated**) and understand the total disk space required based on your streaming setup.
##### Multi host DB mode
In the newer agent versions the parent and child nodes all share `page cache size` and `dbengine multihost disk space`
in a single dbengine multi-host instance.
##### Backward compatibility
All existing metrics belonging to child nodes are automatically converted to legacy dbengine instances and the localhost
metrics are transferred to the multi-host dbengine instance.
All new child nodes are automatically transferred to the mult-host dbengine instance and share its page cache and disk
space. If you want to migrate a child node from its legacy dbengine instance to the multi-host dbengine instance you
must delete the instance's directory located in `/var/cache/netdata/MACHINE_GUID/dbengine` after stopping the netdata
agent.
##### Information
For more information about setting `memory mode` on your nodes, in addition to other streaming configurations, see
[streaming](/streaming/README.md).
@ -70,8 +99,7 @@ For more information about setting `memory mode` on your nodes, in addition to o
Using memory mode `dbengine` we can overcome most memory restrictions and store a dataset that is much larger than the
available memory.
There are explicit memory requirements **per** DB engine **instance**, meaning **per** Netdata **node** (e.g. localhost
and streaming recipient nodes):
There are explicit memory requirements **per** DB engine **instance**:
- The total page cache memory footprint will be an additional `#dimensions-being-collected x 4096 x 2` bytes over what
the user configured with `page cache size`.
@ -83,10 +111,11 @@ and streaming recipient nodes):
- for very highly compressible data (compression ratio > 90%) this RAM overhead is comparable to the disk space
footprint.
An important observation is that RAM usage depends on both the `page cache size` and the `dbengine disk space` options.
An important observation is that RAM usage depends on both the `page cache size` and the `dbengine multihost disk space`
options.
You can use our [database engine calculator](https://learn.netdata.cloud/docs/agent/database/calculator) to
validate the memory requirements for your particular system(s) and configuration.
validate the memory requirements for your particular system(s) and configuration (**out-of-date**).
### File descriptor requirements

7
database/engine/metadata_log/compaction.c
View file

@ -82,7 +82,7 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
RRDSET *st;
RRDDIM *rd;
BUFFER *buffer;
RRDHOST *host = ctx->rrdeng_ctx->host;
RRDHOST *host = NULL;
ret = find_object_by_guid(uuid, NULL, 0);
switch (ret) {
@ -130,8 +130,9 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
}
break;
case GUID_TYPE_HOST:
//TODO: will be enabled when multidb is activated
//RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid);
host = metalog_get_host_from_uuid(ctx, uuid);
if (unlikely(!host))
break;
if (ctx->current_compaction_id > host->compaction_id) {
host->compaction_id = ctx->current_compaction_id;
buffer = metalog_update_host_buffer(host);

1
database/engine/metadata_log/logfile.c
View file

@ -330,6 +330,7 @@ int create_metadata_logfile(struct metadata_logfile *metalogfile)
if (unlikely(ret)) {
fatal("posix_memalign:%s", strerror(ret));
}
memset(superblock, 0, sizeof(*superblock));
(void) strncpy(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ);
superblock->version = RRDENG_METALOG_VER;

4
database/engine/metadata_log/metadatalog.c
View file

@ -157,9 +157,9 @@ void metalog_test_quota(struct metalog_worker_config *wc)
metalogfile = ctx->metadata_logfiles.last;
only_one_metalogfile = (metalogfile == ctx->metadata_logfiles.first) ? 1 : 0;
debug(D_METADATALOG, "records=%lu objects=%lu", (long unsigned)ctx->records_nr,
(long unsigned)ctx->rrdeng_ctx->host->objects_nr);
(long unsigned)ctx->objects_nr);
if (unlikely(!only_one_metalogfile &&
ctx->records_nr > (ctx->rrdeng_ctx->host->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&
ctx->records_nr > (ctx->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&
NO_QUIESCE == ctx->quiesce) {
metalog_do_compaction(wc);
}

2
database/engine/metadata_log/metadatalog.h
View file

@ -112,6 +112,8 @@ struct metalog_instance {
uint32_t current_compaction_id; /* Every compaction run increments this by 1 */
unsigned long disk_space;
unsigned long records_nr;
unsigned long objects_nr; /* total objects (hosts, charts, dimensions) monitored in this context */
uint8_t initialized; /* set to 1 to mark context initialized */
unsigned last_fileno; /* newest index of metadata log file */
uint8_t quiesce; /*

99
database/engine/metadata_log/metadatalogapi.c
View file

@ -3,6 +3,14 @@
#include "metadatalog.h"
static inline struct metalog_instance *get_metalog_ctx(RRDHOST *host)
{
if (host->rrdeng_ctx)
return host->rrdeng_ctx->metalog_ctx;
return NULL;
}
static inline int metalog_is_initialized(struct metalog_instance *ctx)
{
return ctx->rrdeng_ctx->metalog_ctx != NULL;
@ -18,6 +26,16 @@ static inline void metalog_commit_deletion_record(struct metalog_instance *ctx,
metalog_commit_record(ctx, buffer, METALOG_COMMIT_DELETION_RECORD, NULL, 0);
}
void metalog_upd_objcount(RRDHOST *host, int count)
{
struct metalog_instance *ctx = get_metalog_ctx(host);
if (unlikely(!ctx))
return;
rrd_atomic_fetch_add(&ctx->objects_nr, count);
}
BUFFER *metalog_update_host_buffer(RRDHOST *host)
{
BUFFER *buffer;
@ -62,11 +80,10 @@ void metalog_commit_update_host(RRDHOST *host)
BUFFER *buffer;
/* Metadata are only available with dbengine */
if (!host->rrdeng_ctx)
ctx = get_metalog_ctx(host);
if (!ctx)
return;
ctx = host->rrdeng_ctx->metalog_ctx;
if (!ctx) /* metadata log has not been initialized yet */
if (!ctx->initialized) /* metadata log has not been initialized yet */
return;
buffer = metalog_update_host_buffer(host);
@ -157,14 +174,15 @@ void metalog_commit_update_chart(RRDSET *st)
{
struct metalog_instance *ctx;
BUFFER *buffer;
RRDHOST *host = st->rrdhost;
/* Metadata are only available with dbengine */
if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
return;
ctx = host->rrdeng_ctx->metalog_ctx;
if (!ctx) /* metadata log has not been initialized yet */
ctx = get_metalog_ctx(st->rrdhost);
if (!ctx)
return;
if (!ctx->initialized) /* metadata log has not been initialized yet */
return;
buffer = metalog_update_chart_buffer(st, 0);
@ -176,15 +194,16 @@ void metalog_commit_delete_chart(RRDSET *st)
{
struct metalog_instance *ctx;
BUFFER *buffer;
RRDHOST *host = st->rrdhost;
char uuid_str[37];
/* Metadata are only available with dbengine */
if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
return;
ctx = host->rrdeng_ctx->metalog_ctx;
if (!ctx) /* metadata log has not been initialized yet */
ctx = get_metalog_ctx(st->rrdhost);
if (!ctx)
return;
if (!ctx->initialized) /* metadata log has not been initialized yet */
return;
buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
@ -228,14 +247,15 @@ void metalog_commit_update_dimension(RRDDIM *rd)
struct metalog_instance *ctx;
BUFFER *buffer;
RRDSET *st = rd->rrdset;
RRDHOST *host = st->rrdhost;
/* Metadata are only available with dbengine */
if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
return;
ctx = host->rrdeng_ctx->metalog_ctx;
if (!ctx) /* metadata log has not been initialized yet */
ctx = get_metalog_ctx(st->rrdhost);
if (!ctx)
return;
if (!ctx->initialized) /* metadata log has not been initialized yet */
return;
buffer = metalog_update_dimension_buffer(rd);
@ -248,15 +268,16 @@ void metalog_commit_delete_dimension(RRDDIM *rd)
struct metalog_instance *ctx;
BUFFER *buffer;
RRDSET *st = rd->rrdset;
RRDHOST *host = st->rrdhost;
char uuid_str[37];
/* Metadata are only available with dbengine */
if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
if (RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
return;
ctx = host->rrdeng_ctx->metalog_ctx;
if (!ctx) /* metadata log has not been initialized yet */
ctx = get_metalog_ctx(st->rrdhost);
if (!ctx)
return;
if (!ctx->initialized) /* metadata log has not been initialized yet */
return;
buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
@ -273,12 +294,15 @@ RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_g
char machine_guid[37];
uuid_unparse_lower(*host_guid, machine_guid);
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
ret = find_object_by_guid(host_guid, NULL, 0);
if (unlikely(GUID_TYPE_HOST != ret)) {
error("Host with GUID %s not found in the global map", machine_guid);
return NULL;
errno = 0;
if (unlikely(!host))
error("Host with GUID %s not found in the global map or in the list of hosts", machine_guid);
else
error("Host with GUID %s not found in the global map", machine_guid);
}
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
return host;
}
@ -292,9 +316,12 @@ RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_
if (unlikely(GUID_TYPE_CHART != ret))
return NULL;
machine_guid = (uuid_t *)chart_object;
RRDHOST *host = ctx->rrdeng_ctx->host;
machine_guid = (uuid_t *)chart_object;
RRDHOST *host = metalog_get_host_from_uuid(ctx, machine_guid);
if (unlikely(!host))
return NULL;
if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {
errno = 0;
error("Metadata host machine GUID does not match the one assosiated with the chart");
return NULL;
}
@ -311,6 +338,8 @@ RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_
RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
{
UNUSED(ctx);
GUID_TYPE ret;
char dim_object[49], chart_object[33], id_str[PLUGINSD_LINE_MAX], chart_fullid[RRD_ID_LENGTH_MAX + 1];
uuid_t *machine_guid, *chart_guid, *chart_char_guid, *dim_char_guid;
@ -320,8 +349,12 @@ RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *me
return NULL;
machine_guid = (uuid_t *)dim_object;
RRDHOST *host = ctx->rrdeng_ctx->host;
RRDHOST *host = metalog_get_host_from_uuid(ctx, machine_guid);
if (unlikely(!host))
return NULL;
if (unlikely(uuid_compare(host->host_uuid, *machine_guid))) {
errno = 0;
error("Metadata host machine GUID does not match the one assosiated with the dimension");
return NULL;
}
@ -359,10 +392,11 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr
uint8_t empty_chart;
rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
if (!rd) { /* in 8the case of legacy UUID convert to multihost and try again */
// TODO: Check what to do since we have no host
uuid_t multihost_uuid;
rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->host->machine_guid, metric_uuid, &multihost_uuid);
rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->machine_guid, metric_uuid, &multihost_uuid);
rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);
}
if(!rd) {
@ -372,6 +406,10 @@ void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metr
st = rd->rrdset;
host = st->rrdhost;
/* In case there are active metrics in a different database engine do not delete the dimension object */
if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE))
return;
/* Since the metric has no writer it will not be commited to the metadata log by rrddim_free_custom().
* It must be commited explicitly before calling rrddim_free_custom(). */
metalog_commit_delete_dimension(rd);
@ -401,8 +439,11 @@ int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
ctx = callocz(1, sizeof(*ctx));
ctx->records_nr = 0;
ctx->objects_nr = 0;
ctx->current_compaction_id = 0;
ctx->quiesce = NO_QUIESCE;
ctx->initialized = 0;
rrdeng_parent_ctx->metalog_ctx = ctx;
memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
ctx->rrdeng_ctx = rrdeng_parent_ctx;
@ -422,7 +463,7 @@ int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
if (ctx->worker_config.error) {
goto error_after_rrdeng_worker;
}
rrdeng_parent_ctx->metalog_ctx = ctx; /* notify dbengine that the metadata log has finished initializing */
ctx->initialized = 1; /* notify dbengine that the metadata log has finished initializing */
return 0;
error_after_rrdeng_worker:

1
database/engine/metadata_log/metadatalogapi.h
View file

@ -13,6 +13,7 @@ extern void metalog_commit_delete_chart(RRDSET *st);
extern BUFFER *metalog_update_dimension_buffer(RRDDIM *rd);
extern void metalog_commit_update_dimension(RRDDIM *rd);
extern void metalog_commit_delete_dimension(RRDDIM *rd);
extern void metalog_upd_objcount(RRDHOST *host, int count);
extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);

86
database/engine/metadata_log/metalogpluginsd.c
View file

@ -4,15 +4,33 @@
#include "metadatalog.h"
#include "metalogpluginsd.h"
extern struct config stream_config;
PARSER_RC metalog_pluginsd_host_action(
void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,
char *tags)
{
int history = 5;
RRD_MEMORY_MODE mode = RRD_MEMORY_MODE_DBENGINE;
int health_enabled = default_health_enabled;
int rrdpush_enabled = default_rrdpush_enabled;
char *rrdpush_destination = default_rrdpush_destination;
char *rrdpush_api_key = default_rrdpush_api_key;
char *rrdpush_send_charts_matching = default_rrdpush_send_charts_matching;
struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
if (host)
if (host) {
if (unlikely(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
error("Archived host '%s' has memory mode '%s', but the archived one is '%s'. Ignoring archived state.",
host->hostname, rrd_memory_mode_name(host->rrd_memory_mode),
rrd_memory_mode_name(RRD_MEMORY_MODE_DBENGINE));
((PARSER_USER_OBJECT *) user)->host = NULL; /* Ignore objects if memory mode is not dbengine */
return PARSER_RC_OK;
}
goto write_replay;
}
if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {
struct metalog_record record;
@ -20,12 +38,32 @@ PARSER_RC metalog_pluginsd_host_action(
uuid_parse(machine_guid, record.uuid);
mlf_record_insert(metalogfile, &record);
if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
((PARSER_USER_OBJECT *) user)->host = localhost;
else
((PARSER_USER_OBJECT *) user)->host = NULL;
return PARSER_RC_OK;
}
// Ignore HOST command for now
// TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb
return PARSER_RC_OK;
// Fetch configuration options from streaming config
update_every = (int)appconfig_get_number(&stream_config, machine_guid, "update every", update_every);
if(update_every < 0) update_every = 1;
//health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->key, "health enabled by default", health_enabled);
health_enabled = appconfig_get_boolean_ondemand(&stream_config, machine_guid, "health enabled", health_enabled);
//rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->key, "default proxy enabled", rrdpush_enabled);
rrdpush_enabled = appconfig_get_boolean(&stream_config, machine_guid, "proxy enabled", rrdpush_enabled);
//rrdpush_destination = appconfig_get(&stream_config, rpt->key, "default proxy destination", rrdpush_destination);
rrdpush_destination = appconfig_get(&stream_config, machine_guid, "proxy destination", rrdpush_destination);
//rrdpush_api_key = appconfig_get(&stream_config, rpt->key, "default proxy api key", rrdpush_api_key);
rrdpush_api_key = appconfig_get(&stream_config, machine_guid, "proxy api key", rrdpush_api_key);
//rrdpush_send_charts_matching = appconfig_get(&stream_config, rpt->key, "default proxy send charts matching", rrdpush_send_charts_matching);
rrdpush_send_charts_matching = appconfig_get(&stream_config, machine_guid, "proxy send charts matching", rrdpush_send_charts_matching);
host = rrdhost_create(
hostname
@ -37,13 +75,13 @@ PARSER_RC metalog_pluginsd_host_action(
, NULL
, NULL
, update_every
, 3600
, RRD_MEMORY_MODE_DBENGINE
, 0 // health enabled
, 0 // Push enabled
, NULL
, NULL
, NULL
, history // entries
, mode
, health_enabled // health enabled
, rrdpush_enabled // Push enabled
, rrdpush_destination //destination
, rrdpush_api_key // api key
, rrdpush_send_charts_matching // charts matching
, callocz(1, sizeof(struct rrdhost_system_info))
, 0 // localhost
, 1 // archived
@ -57,7 +95,7 @@ write_replay:
uuid_copy(record.uuid, host->host_uuid);
mlf_record_insert(metalogfile, &record);
}
((PARSER_USER_OBJECT *) user)->host = host;
return PARSER_RC_OK;
}
@ -70,6 +108,10 @@ PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
uuid_t *chart_uuid;
if (unlikely(!host)) {
debug(D_METADATALOG, "Ignoring chart belonging to missing or ignored host.");
return PARSER_RC_OK;
}
chart_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
st = rrdset_create_custom(
host, type, id, name, family, context, title, units,
@ -117,6 +159,10 @@ PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, ch
UNUSED(algorithm);
uuid_t *dim_uuid;
if (unlikely(!st)) {
debug(D_METADATALOG, "Ignoring dimension belonging to missing or ignored chart.");
return PARSER_RC_OK;
}
dim_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
RRDDIM *rd = rrddim_add_custom(st, id, name, multiplier, divisor, algorithm_type, RRD_MEMORY_MODE_DBENGINE, 1,
@ -155,8 +201,8 @@ PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)
PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
{
GUID_TYPE ret;
struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
struct metalog_instance *ctx = state->ctx;
//struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
//struct metalog_instance *ctx = state->ctx;
char object[49], chart_object[33], id_str[1024];
uuid_t *chart_guid, *chart_char_guid;
RRDHOST *host;
@ -164,15 +210,16 @@ PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
ret = find_object_by_guid(uuid, object, 49);
switch (ret) {
case GUID_TYPE_NOTFOUND:
if (unlikely(ctx->rrdeng_ctx->host && uuid_compare(ctx->rrdeng_ctx->host->host_uuid, *uuid)))
error_with_guid(uuid, "Failed to find valid context");
error_with_guid(uuid, "Failed to find valid context");
break;
case GUID_TYPE_CHAR:
error_with_guid(uuid, "Ignoring unexpected type GUID_TYPE_CHAR");
break;
case GUID_TYPE_CHART:
case GUID_TYPE_DIMENSION:
host = ctx->rrdeng_ctx->host;
host = metalog_get_host_from_uuid(NULL, (uuid_t *) &object);
if (unlikely(!host))
break;
switch (ret) {
case GUID_TYPE_CHART:
chart_char_guid = (uuid_t *)(object + 16);
@ -204,7 +251,7 @@ PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
}
break;
case GUID_TYPE_HOST:
/* Ignore for now */
((PARSER_USER_OBJECT *)user)->host = metalog_get_host_from_uuid(NULL, (uuid_t *) &object);
break;
case GUID_TYPE_NOSPACE:
error_with_guid(uuid, "Not enough space for object retrieval");
@ -222,7 +269,7 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
GUID_TYPE ret;
struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
struct metalog_instance *ctx = state->ctx;
RRDHOST *host = ctx->rrdeng_ctx->host;
RRDHOST *host = NULL;
RRDSET *st;
RRDDIM *rd;
@ -234,6 +281,7 @@ PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
case GUID_TYPE_CHART:
st = metalog_get_chart_from_uuid(ctx, uuid);
if (st) {
host = st->rrdhost;
rrdhost_wrlock(host);
rrdset_free(st);
rrdhost_unlock(host);

4
database/engine/rrdengine.c
View file

@ -644,7 +644,7 @@ static void delete_old_data(void *arg)
for (i = 0 ; i < count ; ++i) {
descr = extent->pages[i];
can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);
if (unlikely(can_delete_metric && ctx->metalog_ctx)) {
if (unlikely(can_delete_metric && ctx->metalog_ctx->initialized)) {
/*
* If the metric is empty, has no active writers and if the metadata log has been initialized then
* attempt to delete the corresponding netdata dimension.
@ -821,7 +821,7 @@ void timer_cb(uv_timer_t* handle)
uv_stop(handle->loop);
uv_update_time(handle->loop);
if (unlikely(!ctx->metalog_ctx))
if (unlikely(!ctx->metalog_ctx->initialized))
return; /* Wait for the metadata log to initialize */
rrdeng_test_quota(wc);
debug(D_RRDENGINE, "%s: timeout reached.", __func__);

5
database/engine/rrdengine.h
View file

@ -176,7 +176,6 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele
#define QUIESCED (2) /* is set after all threads have finished running */
struct rrdengine_instance {
RRDHOST *host;
struct metalog_instance *metalog_ctx;
struct rrdengine_worker_config worker_config;
struct completion rrdengine_completion;
@ -185,7 +184,9 @@ struct rrdengine_instance {
uint8_t global_compress_alg;
struct transaction_commit_log commit_log;
struct rrdengine_datafile_list datafiles;
char dbfiles_path[FILENAME_MAX+1];
RRDHOST *host; /* the legacy host, or NULL for multi-host DB */
char dbfiles_path[FILENAME_MAX + 1];
char machine_guid[GUID_LEN + 1]; /* the unique ID of the corresponding host, or localhost for multihost DB */
uint64_t disk_space;
uint64_t max_disk_space;
unsigned last_fileno; /* newest index of datafile and journalfile */

50
database/engine/rrdengineapi.c
View file

@ -2,7 +2,7 @@
#include "rrdengine.h"
/* Default global database instance */
static struct rrdengine_instance default_global_ctx;
struct rrdengine_instance multidb_ctx;
int default_rrdeng_page_cache_mb = 32;
int default_rrdeng_disk_quota_mb = 256;
@ -10,6 +10,11 @@ int default_multidb_disk_quota_mb = 256;
/* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host)
{
return host->rrdeng_ctx;
}
/* This UUID is not unique across hosts */
void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
{
@ -53,7 +58,11 @@ void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)
struct pg_cache_page_index *page_index = NULL;
int replace_instead_of_generate = 0;
ctx = rd->rrdset->rrdhost->rrdeng_ctx;
ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
if (unlikely(!ctx)) {
error("Failed to fetch multidb context");
return;
}
pg_cache = &ctx->pg_cache;
rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
@ -132,7 +141,7 @@ void rrdeng_store_metric_init(RRDDIM *rd)
struct rrdengine_instance *ctx;
struct pg_cache_page_index *page_index;
ctx = rd->rrdset->rrdhost->rrdeng_ctx;
ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
handle = &rd->state->handle.rrdeng;
handle->ctx = ctx;
@ -171,6 +180,8 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
handle = &rd->state->handle.rrdeng;
ctx = handle->ctx;
if (unlikely(!ctx))
return;
descr = handle->descr;
if (unlikely(NULL == descr)) {
return;
@ -370,7 +381,7 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e
struct rrdeng_region_info *region_info_array;
uint8_t is_first_region_initialized;
ctx = st->rrdhost->rrdeng_ctx;
ctx = get_rrdeng_ctx_from_host(st->rrdhost);
regions = 1;
*max_intervalp = max_interval = 0;
region_info_array = NULL;
@ -532,7 +543,7 @@ void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_hand
struct rrdengine_instance *ctx;
unsigned pages_nr;
ctx = rd->rrdset->rrdhost->rrdeng_ctx;
ctx = get_rrdeng_ctx_from_host(rd->rrdset->rrdhost);
rrdimm_handle->start_time = start_time;
rrdimm_handle->end_time = end_time;
handle = &rrdimm_handle->rrdeng;
@ -801,6 +812,9 @@ void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_i
*/
void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array)
{
if (ctx == NULL)
return;
struct page_cache *pg_cache = &ctx->pg_cache;
array[0] = (uint64_t)ctx->stats.metric_API_producers;
@ -875,8 +889,7 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
}
if (NULL == ctxp) {
/* for testing */
ctx = &default_global_ctx;
ctx = &multidb_ctx;
memset(ctx, 0, sizeof(*ctx));
} else {
*ctxp = ctx = callocz(1, sizeof(*ctx));
@ -892,6 +905,11 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
ctx->max_disk_space = disk_space_mb * 1048576LLU;
strncpyz(ctx->dbfiles_path, dbfiles_path, sizeof(ctx->dbfiles_path) - 1);
ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';
if (NULL == host)
strncpyz(ctx->machine_guid, registry_get_this_machine_guid(), GUID_LEN);
else
strncpyz(ctx->machine_guid, host->machine_guid, GUID_LEN);
ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
ctx->metric_API_max_producers = 0;
ctx->quiesce = NO_QUIESCE;
@ -916,23 +934,19 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
if (ctx->worker_config.error) {
goto error_after_rrdeng_worker;
}
if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) {
info("Metadatalog init for host %s starting...", host->hostname);
error = metalog_init(ctx);
if (error) {
error("Failed to initialize metadata log file event loop.");
goto error_after_rrdeng_worker;
}
error = metalog_init(ctx);
if (error) {
error("Failed to initialize metadata log file event loop.");
goto error_after_rrdeng_worker;
}
else
info("No metadatalog init for host %s", host->hostname);
return 0;
error_after_rrdeng_worker:
finalize_rrd_files(ctx);
error_after_init_rrd_files:
free_page_cache(ctx);
if (ctx != &default_global_ctx) {
if (ctx != &multidb_ctx) {
freez(ctx);
*ctxp = NULL;
}
@ -961,7 +975,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
metalog_exit(ctx->metalog_ctx);
free_page_cache(ctx);
if (ctx != &default_global_ctx) {
if (ctx != &multidb_ctx) {
freez(ctx);
}
rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);

1
database/engine/rrdengineapi.h
View file

@ -16,6 +16,7 @@ extern int default_rrdeng_page_cache_mb;
extern int default_rrdeng_disk_quota_mb;
extern int default_multidb_disk_quota_mb;
extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
extern struct rrdengine_instance multidb_ctx;
struct rrdeng_region_info {
time_t start_time;

17
database/engine/rrdenginelib.c
View file

@ -272,16 +272,15 @@ int compute_multidb_diskspace()
int rc = count_legacy_children(netdata_configured_cache_dir);
if (likely(rc >= 0)) {
computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
//info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
// TODO: will activate the next block of code when multidb is in place
// fp = fopen(multidb_disk_space_file, "w");
// if (likely(fp)) {
// fprintf(fp, "%d", rc * default_rrdeng_disk_quota_mb);
// info("Created file '%s' to store the computed value", multidb_disk_space_file);
// fclose(fp);
// } else
// error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
fp = fopen(multidb_disk_space_file, "w");
if (likely(fp)) {
fprintf(fp, "%d", computed_multidb_disk_quota_mb);
info("Created file '%s' to store the computed value", multidb_disk_space_file);
fclose(fp);
} else
error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
}
else
computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;

5
database/rrd.h
View file

@ -698,10 +698,6 @@ struct rrdhost {
int rrd_update_every; // the update frequency of the host
long rrd_history_entries; // the number of history entries for the host's charts
#ifdef ENABLE_DBENGINE
unsigned page_cache_mb; // Database Engine page cache size in MiB
unsigned disk_space_mb; // Database Engine disk space quota in MiB
#endif
RRD_MEMORY_MODE rrd_memory_mode; // the memory more for the charts of this host
char *cache_dir; // the directory to save RRD cache files
@ -812,7 +808,6 @@ struct rrdhost {
#ifdef ENABLE_DBENGINE
struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
uuid_t host_uuid; // Global GUID for this host
unsigned long objects_nr; // Number of charts and dimensions in this host
uint32_t compaction_id; // The last metadata log compaction procedure that has processed
// this object.
#endif

4
database/rrddim.c
View file

@ -451,7 +451,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
aclk_update_chart(host, st->id, ACLK_CMD_CHART);
#endif
#ifdef ENABLE_DBENGINE
rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
metalog_upd_objcount(st->rrdhost, 1);
metalog_commit_update_dimension(rd);
#endif
@ -525,7 +525,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
#endif
#ifdef ENABLE_DBENGINE
rrd_atomic_fetch_add(&st->rrdhost->objects_nr, -1);
metalog_upd_objcount(st->rrdhost, -1);
#endif
}

232
database/rrdhost.c
View file

@ -128,18 +128,20 @@ RRDHOST *rrdhost_create(const char *hostname,
) {
debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
#ifdef ENABLE_DBENGINE
int is_legacy = is_archived ? 0 : (memory_mode == RRD_MEMORY_MODE_DBENGINE) && is_legacy_child(guid);
#else
int is_legacy = 1;
#endif
rrd_check_wrlock();
int is_in_multihost = (memory_mode == RRD_MEMORY_MODE_DBENGINE && !is_legacy);
RRDHOST *host = callocz(1, sizeof(RRDHOST));
host->rrd_update_every = (update_every > 0)?update_every:1;
host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
host->rrd_memory_mode = memory_mode;
#ifdef ENABLE_DBENGINE
host->page_cache_mb = default_rrdeng_page_cache_mb;
host->disk_space_mb = default_rrdeng_disk_quota_mb;
#endif
host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled;
host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE) || is_archived) ? 0 : health_enabled;
host->sender = mallocz(sizeof(*host->sender));
sender_init(host->sender, host);
@ -227,12 +229,15 @@ RRDHOST *rrdhost_create(const char *hostname,
}
else {
// this is not localhost - append our GUID to localhost path
if (is_in_multihost) { // don't append to cache dir in multihost
host->cache_dir = strdupz(netdata_configured_cache_dir);
} else {
snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid);
host->cache_dir = strdupz(filename);
}
snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_cache_dir, host->machine_guid);
host->cache_dir = strdupz(filename);
if(host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE ||
host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
if((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP || host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || (
host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_legacy))) {
int r = mkdir(host->cache_dir, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->cache_dir);
@ -241,7 +246,7 @@ RRDHOST *rrdhost_create(const char *hostname,
snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid);
host->varlib_dir = strdupz(filename);
if(host->health_enabled) {
if(!is_archived && host->health_enabled) {
int r = mkdir(host->varlib_dir, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);
@ -249,7 +254,7 @@ RRDHOST *rrdhost_create(const char *hostname,
}
if(host->health_enabled) {
if(!is_archived && host->health_enabled) {
snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
int r = mkdir(filename, 0775);
if(r != 0 && errno != EEXIST)
@ -267,7 +272,7 @@ RRDHOST *rrdhost_create(const char *hostname,
// ------------------------------------------------------------------------
// load health configuration
if(host->health_enabled) {
if(!is_archived && host->health_enabled) {
rrdhost_wrlock(host);
health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
rrdhost_unlock(host);
@ -276,6 +281,14 @@ RRDHOST *rrdhost_create(const char *hostname,
health_alarm_log_open(host);
}
RRDHOST *t = rrdhost_index_add(host);
if(t != host) {
error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid);
rrdhost_free(host);
return NULL;
}
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
if (unlikely(-1 == uuid_parse(host->machine_guid, host->host_uuid))) {
@ -285,26 +298,32 @@ RRDHOST *rrdhost_create(const char *hostname,
error("Failed to store machine GUID to global map");
else
info("Added %s to global map for host %s", host->machine_guid, host->hostname);
host->objects_nr = 1;
host->compaction_id = 0;
char dbenginepath[FILENAME_MAX + 1];
int ret;
snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir);
ret = mkdir(dbenginepath, 0775);
if(ret != 0 && errno != EEXIST)
if (ret != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
else ret = 0; // succeed
if (is_legacy) // initialize legacy dbengine instance as needed
ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, default_rrdeng_page_cache_mb,
default_rrdeng_disk_quota_mb); // may fail here for legacy dbengine initialization
else
ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
if(ret) {
error("Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
host->hostname, host->machine_guid, host->cache_dir);
host->rrdeng_ctx = &multidb_ctx;
if (ret) { // check legacy or multihost initialization success
error(
"Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
host->hostname, host->machine_guid, host->cache_dir);
rrdhost_free(host);
host = NULL;
//rrd_hosts_available++; //TODO: maybe we want this?
return host;
}
metalog_upd_objcount(host, 1);
#else
fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");
#endif
@ -325,55 +344,47 @@ RRDHOST *rrdhost_create(const char *hostname,
else localhost = host;
}
RRDHOST *t = rrdhost_index_add(host);
info("Host '%s' (at registry as '%s') with guid '%s' initialized"
", os '%s'"
", timezone '%s'"
", tags '%s'"
", program_name '%s'"
", program_version '%s'"
", update every %d"
", memory mode %s"
", history entries %ld"
", streaming %s"
" (to '%s' with api key '%s')"
", health %s"
", cache_dir '%s'"
", varlib_dir '%s'"
", health_log '%s'"
", alarms default handler '%s'"
", alarms default recipient '%s'"
, host->hostname
, host->registry_hostname
, host->machine_guid
, host->os
, host->timezone
, (host->tags)?host->tags:""
, host->program_name
, host->program_version
, host->rrd_update_every
, rrd_memory_mode_name(host->rrd_memory_mode)
, host->rrd_history_entries
, host->rrdpush_send_enabled?"enabled":"disabled"
, host->rrdpush_send_destination?host->rrdpush_send_destination:""
, host->rrdpush_send_api_key?host->rrdpush_send_api_key:""
, host->health_enabled?"enabled":"disabled"
, host->cache_dir
, host->varlib_dir
, host->health_log_filename
, host->health_default_exec
, host->health_default_recipient
);
if(t != host) {
error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid);
rrdhost_free(host);
host = NULL;
}
else {
info("Host '%s' (at registry as '%s') with guid '%s' initialized"
", os '%s'"
", timezone '%s'"
", tags '%s'"
", program_name '%s'"
", program_version '%s'"
", update every %d"
", memory mode %s"
", history entries %ld"
", streaming %s"
" (to '%s' with api key '%s')"
", health %s"
", cache_dir '%s'"
", varlib_dir '%s'"
", health_log '%s'"
", alarms default handler '%s'"
", alarms default recipient '%s'"
, host->hostname
, host->registry_hostname
, host->machine_guid
, host->os
, host->timezone
, (host->tags)?host->tags:""
, host->program_name
, host->program_version
, host->rrd_update_every
, rrd_memory_mode_name(host->rrd_memory_mode)
, host->rrd_history_entries
, host->rrdpush_send_enabled?"enabled":"disabled"
, host->rrdpush_send_destination?host->rrdpush_send_destination:""
, host->rrdpush_send_api_key?host->rrdpush_send_api_key:""
, host->health_enabled?"enabled":"disabled"
, host->cache_dir
, host->varlib_dir
, host->health_log_filename
, host->health_default_exec
, host->health_default_recipient
);
}
rrd_hosts_available++;
if (!is_archived)
rrd_hosts_available++;
#ifdef ENABLE_DBENGINE
if (likely(!is_localhost && !is_archived && host && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
@ -408,7 +419,7 @@ void rrdhost_update(RRDHOST *host
UNUSED(rrdpush_api_key);
UNUSED(rrdpush_send_charts_matching);
host->health_enabled = health_enabled;
host->health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
//host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
rrdhost_system_info_free(host->system_info);
@ -453,6 +464,33 @@ void rrdhost_update(RRDHOST *host
// update host tags
rrdhost_init_tags(host, tags);
if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED);
if(host->health_enabled) {
int r;
char filename[FILENAME_MAX + 1];
if (host != localhost) {
r = mkdir(host->varlib_dir, 0775);
if (r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, host->varlib_dir);
}
snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
r = mkdir(filename, 0775);
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", host->hostname, filename);
rrdhost_wrlock(host);
health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
rrdhost_unlock(host);
health_alarm_log_load(host);
health_alarm_log_open(host);
}
rrd_hosts_available++;
info("Host %s is not in archived mode anymore", host->hostname);
}
#ifdef ENABLE_DBENGINE
if (likely(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
metalog_commit_update_host(host);
@ -483,6 +521,13 @@ RRDHOST *rrdhost_find_or_create(
rrd_wrlock();
RRDHOST *host = rrdhost_find_by_guid(guid, 0);
if (unlikely(host && RRD_MEMORY_MODE_DBENGINE != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
/* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
rrdhost_free(host);
host = NULL;
}
if(!host) {
host = rrdhost_create(
hostname
@ -555,7 +600,12 @@ restart_after_removal:
if(rrdhost_should_be_removed(host, protected, now)) {
info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid);
if(rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST))
if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST)
#ifdef ENABLE_DBENGINE
/* don't delete multi-host DB host files */
&& !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)
#endif
)
rrdhost_delete_charts(host);
else
rrdhost_save_charts(host);
@ -602,8 +652,33 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
, 1
, 0
);
if (unlikely(!localhost)) {
rrd_unlock();
return 1;
}
#ifdef ENABLE_DBENGINE
char dbenginepath[FILENAME_MAX + 1];
int ret;
snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir);
ret = mkdir(dbenginepath, 0775);
if (ret != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", localhost->hostname, dbenginepath);
else // Unconditionally create multihost db to support on demand host creation
ret = rrdeng_init(NULL, NULL, dbenginepath, default_rrdeng_page_cache_mb, default_multidb_disk_quota_mb);
if (ret) {
error(
"Host '%s' with machine guid '%s' failed to initialize multi-host DB engine instance at '%s'.",
localhost->hostname, localhost->machine_guid, localhost->cache_dir);
rrdhost_free(localhost);
localhost = NULL;
rrd_unlock();
return 1;
}
#endif
rrd_unlock();
web_client_api_v1_management_init();
web_client_api_v1_management_init();
return localhost==NULL;
}
@ -717,7 +792,8 @@ void rrdhost_free(RRDHOST *host) {
// release its children resources
#ifdef ENABLE_DBENGINE
rrdeng_prepare_exit(host->rrdeng_ctx);
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx)
rrdeng_prepare_exit(host->rrdeng_ctx);
#endif
while(host->rrdset_root)
rrdset_free(host->rrdset_root);
@ -749,11 +825,10 @@ void rrdhost_free(RRDHOST *host) {
health_alarm_log_free(host);
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx)
rrdeng_exit(host->rrdeng_ctx);
#endif
}
// ------------------------------------------------------------------------
// remove it from the indexes
@ -811,7 +886,9 @@ void rrdhost_free(RRDHOST *host) {
void rrdhost_free_all(void) {
rrd_wrlock();
while(localhost) rrdhost_free(localhost);
/* Make sure child-hosts are released before the localhost. */
while(localhost->next) rrdhost_free(localhost->next);
rrdhost_free(localhost);
rrd_unlock();
}
@ -1396,7 +1473,12 @@ void rrdhost_cleanup_all(void) {
RRDHOST *host;
rrdhost_foreach_read(host) {
if(host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS) && !host->receiver)
if (host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) && !host->receiver
#ifdef ENABLE_DBENGINE
/* don't delete multi-host DB host files */
&& !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx)
#endif
)
rrdhost_delete_charts(host);
else
rrdhost_cleanup_charts(host);

9
database/rrdset.c
View file

@ -392,7 +392,7 @@ void rrdset_free(RRDSET *st) {
break;
}
#ifdef ENABLE_DBENGINE
rrd_atomic_fetch_add(&host->objects_nr, -1);
metalog_upd_objcount(host, -1);
#endif
}
@ -543,10 +543,6 @@ RRDSET *rrdset_create_custom(
if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
changed_from_archived_to_active = 1;
if (rrdhost_flag_check(st->rrdhost, RRDHOST_FLAG_ARCHIVED)) {
rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
info("Host %s is not in archived mode anymore", st->rrdhost->hostname);
}
mark_rebuild |= META_CHART_ACTIVATED;
}
char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
@ -689,7 +685,6 @@ RRDSET *rrdset_create_custom(
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
rrdhost_flag_clear(st->rrdhost, RRDHOST_FLAG_ARCHIVED);
}
return st;
}
@ -960,7 +955,7 @@ RRDSET *rrdset_create_custom(
}
#endif
#ifdef ENABLE_DBENGINE
rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
metalog_upd_objcount(host, 1);
metalog_commit_update_chart(st);
#endif

2
streaming/README.md
View file

@ -189,7 +189,7 @@ them `/var/lib/netdata/registry/netdata.unique.id`). So, metrics for Netdata `A`
any number of other Netdata, will have the same `MACHINE_GUID`.
You can also use `default memory mode = dbengine` for an API key or `memory mode = dbengine` for
a single host. The additional `page cache size` and `dbengine disk space` configuration options
a single host. The additional `page cache size` and `dbengine multihost disk space` configuration options
are inherited from the global Netdata configuration.
##### allow from

19
streaming/receiver.c
View file

@ -287,25 +287,6 @@ static int rrdpush_receive(struct receiver_state *rpt)
}
netdata_mutex_unlock(&rpt->host->receiver_lock);
}
else rrdhost_update(rpt->host
, rpt->hostname
, rpt->registry_hostname
, rpt->machine_guid
, rpt->os
, rpt->timezone
, rpt->tags
, rpt->program_name
, rpt->program_version
, rpt->update_every
, history
, mode
, (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
, (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
, rrdpush_destination
, rrdpush_api_key
, rrdpush_send_charts_matching
, rpt->system_info);
int ssl = 0;
#ifdef ENABLE_HTTPS

2
streaming/rrdpush.c
View file

@ -611,6 +611,8 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
*/
struct receiver_state *rpt = callocz(1, sizeof(*rpt));
RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */
host = NULL;
if (host) {
netdata_mutex_lock(&host->receiver_lock);
if (host->receiver != NULL) {

2
web/api/formatters/charts2json.c
View file

@ -111,7 +111,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
size_t found = 0;
RRDHOST *h;
rrdhost_foreach_read(h) {
if(!rrdhost_should_be_removed(h, host, now)) {
if(!rrdhost_should_be_removed(h, host, now) && !rrdhost_flag_check(h, RRDHOST_FLAG_ARCHIVED)) {
buffer_sprintf(wb
, "%s\n\t\t{"
"\n\t\t\t\"hostname\": \"%s\""

2
web/api/web_api_v1.c
View file

@ -778,6 +778,8 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
int count = 0;
rrd_rdlock();
rrdhost_foreach_read(rc) {
if (rrdhost_flag_check(rc, RRDHOST_FLAG_ARCHIVED))
continue;
if(count > 0) buffer_strcat(wb, ",\n");
buffer_sprintf(wb, "\t\t\"%s\"", rc->hostname);
count++;