archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-13 09:11:50 +00:00
Code Issues Projects Releases Wiki Activity

Backend and SSL! ()

Browse source 
* SSL_backend Begin of the encryptation of backend!

* SSL_backend changing opentsdb!

* SSL_backend fix HTTP message with JSON!

* SSL_backend HTTP API done!

* SSL_fix_format preparing to connect with proxy!

* SSL_backend wip SSL send/receive !

* SSL_backend working with proxy

* SSL_backend removing comments!

* SSL_backend docummentation!

* SSL_backend review]!

* SSL_backend organizing!

* Alarm_backend remove comments!

* SSL_backend!

* SSL_backend typedef!

* SSL_backend bring switch!

* SSL_backend commiting format changes!

* SSL_backend fix github parser!

* SSL_Backend fix format!

* SSL_backend switch everything!

* SSL_backend reviewing!

* SSL_backend comments!

* SSL_backend indentation!

* SSL_backend indentation 3!

* SSL_backend documentation!

* SSL_backend hidden pointer!

* SSL_backend missing space

* SSL_backend change documentation!

* SSL_backend change documentation 2!
This commit is contained in:
thiagoftsm 2019-06-27 11:20:28 -03:00 • committed by GitHub
parent 60a73e90de
commit ca1799280d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 589 additions and 109 deletions

6
backends/README.md
View file

@ -22,7 +22,7 @@ X seconds (though, it can send them per second if you need it to).
metrics are sent to the backend server as `prefix.hostname.chart.dimension`. `prefix` is
configured below, `hostname` is the hostname of the machine (can also be configured).
- **opentsdb** (`telnet interface`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc)
- **opentsdb** (`telnet or HTTP interfaces`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc)
metrics are sent to opentsdb as `prefix.chart.dimension` with tag `host=hostname`.
@ -76,7 +76,7 @@ of `netdata.conf` from your netdata):
```
[backend]
enabled = yes | no
type = graphite | opentsdb | json | prometheus_remote_write | kinesis
type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | prometheus_remote_write | json | kinesis
host tags = list of TAG=VALUE
destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis
data source = average | sum | as collected
@ -92,7 +92,7 @@ of `netdata.conf` from your netdata):
- `enabled = yes | no`, enables or disables sending data to a backend
- `type = graphite | opentsdb | json | kinesis`, selects the backend type
- `type = graphite | opentsdb:telnet | opentsdb:http | opentsdb:https | json | kinesis`, selects the backend type
- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames,
IPs (IPv4 and IPv6) and ports to connect to.

416
backends/backends.c
View file

@ -246,6 +246,194 @@ static void backends_main_cleanup(void *ptr) {
static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
}
/**
* Set Kinesis variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_kinesis_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
(void)default_port;
#ifndef HAVE_KINESIS
(void)brc;
(void)brf;
#endif
#if HAVE_KINESIS
*brc = process_json_response;
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
*brf = format_dimension_collected_json_plaintext;
else
*brf = format_dimension_stored_json_plaintext;
#endif
}
/**
* Set Prometheus variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_prometheus_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
(void)default_port;
(void)brf;
#ifndef ENABLE_PROMETHEUS_REMOTE_WRITE
(void)brc;
#endif
#if ENABLE_PROMETHEUS_REMOTE_WRITE
*brc = process_prometheus_remote_write_response;
#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
}
/**
* Set JSON variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_json_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
*default_port = 5448;
*brc = process_json_response;
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
*brf = format_dimension_collected_json_plaintext;
else
*brf = format_dimension_stored_json_plaintext;
}
/**
* Set OpenTSDB HTTP variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_opentsdb_http_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
*default_port = 4242;
*brc = process_opentsdb_response;
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
*brf = format_dimension_collected_opentsdb_http;
else
*brf = format_dimension_stored_opentsdb_http;
}
/**
* Set OpenTSDB Telnet variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_opentsdb_telnet_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
*default_port = 4242;
*brc = process_opentsdb_response;
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
*brf = format_dimension_collected_opentsdb_telnet;
else
*brf = format_dimension_stored_opentsdb_telnet;
}
/**
* Set Graphite variables
*
* Set the variables necessaries to work with this specific backend.
*
* @param default_port the default port of the backend
* @param brc function called to check the result.
* @param brf function called to format the msessage to the backend
* @param type the backend string selector.
*/
void backend_set_graphite_variables(int *default_port,
backend_response_checker_t brc,
backend_request_formatter_t brf)
{
*default_port = 2003;
*brc = process_graphite_response;
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
*brf = format_dimension_collected_graphite_plaintext;
else
*brf = format_dimension_stored_graphite_plaintext;
}
/**
* Select Type
*
* Select the backedn type based in the user input
*
* @param type is the string that defines the backend type
*
* @return It returns the backend id.
*/
BACKEND_TYPE backend_select_type(const char *type) {
if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
return BACKEND_TYPE_GRAPHITE;
}
else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
return BACKEND_TYPE_OPENTSDB_USING_TELNET;
}
else if(!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
return BACKEND_TYPE_OPENTSDB_USING_HTTP;
}
else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
return BACKEND_TYPE_JSON;
}
else if (!strcmp(type, "prometheus_remote_write")) {
return BACKEND_TYPE_PROMETEUS;
}
else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
return BACKEND_TYPE_KINESIS;
}
return BACKEND_TYPE_UNKNOWN;
}
/**
* Backend main
*
* The main thread used to control the backedns.
*
* @param ptr a pointer to netdata_static_structure.
*
* @return It always return NULL.
*/
void *backends_main(void *ptr) {
netdata_thread_cleanup_push(backends_main_cleanup, ptr);
@ -265,6 +453,9 @@ void *backends_main(void *ptr) {
BUFFER *http_request_header = buffer_create(1);
#endif
#ifdef ENABLE_HTTPS
struct netdata_ssl opentsdb_ssl = {NULL , NETDATA_SSL_START};
#endif
// ------------------------------------------------------------------------
// collect configuration options
@ -313,76 +504,66 @@ void *backends_main(void *ptr) {
// ------------------------------------------------------------------------
// select the backend type
if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
default_port = 2003;
backend_response_checker = process_graphite_response;
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_graphite_plaintext;
else
backend_request_formatter = format_dimension_stored_graphite_plaintext;
}
else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
default_port = 4242;
backend_response_checker = process_opentsdb_response;
if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_opentsdb_telnet;
else
backend_request_formatter = format_dimension_stored_opentsdb_telnet;
}
else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
default_port = 5448;
backend_response_checker = process_json_response;
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_json_plaintext;
else
backend_request_formatter = format_dimension_stored_json_plaintext;
}
else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
#if HAVE_KINESIS
do_kinesis = 1;
if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {
error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);
goto cleanup;
}
kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
backend_response_checker = process_json_response;
if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_json_plaintext;
else
backend_request_formatter = format_dimension_stored_json_plaintext;
#else
error("AWS Kinesis support isn't compiled");
#endif /* HAVE_KINESIS */
}
else if (!strcmp(type, "prometheus_remote_write")) {
#if ENABLE_PROMETHEUS_REMOTE_WRITE
do_prometheus_remote_write = 1;
backend_response_checker = process_prometheus_remote_write_response;
init_write_request();
#else
error("Prometheus remote write support isn't compiled");
#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */
}
else {
BACKEND_TYPE work_type = backend_select_type(type);
if (work_type == BACKEND_TYPE_UNKNOWN) {
error("BACKEND: Unknown backend type '%s'", type);
goto cleanup;
}
switch (work_type) {
case BACKEND_TYPE_OPENTSDB_USING_HTTP: {
#ifdef ENABLE_HTTPS
if (!strcmp(type, "opentsdb:https")) {
security_start_ssl(NETDATA_SSL_CONTEXT_OPENTSDB);
}
#endif
backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_PROMETEUS: {
#if ENABLE_PROMETHEUS_REMOTE_WRITE
do_prometheus_remote_write = 1;
init_write_request();
#else
error("BACKEND: Prometheus remote write support isn't compiled");
#endif // ENABLE_PROMETHEUS_REMOTE_WRITE
backend_set_prometheus_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_KINESIS: {
#if HAVE_KINESIS
do_kinesis = 1;
if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {
error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);
goto cleanup;
}
kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);
#else
error("BACKEND: AWS Kinesis support isn't compiled");
#endif // HAVE_KINESIS
backend_set_kinesis_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_GRAPHITE: {
backend_set_graphite_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_OPENTSDB_USING_TELNET: {
backend_set_opentsdb_telnet_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_JSON: {
backend_set_json_variables(&default_port,&backend_response_checker,&backend_request_formatter);
break;
}
case BACKEND_TYPE_UNKNOWN: {
break;
}
}
#if ENABLE_PROMETHEUS_REMOTE_WRITE
if((backend_request_formatter == NULL && !do_prometheus_remote_write) || backend_response_checker == NULL) {
#else
@ -393,25 +574,25 @@ void *backends_main(void *ptr) {
}
// ------------------------------------------------------------------------
// prepare the charts for monitoring the backend operation
// ------------------------------------------------------------------------
// prepare the charts for monitoring the backend operation
struct rusage thread;
collected_number
chart_buffered_metrics = 0,
chart_lost_metrics = 0,
chart_sent_metrics = 0,
chart_buffered_bytes = 0,
chart_received_bytes = 0,
chart_sent_bytes = 0,
chart_receptions = 0,
chart_transmission_successes = 0,
chart_transmission_failures = 0,
chart_data_lost_events = 0,
chart_lost_bytes = 0,
chart_backend_reconnects = 0;
// chart_backend_latency = 0;
chart_buffered_metrics = 0,
chart_lost_metrics = 0,
chart_sent_metrics = 0,
chart_buffered_bytes = 0,
chart_received_bytes = 0,
chart_sent_bytes = 0,
chart_receptions = 0,
chart_transmission_successes = 0,
chart_transmission_failures = 0,
chart_data_lost_events = 0,
chart_lost_bytes = 0,
chart_backend_reconnects = 0;
// chart_backend_latency = 0;
RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, global_backend_update_every, RRDSET_TYPE_LINE);
rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@ -432,12 +613,12 @@ void *backends_main(void *ptr) {
rrddim_add(chart_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
/*
* this is misleading - we can only measure the time we need to send data
* this time is not related to the time required for the data to travel to
* the backend database and the time that server needed to process them
*
* issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
*
* this is misleading - we can only measure the time we need to send data
* this time is not related to the time required for the data to travel to
* the backend database and the time that server needed to process them
*
* issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
*
RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, global_backend_update_every, RRDSET_TYPE_AREA);
rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
*/
@ -668,7 +849,16 @@ void *backends_main(void *ptr) {
while(sock != -1 && errno != EWOULDBLOCK) {
buffer_need_bytes(response, 4096);
ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
ssize_t r;
#ifdef ENABLE_HTTPS
if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {
r = SSL_read(opentsdb_ssl.conn, &response->buffer[response->len], response->size - response->len);
} else {
r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
}
#else
r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);
#endif
if(likely(r > 0)) {
// we received some data
response->len += r;
@ -701,7 +891,37 @@ void *backends_main(void *ptr) {
size_t reconnects = 0;
sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
#ifdef ENABLE_HTTPS
if(sock != -1) {
if(netdata_opentsdb_ctx) {
if(!opentsdb_ssl.conn) {
opentsdb_ssl.conn = SSL_new(netdata_opentsdb_ctx);
if(!opentsdb_ssl.conn) {
error("Failed to allocate SSL structure %d.", sock);
opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
}
} else {
SSL_clear(opentsdb_ssl.conn);
}
}
if(opentsdb_ssl.conn) {
if(SSL_set_fd(opentsdb_ssl.conn, sock) != 1) {
error("Failed to set the socket to the SSL on socket fd %d.", host->rrdpush_sender_socket);
opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
} else {
opentsdb_ssl.flags = NETDATA_SSL_HANDSHAKE_COMPLETE;
SSL_set_connect_state(opentsdb_ssl.conn);
int err = SSL_connect(opentsdb_ssl.conn);
if (err != 1) {
err = SSL_get_error(opentsdb_ssl.conn, err);
error("SSL cannot connect with the server: %s ", ERR_error_string((long)SSL_get_error(opentsdb_ssl.conn, err), NULL));
opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
} //TODO: check certificate here
}
}
}
#endif
chart_backend_reconnects += reconnects;
// chart_backend_latency += now_monotonic_usec() - start_ut;
}
@ -756,7 +976,17 @@ void *backends_main(void *ptr) {
}
#endif
ssize_t written = send(sock, buffer_tostring(b), len, flags);
ssize_t written;
#ifdef ENABLE_HTTPS
if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {
written = SSL_write(opentsdb_ssl.conn, buffer_tostring(b), len);
} else {
written = send(sock, buffer_tostring(b), len, flags);
}
#else
written = send(sock, buffer_tostring(b), len, flags);
#endif
// chart_backend_latency += now_monotonic_usec() - start_ut;
if(written != -1 && (size_t)written == len) {
// we sent the data successfully
@ -883,6 +1113,14 @@ cleanup:
buffer_free(b);
buffer_free(response);
#ifdef ENABLE_HTTPS
if(netdata_opentsdb_ctx) {
if(opentsdb_ssl.conn) {
SSL_free(opentsdb_ssl.conn);
}
}
#endif
netdata_thread_cleanup_pop(1);
return NULL;
}

14
backends/backends.h
View file

@ -15,6 +15,20 @@ typedef enum backend_options {
BACKEND_OPTION_SEND_NAMES = (1 << 16)
} BACKEND_OPTIONS;
typedef enum backend_types {
BACKEND_TYPE_UNKNOWN, //Invalid type
BACKEND_TYPE_GRAPHITE, //Send plain text to Graphite
BACKEND_TYPE_OPENTSDB_USING_TELNET, //Send data to OpenTSDB using telnet API
BACKEND_TYPE_OPENTSDB_USING_HTTP, //Send data to OpenTSDB using HTTP API
BACKEND_TYPE_JSON, //Stores the data using JSON.
BACKEND_TYPE_PROMETEUS, //The user selected to use Prometheus backend
BACKEND_TYPE_KINESIS //Send message to AWS Kinesis
} BACKEND_TYPE;
typedef int (**backend_response_checker_t)(BUFFER *);
typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);
#define BACKEND_OPTIONS_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)
#define BACKEND_OPTIONS_DATA_SOURCE(backend_options) (backend_options & BACKEND_OPTIONS_SOURCE_BITS)

26
backends/opentsdb/README.md Normal file
View file

@ -0,0 +1,26 @@
# OpenTSDB with HTTP
Since version 1.16 the Netdata has the feature to communicate with OpenTSDB using HTTP API. To enable this channel
it is necessary to set the following options in your netdata.conf
```
[backend]
type = opentsdb:http
destination = localhost:4242
```
, in this example we are considering that OpenTSDB is running with its default port (4242).
## HTTPS
Netdata also supports sending the metrics using SSL/TLS, but OpenTDSB does not have support to safety connections,
so it will be necessary to configure a reverse-proxy to enable the HTTPS communication. After to configure your proxy the
following changes must be done in the netdata.conf:
```
[backend]
type = opentsdb:https
destination = localhost:8082
```
In this example we used the port 8082 for our reverse proxy.

115
backends/opentsdb/opentsdb.c
View file

@ -80,6 +80,7 @@ int format_dimension_stored_opentsdb_telnet(
return 1;
}
return 0;
}
@ -87,4 +88,118 @@ int process_opentsdb_response(BUFFER *b) {
return discard_response(b, "opentsdb");
}
static inline void opentsdb_build_message(BUFFER *b, char *message, const char *hostname, int length) {
buffer_sprintf(
b
, "POST /api/put HTTP/1.1\r\n"
"Host: %s\r\n"
"Content-Type: application/json\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s"
, hostname
, length
, message
);
}
int format_dimension_collected_opentsdb_http(
BUFFER *b // the buffer to write data to
, const char *prefix // the prefix to use
, RRDHOST *host // the host this chart comes from
, const char *hostname // the hostname (to override host->hostname)
, RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
) {
(void)host;
(void)after;
(void)before;
char message[1024];
char chart_name[RRD_ID_LENGTH_MAX + 1];
char dimension_name[RRD_ID_LENGTH_MAX + 1];
backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
int length = snprintfz(message
, sizeof(message)
, "{"
" \"metric\": \"%s.%s.%s\","
" \"timestamp\": %llu,"
" \"value\": "COLLECTED_NUMBER_FORMAT ","
" \"tags\": {"
" \"host\": \"%s%s%s\""
" }"
"}"
, prefix
, chart_name
, dimension_name
, (unsigned long long)rd->last_collected_time.tv_sec
, rd->last_collected_value
, hostname
, (host->tags)?" ":""
, (host->tags)?host->tags:""
);
if(length > 0) {
opentsdb_build_message(b, message, hostname, length);
}
return 1;
}
int format_dimension_stored_opentsdb_http(
BUFFER *b // the buffer to write data to
, const char *prefix // the prefix to use
, RRDHOST *host // the host this chart comes from
, const char *hostname // the hostname (to override host->hostname)
, RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
) {
(void)host;
time_t first_t = after, last_t = before;
calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);
if(!isnan(value)) {
char chart_name[RRD_ID_LENGTH_MAX + 1];
char dimension_name[RRD_ID_LENGTH_MAX + 1];
backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
char message[1024];
int length = snprintfz(message
, sizeof(message)
, "{"
" \"metric\": \"%s.%s.%s\","
" \"timestamp\": %llu,"
" \"value\": "CALCULATED_NUMBER_FORMAT ","
" \"tags\": {"
" \"host\": \"%s%s%s\""
" }"
"}"
, prefix
, chart_name
, dimension_name
, (unsigned long long)last_t
, value
, hostname
, (host->tags)?" ":""
, (host->tags)?host->tags:""
);
if(length > 0) {
opentsdb_build_message(b, message, hostname, length);
}
return 1;
}
return 0;
}

23
backends/opentsdb/opentsdb.h
View file

@ -31,5 +31,28 @@ extern int format_dimension_stored_opentsdb_telnet(
extern int process_opentsdb_response(BUFFER *b);
int format_dimension_collected_opentsdb_http(
BUFFER *b // the buffer to write data to
, const char *prefix // the prefix to use
, RRDHOST *host // the host this chart comes from
, const char *hostname // the hostname (to override host->hostname)
, RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
);
int format_dimension_stored_opentsdb_http(
BUFFER *b // the buffer to write data to
, const char *prefix // the prefix to use
, RRDHOST *host // the host this chart comes from
, const char *hostname // the hostname (to override host->hostname)
, RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap
);
#endif //NETDATA_BACKEND_OPENTSDB_H

87
libnetdata/socket/security.c
View file

@ -2,6 +2,7 @@
#ifdef ENABLE_HTTPS
SSL_CTX *netdata_opentsdb_ctx=NULL;
SSL_CTX *netdata_client_ctx=NULL;
SSL_CTX *netdata_srv_ctx=NULL;
const char *security_key=NULL;
@ -10,6 +11,15 @@ int netdata_use_ssl_on_stream = NETDATA_SSL_OPTIONAL;
int netdata_use_ssl_on_http = NETDATA_SSL_FORCE; //We force SSL due safety reasons
int netdata_validate_server = NETDATA_SSL_VALID_CERTIFICATE;
/**
* Info Callback
*
* Function used as callback for the OpenSSL Library
*
* @param ssl a pointer to the SSL structure of the client
* @param where the variable with the flags set.
* @param ret the return of the caller
*/
static void security_info_callback(const SSL *ssl, int where, int ret) {
(void)ssl;
if (where & SSL_CB_ALERT) {
@ -17,6 +27,11 @@ static void security_info_callback(const SSL *ssl, int where, int ret) {
}
}
/**
* OpenSSL Library
*
* Starts the openssl library for the Netdata.
*/
void security_openssl_library()
{
#if OPENSSL_VERSION_NUMBER < 0x10100000L
@ -36,6 +51,13 @@ void security_openssl_library()
#endif
}
/**
* OpenSSL common options
*
* Clients and SERVER have common options, this function is responsible to set them in the context.
*
* @param ctx
*/
void security_openssl_common_options(SSL_CTX *ctx) {
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
static char *ciphers = {"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"};
@ -55,10 +77,15 @@ void security_openssl_common_options(SSL_CTX *ctx) {
error("SSL error. cannot set the cipher list");
}
#endif
}
/**
* Initialize Openssl Client
*
* Starts the client context with TLS 1.2.
*
* @return It returns the context on success or NULL otherwise
*/
static SSL_CTX * security_initialize_openssl_client() {
SSL_CTX *ctx;
#if OPENSSL_VERSION_NUMBER < 0x10100000L
@ -66,11 +93,20 @@ static SSL_CTX * security_initialize_openssl_client() {
#else
ctx = SSL_CTX_new(TLS_client_method());
#endif
security_openssl_common_options(ctx);
if(ctx) {
security_openssl_common_options(ctx);
}
return ctx;
}
/**
* Initialize OpenSSL server
*
* Starts the server context with TLS 1.2 and load the certificate.
*
* @return It returns the context on success or NULL otherwise
*/
static SSL_CTX * security_initialize_openssl_server() {
SSL_CTX *ctx;
char lerror[512];
@ -116,18 +152,36 @@ static SSL_CTX * security_initialize_openssl_server() {
return ctx;
}
void security_start_ssl(int type) {
if (!type) {
struct stat statbuf;
if (stat(security_key,&statbuf) || stat(security_cert,&statbuf)) {
info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n");
return;
}
/**
* Start SSL
*
* Call the correct function to start the SSL context.
*
* @param selector informs the context that must be initialized, the following list has the valid values:
* NETDATA_SSL_CONTEXT_SERVER - the server context
* NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context.
* NETDATA_SSL_CONTEXT_OPENTSDB - Starts the OpenTSDB contextv
*/
void security_start_ssl(int selector) {
switch (selector) {
case NETDATA_SSL_CONTEXT_SERVER: {
struct stat statbuf;
if (stat(security_key,&statbuf) || stat(security_cert,&statbuf)) {
info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n");
return;
}
netdata_srv_ctx = security_initialize_openssl_server();
}
else {
netdata_client_ctx = security_initialize_openssl_client();
netdata_srv_ctx = security_initialize_openssl_server();
break;
}
case NETDATA_SSL_CONTEXT_STREAMING: {
netdata_client_ctx = security_initialize_openssl_client();
break;
}
case NETDATA_SSL_CONTEXT_OPENTSDB: {
netdata_opentsdb_ctx = security_initialize_openssl_client();
break;
}
}
}
@ -142,6 +196,11 @@ void security_clean_openssl() {
SSL_CTX_free(netdata_client_ctx);
}
if ( netdata_opentsdb_ctx )
{
SSL_CTX_free(netdata_opentsdb_ctx);
}
#if OPENSSL_VERSION_NUMBER < 0x10100000L
ERR_free_strings();
#endif

7
libnetdata/socket/security.h
View file

@ -11,6 +11,10 @@
# define NETDATA_SSL_INVALID_CERTIFICATE 64 //Accepts invalid certificate
# define NETDATA_SSL_VALID_CERTIFICATE 128 //Accepts invalid certificate
#define NETDATA_SSL_CONTEXT_SERVER 0
#define NETDATA_SSL_CONTEXT_STREAMING 1
#define NETDATA_SSL_CONTEXT_OPENTSDB 2
# ifdef ENABLE_HTTPS
# include <openssl/ssl.h>
@ -24,6 +28,7 @@ struct netdata_ssl{
int flags;
};
extern SSL_CTX *netdata_opentsdb_ctx;
extern SSL_CTX *netdata_client_ctx;
extern SSL_CTX *netdata_srv_ctx;
extern const char *security_key;
@ -34,7 +39,7 @@ extern int netdata_validate_server;
void security_openssl_library();
void security_clean_openssl();
void security_start_ssl(int type);
void security_start_ssl(int selector);
int security_process_accept(SSL *ssl,int msg);
int security_test_certificate(SSL *ssl);

2
streaming/rrdpush.c
View file

@ -651,7 +651,7 @@ void *rrdpush_sender_thread(void *ptr) {
#ifdef ENABLE_HTTPS
if (netdata_use_ssl_on_stream & NETDATA_SSL_FORCE ){
security_start_ssl(1);
security_start_ssl(NETDATA_SSL_CONTEXT_STREAMING);
}
#endif

2
web/server/static/static-threaded.c
View file

@ -458,7 +458,7 @@ void *socket_listen_main_static_threaded(void *ptr) {
fatal("LISTENER: no listen sockets available.");
#ifdef ENABLE_HTTPS
security_start_ssl(0);
security_start_ssl(NETDATA_SSL_CONTEXT_SERVER);
#endif
// 6 threads is the optimal value
// since 6 are the parallel connections browsers will do