0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-25 05:31:37 +00:00

Fixing DNS-lookup performance issue on FreeBSD. ()

Our default configuration includes:
   allow connections from = localhost *
   allow management from = localhost

The problem occurs when a connection is received that passes the `allow connections` pattern
match, but fails the ACL check for `allow management`. During the failure processing path the
DNS lookup is triggered to allow the FQDN to be checked against the pattern. On a FreeBSD
system this lookup fails more slowly than linux and causes a visible performance problem
during stress-testing.

The fix adds a heuristic to analyse the patterns and determine if it is possible to match a DNS name,
or only match a numeric IP address (either IPv4 or IPv6), or only match a constant value. This
heuristic is used to disable the DNS checks when they cannot produce anything that may match
the pattern. Each heuristic is evaluated once, when the configuration is loaded, not per-connection to the agent.

Because the heuristic is not exact it can be overridden using the new config options for each of the ACL connection filters to set it to "yes", "no" or "heuristic". The default for everything *except* the netdata.conf ACL is "heuristic". Because of the numeric-patterns in the netdata.conf ACL the default is set to "no".
This commit is contained in:
Andrew Moss 2019-10-24 20:44:56 +02:00 committed by GitHub
parent 88f966593a
commit 01aaa90939
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 213 additions and 33 deletions
collectors/statsd.plugin
daemon
libnetdata
registry
web/server

View file

@ -1014,7 +1014,8 @@ void *statsd_collector_thread(void *ptr) {
, statsd_rcv_callback
, statsd_snd_callback
, statsd_timer_callback
, NULL
, NULL // No access control pattern
, 0 // No dns lookups for access control pattern
, (void *)d
, 0 // tcp request timeout, 0 = disabled
, statsd.tcp_idle_timeout // tcp idle timeout, 0 = disabled

View file

@ -96,22 +96,69 @@ void web_server_threading_selection(void) {
}
}
void web_server_config_options(void) {
web_client_timeout = (int) config_get_number(CONFIG_SECTION_WEB, "disconnect idle clients after seconds", web_client_timeout);
web_client_first_request_timeout = (int) config_get_number(CONFIG_SECTION_WEB, "timeout for first request", web_client_first_request_timeout);
web_client_streaming_rate_t = config_get_number(CONFIG_SECTION_WEB, "accept a streaming request every seconds", web_client_streaming_rate_t);
int make_dns_decision(const char *section_name, const char *config_name, const char *default_value, SIMPLE_PATTERN *p)
{
char *value = config_get(section_name,config_name,default_value);
if(!strcmp("yes",value))
return 1;
if(!strcmp("no",value))
return 0;
if(strcmp("heuristic",value))
error("Invalid configuration option '%s' for '%s'/'%s'. Valid options are 'yes', 'no' and 'heuristic'. Proceeding with 'heuristic'",
value, section_name, config_name);
return simple_pattern_is_potential_name(p);
}
respect_web_browser_do_not_track_policy = config_get_boolean(CONFIG_SECTION_WEB, "respect do not track policy", respect_web_browser_do_not_track_policy);
void web_server_config_options(void)
{
web_client_timeout =
(int)config_get_number(CONFIG_SECTION_WEB, "disconnect idle clients after seconds", web_client_timeout);
web_client_first_request_timeout =
(int)config_get_number(CONFIG_SECTION_WEB, "timeout for first request", web_client_first_request_timeout);
web_client_streaming_rate_t =
config_get_number(CONFIG_SECTION_WEB, "accept a streaming request every seconds", web_client_streaming_rate_t);
respect_web_browser_do_not_track_policy =
config_get_boolean(CONFIG_SECTION_WEB, "respect do not track policy", respect_web_browser_do_not_track_policy);
web_x_frame_options = config_get(CONFIG_SECTION_WEB, "x-frame-options response header", "");
if(!*web_x_frame_options) web_x_frame_options = NULL;
if(!*web_x_frame_options)
web_x_frame_options = NULL;
web_allow_connections_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_dashboard_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_badges_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_registry_from = simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_streaming_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from", "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_mgmt_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_connections_from =
simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"),
NULL, SIMPLE_PATTERN_EXACT);
web_allow_connections_dns =
make_dns_decision(CONFIG_SECTION_WEB, "allow connections by dns", "heuristic", web_allow_connections_from);
web_allow_dashboard_from =
simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"),
NULL, SIMPLE_PATTERN_EXACT);
web_allow_dashboard_dns =
make_dns_decision(CONFIG_SECTION_WEB, "allow dashboard by dns", "heuristic", web_allow_dashboard_from);
web_allow_badges_from =
simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_badges_dns =
make_dns_decision(CONFIG_SECTION_WEB, "allow badges by dns", "heuristic", web_allow_badges_from);
web_allow_registry_from =
simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_registry_dns = make_dns_decision(CONFIG_SECTION_REGISTRY, "allow by dns", "heuristic",
web_allow_registry_from);
web_allow_streaming_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"),
NULL, SIMPLE_PATTERN_EXACT);
web_allow_streaming_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow streaming by dns", "heuristic",
web_allow_streaming_from);
// Note the default is not heuristic, the wildcards could match DNS but the intent is ip-addresses.
web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from",
"localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*"
" 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*"
" 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*"
" 172.31.*"), NULL, SIMPLE_PATTERN_EXACT);
web_allow_netdataconf_dns =
make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_mgmt_from);
web_allow_mgmt_from =
simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"),
NULL, SIMPLE_PATTERN_EXACT);
web_allow_mgmt_dns =
make_dns_decision(CONFIG_SECTION_WEB, "allow management by dns","heuristic",web_allow_mgmt_from);
#ifdef NETDATA_WITH_ZLIB

View file

@ -260,3 +260,74 @@ void simple_pattern_free(SIMPLE_PATTERN *list) {
free_pattern(((struct simple_pattern *)list));
}
/* Debugging patterns
This code should be dead - it is useful for debugging but should not be called by production code.
Feel free to comment it out, but please leave it in the file.
*/
extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p)
{
struct simple_pattern *root = (struct simple_pattern *)p;
if(root==NULL) {
debug(debug_type,"dump_pattern(NULL)");
return;
}
debug(debug_type,"dump_pattern(%p) child=%p next=%p mode=%d match=%s", root, root->child, root->next, root->mode,
root->match);
if(root->child!=NULL)
simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->child);
if(root->next!=NULL)
simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->next);
}
/* Heuristic: decide if the pattern could match a DNS name.
Although this functionality is used directly by socket.c:connection_allowed() it must be in this file
because of the SIMPLE_PATTERN/simple_pattern structure hiding.
Based on RFC952 / RFC1123. We need to decide if the pattern may match a DNS name, or not. For the negative
cases we need to be sure that it can only match an ipv4 or ipv6 address:
* IPv6 addresses contain ':', which are illegal characters in DNS.
* IPv4 addresses cannot contain alpha- characters.
* DNS TLDs must be alphanumeric to distinguish from IPv4.
Some patterns (e.g. "*a*" ) could match multiple cases (i.e. DNS or IPv6).
Some patterns will be awkward (e.g. "192.168.*") as they look like they are intended to match IPv4-only
but could match DNS (i.e. "192.168.com" is a valid name).
*/
static void scan_is_potential_name(struct simple_pattern *p, int *alpha, int *colon, int *wildcards)
{
while (p) {
if (p->match) {
if(p->mode == SIMPLE_PATTERN_EXACT && !strcmp("localhost", p->match)) {
p = p->child;
continue;
}
char const *scan = p->match;
while (*scan != 0) {
if ((*scan >= 'a' && *scan <= 'z') || (*scan >= 'A' && *scan <= 'Z'))
*alpha = 1;
if (*scan == ':')
*colon = 1;
scan++;
}
if (p->mode != SIMPLE_PATTERN_EXACT)
*wildcards = 1;
p = p->child;
}
}
}
extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p)
{
int alpha=0, colon=0, wildcards=0;
struct simple_pattern *root = (struct simple_pattern*)p;
while (root != NULL) {
if (root->match != NULL) {
scan_is_potential_name(root, &alpha, &colon, &wildcards);
}
if (root->mode != SIMPLE_PATTERN_EXACT)
wildcards = 1;
root = root->next;
}
return (alpha || wildcards) && !colon;
}

View file

@ -30,4 +30,7 @@ extern int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str,
// list can be NULL, in which case, this does nothing.
extern void simple_pattern_free(SIMPLE_PATTERN *list);
extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) ;
extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) ;
#endif //NETDATA_SIMPLE_PATTERN_H

View file

@ -1007,13 +1007,16 @@ int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
* of *writable* bytes (i.e. be aware of the strdup used to compact the pollinfo).
*/
extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list,
const char *patname) {
const char *patname, int allow_dns)
{
debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns);
if (!access_list)
return 1;
if (simple_pattern_matches(access_list, client_ip))
return 1;
// If the hostname is unresolved (and needed) then attempt the DNS lookups.
if (client_host[0]==0)
//if (client_host[0]==0 && simple_pattern_is_potential_name(access_list))
if (client_host[0]==0 && allow_dns)
{
struct sockaddr_storage sadr;
socklen_t addrlen = sizeof(sadr);
@ -1021,8 +1024,8 @@ extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t
if (err != 0 ||
(err = getnameinfo((struct sockaddr *)&sadr, addrlen, client_host, (socklen_t)hostsize,
NULL, 0, NI_NAMEREQD)) != 0) {
error("Incoming connection on '%s' does not match a numeric pattern, "
"and host could not be resolved (err=%s)", client_ip, gai_strerror(err));
error("Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)",
patname, client_ip, gai_strerror(err));
if (hostsize >= 8)
strcpy(client_host,"UNKNOWN");
return 0;
@ -1074,9 +1077,8 @@ extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t
// --------------------------------------------------------------------------------------------------------------------
// accept_socket() - accept a socket and store client IP and port
int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list) {
char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns) {
struct sockaddr_storage sadr;
socklen_t addrlen = sizeof(sadr);
@ -1088,7 +1090,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien
strncpyz(client_ip, "UNKNOWN", ipsize - 1);
strncpyz(client_port, "UNKNOWN", portsize - 1);
}
if(!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) {
if (!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) {
strncpy(client_ip, "localhost", ipsize);
client_ip[ipsize - 1] = '\0';
}
@ -1126,7 +1128,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien
debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd);
break;
}
if(!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection")) {
if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) {
errno = 0;
error("Permission denied for client '%s', port '%s'", client_ip, client_port);
close(nfd);
@ -1135,7 +1137,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien
}
}
#ifdef HAVE_ACCEPT4
else if(errno == ENOSYS)
else if (errno == ENOSYS)
error("netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. Recompile netdata like this: ./configure --disable-accept4 ...");
#endif
@ -1444,7 +1446,7 @@ static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, sho
debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd);
nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, INET6_ADDRSTRLEN, client_port, NI_MAXSERV,
client_host, NI_MAXHOST, p->access_list);
client_host, NI_MAXHOST, p->access_list, p->allow_dns);
if (unlikely(nfd < 0)) {
// accept failed
@ -1562,6 +1564,7 @@ void poll_events(LISTEN_SOCKETS *sockets
, int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/)
, void (*tmr_callback)(void * /*timer_data*/)
, SIMPLE_PATTERN *access_list
, int allow_dns
, void *data
, time_t tcp_request_timeout_seconds
, time_t tcp_idle_timeout_seconds
@ -1592,6 +1595,7 @@ void poll_events(LISTEN_SOCKETS *sockets
.checks_every = (tcp_idle_timeout_seconds / 3) + 1,
.access_list = access_list,
.allow_dns = allow_dns,
.timer_milliseconds = timer_milliseconds,
.timer_data = timer_data,

View file

@ -73,9 +73,9 @@ extern int sock_enlarge_in(int fd);
extern int sock_enlarge_out(int fd);
extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize,
SIMPLE_PATTERN *access_list, const char *patname);
SIMPLE_PATTERN *access_list, const char *patname, int allow_dns);
extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list);
char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns);
#ifndef HAVE_ACCEPT4
extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags);
@ -155,6 +155,7 @@ struct poll {
struct pollinfo *first_free;
SIMPLE_PATTERN *access_list;
int allow_dns;
void *(*add_callback)(POLLINFO *pi, short int *events, void *data);
void (*del_callback)(POLLINFO *pi);
@ -193,6 +194,7 @@ extern void poll_events(LISTEN_SOCKETS *sockets
, int (*snd_callback)(POLLINFO *pi, short int *events)
, void (*tmr_callback)(void *timer_data)
, SIMPLE_PATTERN *access_list
, int allow_dns
, void *data
, time_t tcp_request_timeout_seconds
, time_t tcp_idle_timeout_seconds

View file

@ -122,6 +122,23 @@ Netdata v1.9+ support limiting access to the registry from given IPs, like this:
Keep in mind that connections to Netdata API ports are filtered by `[web].allow connections from`. So, IPs allowed by `[registry].allow from` should also be allowed by `[web].allow connection from`.
The patterns can be matches over IP addresses or FQDN of the host.
In order to check the FQDN of the connection without opening the Netdata agent to DNS-spoofing, a reverse-dns record
must be setup for the connecting host. At connection time the reverse-dns of the peer IP address is resolved, and
a forward DNS resolution is made to validate the IP address against the name-pattern.
Please note that this process can be expensive on a machine that is serving many connections. The behaviour of
the pattern matching can be controlled with the following setting:
```
[registry]
allow by dns = heuristic
```
The settings are:
* `yes` allows the pattern to match DNS names.
* `no` disables DNS matching for the patterns (they only match IP addresses).
* `heuristic` will estimate if the patterns should match FQDNs by the presence or absence of `:`s or alpha-characters.
### Where is the registry database stored?
`/var/lib/netdata/registry/*.db`

View file

@ -149,7 +149,7 @@ Netdata supports access lists in `netdata.conf`:
allow management from = localhost
```
`*` does string matches on the IPs of the clients.
`*` does string matches on the IPs or FQDNs of the clients.
- `allow connections from` matches anyone that connects on the Netdata port(s).
So, if someone is not allowed, it will be connected and disconnected immediately, without reading even
@ -169,6 +169,26 @@ Netdata supports access lists in `netdata.conf`:
- `allow management from` checks the IPs to allow API management calls. Management via the API is currently supported for [health](../api/health/#health-management-api)
In order to check the FQDN of the connection without opening the Netdata agent to DNS-spoofing, a reverse-dns record
must be setup for the connecting host. At connection time the reverse-dns of the peer IP address is resolved, and
a forward DNS resolution is made to validate the IP address against the name-pattern.
Please note that this process can be expensive on a machine that is serving many connections. Each access list has an
associated configuration option to turn off DNS-based patterns completely to avoid incurring this cost at run-time:
```
allow connections by dns = heuristic
allow dashboard by dns = heuristic
allow badges by dns = heuristic
allow streaming by dns = heuristic
allow netdata.conf by dns = no
allow management by dns = heuristic
```
The three possible values for each of these options are `yes`, `no` and `heuristic`. The `heuristic` option disables
the check when the pattern only contains IPv4/IPv6 addresses or `localhost`, and enables it when wildcards are
present that may match DNS FQDNs.
### Other netdata.conf [web] section options
|setting|default|info|

View file

@ -396,6 +396,7 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {
, web_server_snd_callback
, web_server_tmr_callback
, web_allow_connections_from
, web_allow_connections_dns
, NULL
, web_client_first_request_timeout
, web_client_timeout

View file

@ -74,46 +74,53 @@ void api_listen_sockets_setup(void) {
// access lists
SIMPLE_PATTERN *web_allow_connections_from = NULL;
int web_allow_connections_dns;
// WEB_CLIENT_ACL
SIMPLE_PATTERN *web_allow_dashboard_from = NULL;
int web_allow_dashboard_dns;
SIMPLE_PATTERN *web_allow_registry_from = NULL;
int web_allow_registry_dns;
SIMPLE_PATTERN *web_allow_badges_from = NULL;
int web_allow_badges_dns;
SIMPLE_PATTERN *web_allow_mgmt_from = NULL;
int web_allow_mgmt_dns;
SIMPLE_PATTERN *web_allow_streaming_from = NULL;
int web_allow_streaming_dns;
SIMPLE_PATTERN *web_allow_netdataconf_from = NULL;
int web_allow_netdataconf_dns;
void web_client_update_acl_matches(struct web_client *w) {
w->acl = WEB_CLIENT_ACL_NONE;
if (!web_allow_dashboard_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_dashboard_from, "dashboard"))
web_allow_dashboard_from, "dashboard", web_allow_dashboard_dns))
w->acl |= WEB_CLIENT_ACL_DASHBOARD;
if (!web_allow_registry_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_registry_from, "registry"))
web_allow_registry_from, "registry", web_allow_registry_dns))
w->acl |= WEB_CLIENT_ACL_REGISTRY;
if (!web_allow_badges_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_badges_from, "badges"))
web_allow_badges_from, "badges", web_allow_badges_dns))
w->acl |= WEB_CLIENT_ACL_BADGE;
if (!web_allow_mgmt_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_mgmt_from, "management"))
web_allow_mgmt_from, "management", web_allow_mgmt_dns))
w->acl |= WEB_CLIENT_ACL_MGMT;
if (!web_allow_streaming_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_streaming_from, "streaming"))
web_allow_streaming_from, "streaming", web_allow_streaming_dns))
w->acl |= WEB_CLIENT_ACL_STREAMING;
if (!web_allow_netdataconf_from ||
connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
web_allow_netdataconf_from, "netdata.conf"))
web_allow_netdataconf_from, "netdata.conf", web_allow_netdataconf_dns))
w->acl |= WEB_CLIENT_ACL_NETDATACONF;
w->acl &= w->port_acl;

View file

@ -20,12 +20,19 @@ typedef enum web_server_mode {
} WEB_SERVER_MODE;
extern SIMPLE_PATTERN *web_allow_connections_from;
extern int web_allow_connections_dns;
extern SIMPLE_PATTERN *web_allow_dashboard_from;
extern int web_allow_dashboard_dns;
extern SIMPLE_PATTERN *web_allow_registry_from;
extern int web_allow_registry_dns;
extern SIMPLE_PATTERN *web_allow_badges_from;
extern int web_allow_badges_dns;
extern SIMPLE_PATTERN *web_allow_streaming_from;
extern int web_allow_streaming_dns;
extern SIMPLE_PATTERN *web_allow_netdataconf_from;
extern int web_allow_netdataconf_dns;
extern SIMPLE_PATTERN *web_allow_mgmt_from;
extern int web_allow_mgmt_dns;
extern WEB_SERVER_MODE web_server_mode;