mirror of
https://github.com/netdata/netdata.git
synced 2025-04-14 09:38:34 +00:00
systemd-Journal by file (#16038)
* query journal file by file: 17% faster * maintain a registry of journal files in memory and support multiple journal directories; offer sources of journal directories * fixes * overloaded libc fstat64() call to speed up libsystemd * do not just copy unset values, there is a flag that tracks them * optimize facets_row_finished() * use container name in ND_JOURNAL_PROCESS * fix compatibility with versions of libsystemd without sd_journal_open_files_fd() * added more statistics about the time spent per journal file * optimize facets_row_finished() * optimize facets_rows_begin() * tuning * progress reporting * fix journal seek to precisely match log timestamps * support remote sources and namespaces * jf_is_mine() as function * fixes * fixes 2 * fixes 3 * added debug * fixed log * added source for fqs * fix all source names * fix jf_is_mine() to return a value * add rows_useful to journal files * sorted list of all sources * make hostname visible by default * rename sources * increase number of columns * updated apps_groups.conf * support view only transformations * add support for slicing * add support for older versions of systemd * cleanup * added ordering of key values * convert remote IPs to hostnames * fix for hostname resolution * standardize the source name length * added versions * added sources pills and info * fix plural * better formatting for durations * support dynamic unset value * fix sorting * errno to still show numeric values * maintain a used hashes registry * fixed severity * updated function help message with all current parameters accepted * remove internal error * always return null as empty values in data * add default sd_journal_open flags * validate anchor * fix compiler warning * calculate journal vs realtime delta per journal file * up to 2 minutes journal vs realtime delta * more detailed message * do not log zero anchor * fixed message * fix seek to db * request details and dump of all journal files in response * sort files before processing them * do not sort if fewer than 2 files * documentation * added documentation about performance * added field transformations documentation and annotated _CAP_EFFECTIVE * updated docs * updated docs * annotated SOURCE_REALTIME_TIMESTAMP * updated docs * workaround for old systems * updated docs * updated docs * updated docs * updated docs * more fields to show by default * filter data-only query by libsystemd on slice mode * better tail * restore operation of full queries * updated docs * updated docs * added smart field _BOOT_ID to automatically extract the timestamp of the first message of this boot_id * do not seek to anchor on full queries * added tail and delta * alphabetical sort on calculated columns * simplify sorting of facet values * fix sorting of transformed values * simplify code * numeric values for capabilities that do not exist in old systems * do not log if directories do not exist or are not directories
This commit is contained in:
Costa Tsaousis
GitHub
parent
d34dbf844f
commit
9911045b59
9 changed files with 3067 additions and 506 deletions
|
|
@ -83,14 +83,14 @@ xenstat.plugin: xenstat.plugin
|
|||
perf.plugin: perf.plugin
|
||||
charts.d.plugin: *charts.d.plugin*
|
||||
python.d.plugin: *python.d.plugin*
|
||||
systemd-journal.plugin:*systemd-journal.plugin*
|
||||
tc-qos-helper: *tc-qos-helper.sh*
|
||||
fping: fping
|
||||
ioping: ioping
|
||||
go.d.plugin: *go.d.plugin*
|
||||
slabinfo.plugin: slabinfo.plugin
|
||||
slabinfo.plugin: *slabinfo.plugin*
|
||||
ebpf.plugin: *ebpf.plugin*
|
||||
debugfs.plugin: *debugfs.plugin*
|
||||
systemd-journal.plugin: systemd-journal
|
||||
|
||||
# agent-service-discovery
|
||||
agent_sd: agent_sd
|
||||
|
|
@ -137,7 +137,7 @@ modem: ModemManager
|
|||
netmanager: NetworkManager nm* systemd-networkd networkctl netplan connmand wicked* avahi-autoipd networkd-dispatcher
|
||||
firewall: firewalld ufw nft
|
||||
tor: tor
|
||||
bluetooth: bluetooth bluez bluedevil obexd
|
||||
bluetooth: bluetooth bluetoothd bluez bluedevil obexd
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# high availability and balancers
|
||||
|
|
@ -160,7 +160,7 @@ chat: irssi *vines* *prosody* murmurd
|
|||
# -----------------------------------------------------------------------------
|
||||
# monitoring
|
||||
|
||||
logs: ulogd* syslog* rsyslog* logrotate systemd-journald rotatelogs sysklogd metalog
|
||||
logs: ulogd* syslog* rsyslog* logrotate *systemd-journal* rotatelogs sysklogd metalog
|
||||
nms: snmpd vnstatd smokeping zabbix* munin* mon openhpid tailon nrpe
|
||||
monit: monit
|
||||
splunk: splunkd
|
||||
|
|
@ -210,7 +210,7 @@ proxmox-ve: pve* spiceproxy
|
|||
# -----------------------------------------------------------------------------
|
||||
# containers & virtual machines
|
||||
|
||||
containers: lxc* docker* balena*
|
||||
containers: lxc* docker* balena* containerd
|
||||
VMs: vbox* VBox* qemu* kvm*
|
||||
libvirt: virtlogd virtqemud virtstoraged virtnetworkd virtlockd virtinterfaced
|
||||
libvirt: virtnodedevd virtproxyd virtsecretd libvirtd
|
||||
|
|
@ -239,7 +239,7 @@ dhcp: *dhcp* dhclient
|
|||
# -----------------------------------------------------------------------------
|
||||
# name servers and clients
|
||||
|
||||
dns: named unbound nsd pdns_server knotd gdnsd yadifad dnsmasq systemd-resolve* pihole* avahi-daemon avahi-dnsconfd
|
||||
dns: named unbound nsd pdns_server knotd gdnsd yadifad dnsmasq *systemd-resolve* pihole* avahi-daemon avahi-dnsconfd
|
||||
dnsdist: dnsdist
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -272,7 +272,7 @@ backup: rsync lsyncd bacula* borg rclone
|
|||
# -----------------------------------------------------------------------------
|
||||
# cron
|
||||
|
||||
cron: cron* atd anacron systemd-cron* incrond
|
||||
cron: cron* atd anacron *systemd-cron* incrond
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# UPS
|
||||
|
|
@ -320,7 +320,7 @@ airflow: *airflow*
|
|||
# -----------------------------------------------------------------------------
|
||||
# GUI
|
||||
|
||||
X: X Xorg xinit xdm Xwayland xsettingsd
|
||||
X: X Xorg xinit xdm Xwayland xsettingsd touchegg
|
||||
wayland: swaylock swayidle waypipe wayvnc
|
||||
kde: *kdeinit* kdm sddm plasmashell startplasma-* kwin* kwallet* krunner kactivitymanager*
|
||||
gnome: gnome-* gdm gconf* mutter
|
||||
|
|
@ -354,11 +354,11 @@ kswapd: kswapd
|
|||
zswap: zswap
|
||||
kcompactd: kcompactd
|
||||
|
||||
system: systemd-* udisks* udevd* *udevd ipv6_addrconf dbus-* rtkit*
|
||||
system: systemd* udisks* udevd* *udevd ipv6_addrconf dbus-* rtkit*
|
||||
system: mdadm acpid uuidd upowerd elogind* eudev mdev lvmpolld dmeventd
|
||||
system: accounts-daemon rngd haveged rasdaemon irqbalance start-stop-daemon
|
||||
system: supervise-daemon openrc* init runit runsvdir runsv auditd lsmd
|
||||
system: abrt* nscd rtkit-daemon gpg-agent usbguard*
|
||||
system: abrt* nscd rtkit-daemon gpg-agent usbguard* boltd geoclue
|
||||
|
||||
kernel: kworker kthreadd kauditd lockd khelper kdevtmpfs khungtaskd rpciod
|
||||
kernel: fsnotify_mark kthrotld deferwq scsi_* kdmflush oom_reaper kdevtempfs
|
||||
|
|
|
|||
|
|
@ -888,7 +888,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may
|
|||
pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data);
|
||||
|
||||
string_freez(pf->function);
|
||||
freez(pf->payload);
|
||||
freez((void *)pf->payload);
|
||||
}
|
||||
|
||||
void inflight_functions_init(PARSER *parser) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,357 @@
|
|||
<!--
|
||||
title: "SystemD-Journal"
|
||||
description: "View and analyze logs available in systemd journal"
|
||||
custom_edit_url: "https://github.com/netdata/netdata/edit/master/collectors/systemd-journal.plugin/README.md"
|
||||
sidebar_label: "SystemD-Journal"
|
||||
learn_status: "Published"
|
||||
learn_rel_path: "Integrations/Logs"
|
||||
-->
|
||||
|
||||
[KEY FEATURES](#key-features) | [JOURNAL SOURCES](#journal-sources) | [JOURNAL FIELDS](#journal-fields) |
|
||||
[PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [PERFORMANCE](#query-performance) |
|
||||
[CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq)
|
||||
|
||||
# SystemD Journal
|
||||
|
||||
The SystemD Journal plugin by Netdata makes viewing, exploring and analyzing systemd journal logs simple and efficient.
|
||||
It automatically discovers available journal sources, allows advanced filtering, offers interactive visual
|
||||
representations and supports exploring the logs of both individual servers and the logs on infrastructure wide
|
||||
journal centralization servers.
|
||||
|
||||
|
||||
|
||||
## Key features:
|
||||
|
||||
- Works on both **individual servers** and **journal centralization servers**.
|
||||
- Supports `persistent` and `volatile` journals.
|
||||
- Supports `system`, `user`, `namespaces` and `remote` journals.
|
||||
- Allows filtering on **any journal field** or **field value**, for any time-frame.
|
||||
- Allows **full text search** (`grep`) on all journal fields, for any time-frame.
|
||||
- Provides a **histogram** for log entries over time, with a break down per field-value, for any field and any time-frame.
|
||||
- Works directly on journal files, without any other third party components.
|
||||
- Supports coloring log entries, the same way `journalctl` does.
|
||||
- In PLAY mode provides the same experience as `journalctl -f`, showing new logs entries immediately after they are received.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
`systemd-journal.plugin` is a Netdata Function Plugin.
|
||||
|
||||
To protect your privacy, as with all Netdata Functions, a free Netdata Cloud user account is required to access it.
|
||||
|
||||
### Limitations:
|
||||
|
||||
- This plugin is not available when Netdata is installed in a container. The problem is that `libsystemd` is not available in Alpine Linux (there is a `libsystemd`, but it is a dummy that returns failure on all calls). We plan to change this, by shipping Netdata containers based on Debian.
|
||||
- For the same reason (lack of `systemd` support for Alpine Linux), the plugin is not available on `static` builds of Netdata (which are based on `muslc`, not `glibc`).
|
||||
|
||||
## Journal Sources
|
||||
|
||||
The plugin automatically detects the available journal sources, based on the journal files available in
|
||||
`/var/log/journal` (persistent logs) and `/run/log/journal` (volatile logs).
|
||||
|
||||
|
||||
|
||||
The plugin, by default, merges all journal sources together, to provide a unified view of all log messages available.
|
||||
|
||||
> To improve query performance, we recommend selecting the relevant journal source, before doing more analysis on the logs.
|
||||
|
||||
### `system` journals
|
||||
|
||||
These are the default journals available on all systems.
|
||||
|
||||
`system` journals contain:
|
||||
|
||||
- kernel log messages (via `kmsg`),
|
||||
- audit records, originating from the kernel audit subsystem,
|
||||
- messages received via `syslog`,
|
||||
- messages received via the standard output and error of service units,
|
||||
- structured messages received via the native journal API.
|
||||
|
||||
### `user` journals
|
||||
|
||||
By default, each user, with a UID outside the range of system users (0 - 999), dynamic service users,
|
||||
and the nobody user (65534), will get their own set of `user` journal files. For more information about
|
||||
this policy check [Users, Groups, UIDs and GIDs on systemd Systems](https://systemd.io/UIDS-GIDS/).
|
||||
|
||||
The plugin allows viewing, exploring and querying the journal files of all users.
|
||||
|
||||
### `namespaces` journals
|
||||
|
||||
Journal 'namespaces' are both a mechanism for logically isolating the log stream of projects consisting
|
||||
of one or more services from the rest of the system and a mechanism for improving performance. Systemd service
|
||||
units may be assigned to a specific journal namespace through the `LogNamespace=` unit file setting.
|
||||
|
||||
The plugin auto-detects the namespaces available and provides a list of all namespaces at the "sources" list on the UI.
|
||||
|
||||
### `remote` journals
|
||||
|
||||
Remote journals are created by `systemd-journal-remote`. This feature allows creating logs centralization points within
|
||||
your infrastructure.
|
||||
|
||||
Usually `remote` journals are named by the IP of the server sending these logs. The Netdata plugin automatically
|
||||
extracts these IPs and performs a reverse DNS lookup to find their hostnames. When this is successful,
|
||||
`remote` journals are named by the hostnames of the origin servers.
|
||||
|
||||
For information about configuring a journals' centralization server, check [this FAQ item](#how-do-i-configure-a-journals-centralization-server).
|
||||
|
||||
## Journal Fields
|
||||
|
||||
Fields found in the journal files are automatically added to the UI in multiple places to help you explore
|
||||
and filter the data.
|
||||
|
||||
The plugin automatically enriches certain fields to make them more user-friendly:
|
||||
|
||||
- `_BOOT_ID`: the hex value is annotated with the timestamp of the first message encountered for this boot id.
|
||||
- `PRIORITY`: the numeric value is replaced with the human-readable name of each priority.
|
||||
- `SYSLOG_FACILITY`: the encoded value is replaced with the human-readable name of each value.
|
||||
- `ERRNO`: the numeric value is annotated with the short name of each value.
|
||||
- `_UID` `_AUDIT_LOGINUID` and `_SYSTEMD_OWNER_UID`: the local user database is consulted to annotate them with usernames.
|
||||
- `_GID`: the local group database is consulted to annotate them with group names.
|
||||
- `_CAP_EFFECTIVE`: the encoded value is annotated with a human-readable list of the linux capabilities.
|
||||
- `_SOURCE_REALTIME_TIMESTAMP`: the numeric value is annotated with human-readable datetime in UTC.
|
||||
|
||||
The values of all other fields are presented as found in the journals.
|
||||
|
||||
> IMPORTANT:</br>
|
||||
> `_UID` `_AUDIT_LOGINUID`, `_SYSTEMD_OWNER_UID` and `_GID` annotations are added during presentation and are taken
|
||||
> from the server running the plugin. For `remote` sources, the names presented may not reflect the actual user and
|
||||
> group names on the origin server.
|
||||
|
||||
The annotations are not searchable with full text search. They are only added for the presentation of the fields.
|
||||
|
||||
### Journal fields as columns in the table
|
||||
|
||||
All journal fields available in the journal files are offered as columns on the UI. Use the gear button above the table:
|
||||
|
||||
|
||||
|
||||
### Journal fields as additional info to each log entry
|
||||
|
||||
When you click a log line, the sidebar, on the right of the screen, provides the full list of fields related to this
|
||||
log line. You can close this info sidebar, by selecting the filter icon at its top.
|
||||
|
||||
|
||||
|
||||
### Journal fields as filters
|
||||
|
||||
The plugin presents a select list of fields as filters to the query, with counters for each of the possible values
|
||||
for the field. This list can used to quickly check which fields and values are available for the entire time-frame
|
||||
of the query.
|
||||
|
||||
Internally the plugin has:
|
||||
|
||||
1. A white-list of fields, to be presented as filters.
|
||||
2. A black-list of fields, to prevent them from becoming filters. This list includes fields with a very high cardinality, like timestamps, unique message ids, etc. This is mainly for protecting the server's performance, to avoid building in memory indexes for the fields that almost each of their values is unique.
|
||||
|
||||
Keep in mind that the values presented in the filters, and their sorting is affected by the "full data queries"
|
||||
setting:
|
||||
|
||||
|
||||
|
||||
When "full data queries" is off, empty values are hidden and cannot be selected. This is due to a limitation of
|
||||
`libsystemd` that does not allow negative or empty matches. Also, values with zero counters may appear in the list.
|
||||
|
||||
When "full data queries" is on, Netdata is applying all filtering to the data (not `libsystemd`), but this means
|
||||
that all the data of the entire time-frame, without any filtering applied, have to be read by the plugin to prepare
|
||||
the response required.
|
||||
|
||||
### Journal fields as histogram sources
|
||||
|
||||
The plugin presents a histogram of the number of log entries across time.
|
||||
|
||||
The data source of this histogram can be any of the fields that are available as filters.
|
||||
For each of the values this field has, across the entire time-frame of the query, the histogram will get corresponding
|
||||
dimensions, showing the number of log entries, per value, over time.
|
||||
|
||||
The granularity of the histogram is adjusted automatically to have about 150 columns visible on screen.
|
||||
|
||||
The histogram presented by the plugin is interactive:
|
||||
|
||||
- **Zoom**, either with the global date-time picker, or the zoom tool in the histogram's toolbox.
|
||||
- **Pan**, either with global date-time picker, or by dragging with the mouse the chart to the left or the right.
|
||||
- **Click**, to quickly jump to the highlighted point in time in the log entries.
|
||||
|
||||
|
||||
|
||||
## PLAY mode
|
||||
|
||||
The plugin supports PLAY mode, to continuously update the screen with new log entries found in the journal files.
|
||||
|
||||
On centralized log servers, this provides a unified view of all the logs encountered across the entire infrastructure.
|
||||
|
||||
## Full-text search
|
||||
|
||||
The plugin supports searching for any text on all fields of the log entries.
|
||||
|
||||
Full text search is combined with the selected filters.
|
||||
|
||||
## Query performance
|
||||
|
||||
Journal files are designed to be accessed by multiple readers and one writer, concurrently.
|
||||
|
||||
Readers (like this Netdata plugin), open the journal files and `libsystemd`, behind the scenes, maps regions
|
||||
of the files into memory, to satisfy each query.
|
||||
|
||||
On logs aggregation servers, the performance of the queries depend on the following factors:
|
||||
|
||||
1. The number of files involved in each query. This is why we suggest to select a source when possible.
|
||||
2. The speed of the disks hosting the journal files. Journal files perform a lot of reading while querying, so the fastest the disks, the faster the query will finish.
|
||||
3. The memory available for caching parts of the files. Increased memory will help the kernel cache the most frequently used parts of the journal files, avoiding disk I/O and speeding up queries.
|
||||
4. The number of filters applied. Queries are significantly faster when just a few filters are selected.
|
||||
|
||||
In general, for a faster experience, keep a low number of rows within the visible timeframe.
|
||||
|
||||
Even on long timeframes, selecting a couple of filters that will result in a few dozen thousand log entries
|
||||
will provide fast / rapid responses, usually less than a second. To the contrary, viewing timeframes with millions
|
||||
of entries may result in longer delays.
|
||||
|
||||
The plugin aborts journal queries when your browser cancels inflight requests. This allows you to work on the UI
|
||||
while there are background queries running.
|
||||
|
||||
At the time of this writing, this Netdata plugin is about 25-30 times faster than `journalctl` on queries that access
|
||||
multiple journal files, over long time-frames.
|
||||
|
||||
During the development of this plugin, we submitted, to `systemd`, a number of patches to improve `journalctl`
|
||||
performance by a factor of 14:
|
||||
|
||||
- https://github.com/systemd/systemd/pull/29365
|
||||
- https://github.com/systemd/systemd/pull/29366
|
||||
- https://github.com/systemd/systemd/pull/29261
|
||||
|
||||
However, even after these patches are merged, `journalctl` will still be 2x slower than this Netdata plugin,
|
||||
on multi-journal queries.
|
||||
|
||||
The problem lies in the way `libsystemd` handles multi-journal file queries. To overcome this problem,
|
||||
the Netdata plugin queries each file individually and it then it merges the results to be returned.
|
||||
This is transparent, thanks to the `facets` library in `libnetdata` that handles on-the-fly indexing, filtering,
|
||||
and searching of any dataset, independently of its source.
|
||||
|
||||
## Configuration and maintenance
|
||||
|
||||
This Netdata plugin does not require any configuration or maintenance.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Can I use this plugin on journals' centralization servers?
|
||||
|
||||
Yes. You can centralize your logs using systemd journal, and then install Netdata
|
||||
on this logs centralization server to explore the logs of all your infrastructure.
|
||||
|
||||
This plugin will automatically provide multi-node views of your logs and also give you the ability to combine the logs
|
||||
of multiple servers, as you see fit.
|
||||
|
||||
Check [configuring a logs centralization server](#configuring-a-journals-centralization-server).
|
||||
|
||||
### Can I use this plugin from a parent Netdata?
|
||||
|
||||
Yes. When your nodes are connected to a Netdata parent, all their functions are available
|
||||
via the parent's UI. So, from the parent UI, you can access the functions of all your nodes.
|
||||
|
||||
Keep in mind that to protect your privacy, in order to access Netdata functions, you need a
|
||||
free Netdata Cloud account.
|
||||
|
||||
### Is any of my data exposed to Netdata Cloud from this plugin?
|
||||
|
||||
No. When you access the agent directly, none of your data passes through Netdata Cloud.
|
||||
You need a free Netdata Cloud account only to verify your identity and enable the use of
|
||||
Netdata Functions. Once this is done, all the data flow directly from your Netdata agent
|
||||
to your web browser.
|
||||
|
||||
When you access Netdata via `https://app.netdata.cloud`, your data travel via Netdata Cloud,
|
||||
but they are not stored in Netdata Cloud. This is to allow you access your Netdata agents from
|
||||
anywhere. All communication from/to Netdata Cloud is encrypted.
|
||||
|
||||
### What are `volatile` and `persistent` journals?
|
||||
|
||||
SystemD JournalD allows creating both `volatile` journals in a `tmpfs` ram drive,
|
||||
and `persistent` journals stored on disk.
|
||||
|
||||
`volatile` journals are particularly useful when the system monitored is sensitive to
|
||||
disk I/O, or does not have any writable disks at all.
|
||||
|
||||
For more information check `man systemd-journald`.
|
||||
|
||||
### Is it worth to build a systemd logs centralization server?
|
||||
|
||||
Yes. It is simple, fast and the software to do it is already in your systems.
|
||||
|
||||
For application and system logs, systemd journal is ideal and the visibility you can get
|
||||
by centralizing your system logs and the use of this Netdata plugin, is unparalleled.
|
||||
|
||||
### How do I configure a journals' centralization server?
|
||||
|
||||
A short summary to get journal server running can be found below.
|
||||
|
||||
For more options and reference to documentation, check `man systemd-journal-remote` and `man systemd-journal-upload`.
|
||||
|
||||
#### Configuring a journals' centralization server
|
||||
|
||||
On the centralization server install `systemd-journal-remote`, and enable it with `systemctl`, like this:
|
||||
|
||||
```sh
|
||||
# change this according to your distro
|
||||
sudo apt-get install systemd-journal-remote
|
||||
|
||||
# enable receiving
|
||||
sudo systemctl enable --now systemd-journal-remote.socket
|
||||
sudo systemctl enable systemd-journal-remote.service
|
||||
```
|
||||
|
||||
`systemd-journal-remote` is now listening for incoming journals from remote hosts, on port `19532`.
|
||||
Please note that `systemd-journal-remote` supports using secure connections.
|
||||
To learn more run `man systemd-journal-remote`.
|
||||
|
||||
To change the protocol of the journal transfer (HTTP/HTTPS) and the save location, do:
|
||||
|
||||
```sh
|
||||
# copy the service file
|
||||
sudo cp /lib/systemd/system/systemd-journal-remote.service /etc/systemd/system/
|
||||
|
||||
# edit it
|
||||
# --listen-http=-3 specifies the incoming journal for http.
|
||||
# If you want to use https, change it to --listen-https=-3.
|
||||
nano /etc/systemd/system/systemd-journal-remote.service
|
||||
|
||||
# reload systemd
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
To change the port, copy `/lib/systemd/system/systemd-journal-remote.socket` to `/etc/systemd/system/` and edit it.
|
||||
Then do `sudo systemctrl daemon-reload`
|
||||
|
||||
|
||||
#### Configuring journal clients to push their logs to the server
|
||||
|
||||
On the clients you want to centralize their logs, install `systemd-journal-remote`, configure `systemd-journal-upload`, enable it and start it with `systemctl`.
|
||||
|
||||
To install it run:
|
||||
|
||||
```sh
|
||||
# change this according to your distro
|
||||
sudo apt-get install systemd-journal-remote
|
||||
```
|
||||
|
||||
Then, edit `/etc/systemd/journal-upload.conf` and set the IP address and the port of the server, like this:
|
||||
|
||||
```
|
||||
[Upload]
|
||||
URL=http://centralization.server.ip:19532
|
||||
```
|
||||
|
||||
Remember to match the protocol (http/https) the server expects.
|
||||
|
||||
Finally, enable and start `systemd-journal-upload`, like this:
|
||||
|
||||
```sh
|
||||
sudo systemctl enable systemd-journal-upload
|
||||
sudo systemctl start systemd-journal-upload
|
||||
```
|
||||
|
||||
Keep in mind that immediately after starting `systemd-journal-upload` on a server, a replication process starts pushing logs in the order they have been received. This means that depending on the size of the available logs, some time may be needed for Netdata to show the most recent logs of that server.
|
||||
|
||||
#### Limitations when using a logs centralization server
|
||||
|
||||
As of this writing `namespaces` support by systemd is limited:
|
||||
|
||||
- Docker containers cannot log to namespaces. Check [this issue](https://github.com/moby/moby/issues/41879).
|
||||
- `systemd-journal-upload` automatically uploads `system` and `user` journals, but not `namespaces` journals. For this you need to spawn a `systemd-journal-upload` per namespace.
|
||||
|
||||
File diff suppressed because it is too large
Load diff
10
configure.ac
10
configure.ac
|
|
@ -1141,6 +1141,16 @@ fi
|
|||
AC_MSG_RESULT([${enable_plugin_systemd_journal}])
|
||||
AM_CONDITIONAL([ENABLE_PLUGIN_SYSTEMD_JOURNAL], [test "${enable_plugin_systemd_journal}" = "yes"])
|
||||
|
||||
AC_CHECK_LIB([systemd], [sd_journal_open_files_fd], [have_sd_journal_open_files_fd=yes], [have_sd_journal_open_files_fd=no])
|
||||
if test "${have_sd_journal_open_files_fd}" = "yes"; then
|
||||
AC_DEFINE([HAVE_SD_JOURNAL_OPEN_FILES_FD], [1], [sd_journal_open_files_fd usability])
|
||||
fi
|
||||
|
||||
AC_CHECK_LIB([systemd], [sd_journal_restart_fields], [have_sd_journal_restart_fields=yes], [have_sd_journal_restart_fields=no])
|
||||
if test "${have_sd_journal_restart_fields}" = "yes"; then
|
||||
AC_DEFINE([HAVE_SD_JOURNAL_RESTART_FIELDS], [1], [sd_journal_restart_fields usability])
|
||||
fi
|
||||
|
||||
AC_MSG_NOTICE([OPTIONAL_SYSTEMD_LIBS is set to: ${OPTIONAL_SYSTEMD_LIBS}])
|
||||
|
||||
if test "${enable_plugin_systemd_journal}" = "yes"; then
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -12,6 +12,14 @@ typedef enum __attribute__((packed)) {
|
|||
FACETS_ANCHOR_DIRECTION_BACKWARD,
|
||||
} FACETS_ANCHOR_DIRECTION;
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
FACETS_TRANSFORM_VALUE,
|
||||
FACETS_TRANSFORM_HISTOGRAM,
|
||||
FACETS_TRANSFORM_FACET,
|
||||
FACETS_TRANSFORM_DATA,
|
||||
FACETS_TRANSFORM_FACET_SORT,
|
||||
} FACETS_TRANSFORMATION_SCOPE;
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
FACET_KEY_OPTION_FACET = (1 << 0), // filterable values
|
||||
FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value
|
||||
|
|
@ -22,6 +30,7 @@ typedef enum __attribute__((packed)) {
|
|||
FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap
|
||||
FACET_KEY_OPTION_RICH_TEXT = (1 << 7),
|
||||
FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter
|
||||
FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 9), // when registering the transformation, do it only at the view, not on all data
|
||||
} FACET_KEY_OPTIONS;
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
|
|
@ -48,17 +57,22 @@ typedef struct facet_row {
|
|||
typedef struct facets FACETS;
|
||||
typedef struct facet_key FACET_KEY;
|
||||
|
||||
typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data);
|
||||
typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
|
||||
typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
|
||||
typedef FACET_ROW_SEVERITY (*facet_row_severity_t)(FACETS *facets, FACET_ROW *row, void *data);
|
||||
FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data);
|
||||
FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data);
|
||||
void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data);
|
||||
|
||||
typedef enum __attribute__((packed)) {
|
||||
FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table
|
||||
FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
|
||||
FACETS_OPTION_DISABLE_ALL_FACETS = (1 << 2),
|
||||
FACETS_OPTION_DISABLE_HISTOGRAM = (1 << 3),
|
||||
FACETS_OPTION_DATA_ONLY = (1 << 4),
|
||||
FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets should be visible by default in the table
|
||||
FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
|
||||
FACETS_OPTION_DONT_SEND_FACETS = (1 << 2), // "facets" object will not be included in the report
|
||||
FACETS_OPTION_DONT_SEND_HISTOGRAM = (1 << 3), // "histogram" object will not be included in the report
|
||||
FACETS_OPTION_DATA_ONLY = (1 << 4),
|
||||
FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report
|
||||
FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6),
|
||||
FACETS_OPTION_SHOW_DELTAS = (1 << 7),
|
||||
} FACETS_OPTIONS;
|
||||
|
||||
FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);
|
||||
|
|
@ -67,23 +81,37 @@ void facets_destroy(FACETS *facets);
|
|||
void facets_accepted_param(FACETS *facets, const char *param);
|
||||
|
||||
void facets_rows_begin(FACETS *facets);
|
||||
void facets_row_finished(FACETS *facets, usec_t usec);
|
||||
bool facets_row_finished(FACETS *facets, usec_t usec);
|
||||
|
||||
FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options);
|
||||
void facets_set_query(FACETS *facets, const char *query);
|
||||
void facets_set_items(FACETS *facets, uint32_t items);
|
||||
void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction);
|
||||
void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction);
|
||||
void facets_enable_slice_mode(FACETS *facets);
|
||||
|
||||
FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options);
|
||||
void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options);
|
||||
void facets_set_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut);
|
||||
void facets_set_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut);
|
||||
void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut);
|
||||
void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut);
|
||||
|
||||
void facets_add_key_value(FACETS *facets, const char *key, const char *value);
|
||||
void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len);
|
||||
|
||||
void facets_report(FACETS *facets, BUFFER *wb);
|
||||
void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry);
|
||||
void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys);
|
||||
void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity);
|
||||
void facets_data_only_mode(FACETS *facets);
|
||||
void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options);
|
||||
|
||||
bool facets_key_name_is_filter(FACETS *facets, const char *key);
|
||||
bool facets_key_name_is_facet(FACETS *facets, const char *key);
|
||||
bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length);
|
||||
void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length);
|
||||
|
||||
void facets_sort_and_reorder_keys(FACETS *facets);
|
||||
usec_t facets_row_oldest_ut(FACETS *facets);
|
||||
usec_t facets_row_newest_ut(FACETS *facets);
|
||||
uint32_t facets_rows(FACETS *facets);
|
||||
|
||||
void facets_table_config(BUFFER *wb);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -10,6 +10,40 @@
|
|||
|
||||
#include "../libnetdata.h"
|
||||
|
||||
bool ip_to_hostname(const char *ip, char *dst, size_t dst_len) {
|
||||
if(!dst || !dst_len)
|
||||
return false;
|
||||
|
||||
struct sockaddr_in sa;
|
||||
struct sockaddr_in6 sa6;
|
||||
struct sockaddr *sa_ptr;
|
||||
int sa_len;
|
||||
|
||||
// Try to convert the IP address to sockaddr_in (IPv4)
|
||||
if (inet_pton(AF_INET, ip, &(sa.sin_addr)) == 1) {
|
||||
sa.sin_family = AF_INET;
|
||||
sa_ptr = (struct sockaddr *)&sa;
|
||||
sa_len = sizeof(sa);
|
||||
}
|
||||
// Try to convert the IP address to sockaddr_in6 (IPv6)
|
||||
else if (inet_pton(AF_INET6, ip, &(sa6.sin6_addr)) == 1) {
|
||||
sa6.sin6_family = AF_INET6;
|
||||
sa_ptr = (struct sockaddr *)&sa6;
|
||||
sa_len = sizeof(sa6);
|
||||
}
|
||||
|
||||
else {
|
||||
dst[0] = '\0';
|
||||
return false;
|
||||
}
|
||||
|
||||
// Perform the reverse lookup
|
||||
int res = getnameinfo(sa_ptr, sa_len, dst, dst_len, NULL, 0, NI_NAMEREQD);
|
||||
if(res != 0)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
SOCKET_PEERS socket_peers(int sock_fd) {
|
||||
SOCKET_PEERS peers;
|
||||
|
|
|
|||
|
|
@ -243,5 +243,6 @@ typedef struct socket_peers {
|
|||
} SOCKET_PEERS;
|
||||
|
||||
SOCKET_PEERS socket_peers(int sock_fd);
|
||||
bool ip_to_hostname(const char *ip, char *dst, size_t dst_len);
|
||||
|
||||
#endif //NETDATA_SOCKET_H
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue