detect the system ca bundle at runtime (#19622)

* detect the system ca bundle at runtime

* minor fix

* fix for older libcurl versions

* added X509_get_default_cert_file()

* added validation for the certificates

* moved ssl/curl code to separate file; now it configured both libcurl and openssl; added defaults to libcurl static install

* run the new code only in netdata static builds

* auto to check

* disable runtime ssl checks

CMakeLists.txt

@@ -1182,6 +1182,8 @@ set(DAEMON_FILES
         src/daemon/pulse/pulse-db-dbengine-retention.h
         src/daemon/pulse/pulse-parents.c
         src/daemon/pulse/pulse-parents.h
+        src/daemon/config/netdata-conf-ssl.c
+        src/daemon/config/netdata-conf-ssl.h
 )
 
 set(H2O_FILES

packaging/makeself/install-or-update.sh

@@ -27,7 +27,7 @@ fi
 
 STARTIT=1
 REINSTALL_OPTIONS=""
-NETDATA_CERT_MODE="${NETDATA_CERT_MODE:-auto}"
+NETDATA_CERT_MODE="${NETDATA_CERT_MODE:-check}"
 NETDATA_CERT_TEST_URL="${NETDATA_CERT_TEST_URL:-https://app.netdata.cloud}"
 RELEASE_CHANNEL="nightly"
 

packaging/makeself/jobs/50-curl.install.sh

@@ -57,6 +57,8 @@ if [ "${CACHE_HIT:-0}" -eq 0 ]; then
         --enable-cookies \
         --with-ca-fallback \
         --with-openssl \
+        --with-ca-bundle=/opt/netdata/etc/ssl/certs/ca-certificates.crt \
+        --with-ca-path=/opt/netdata/etc/ssl/certs \
         --disable-dependency-tracking
 
     # Curl autoconf does not honour the curl_LDFLAGS environment variable

packaging/makeself/jobs/70-netdata-git.install.sh

@@ -7,9 +7,9 @@
 cd "${NETDATA_SOURCE_PATH}" || exit 1
 
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
-  export CFLAGS="${TUNING_FLAGS} -ffunction-sections -fdata-sections -static -O2 -funroll-loops -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl -pipe"
+  export CFLAGS="${TUNING_FLAGS} -ffunction-sections -fdata-sections -static -O2 -funroll-loops -DNETDATA_STATIC_BUILD=1 -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl -pipe"
 else
-  export CFLAGS="${TUNING_FLAGS} -static -O1 -pipe -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_INTERNAL_CHECKS=1 -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl"
+  export CFLAGS="${TUNING_FLAGS} -static -O1 -pipe -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_STATIC_BUILD=1 -DNETDATA_INTERNAL_CHECKS=1 -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl"
 fi
 
 export LDFLAGS="-Wl,--gc-sections -static -L/openssl-static/lib64 -L/libnetfilter-acct-static/lib -lnetfilter_acct -L/usr/lib -lmnl -L/usr/lib -lzstd -L/curl-local/lib"

src/daemon/config/netdata-conf-ssl.c

@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "daemon/common.h"
+#include "netdata-conf-ssl.h"
+#include <curl/curl.h>
+
+static bool is_ca_bundle_valid(const char *ca_path) {
+    if (!ca_path || !*ca_path)
+        return false;
+
+    FILE *fp = fopen(ca_path, "r");
+    if (!fp)
+        return false;
+
+    ERR_clear_error();
+
+    int valid_certs = 0;
+    X509 *cert = NULL;
+    while ((cert = PEM_read_X509(fp, NULL, NULL, NULL)) != NULL) {
+        ASN1_TIME *not_after = X509_get_notAfter(cert);
+        int day, sec;
+
+        if (ASN1_TIME_diff(&day, &sec, NULL, not_after) == 1) {
+            if (day > 0 || (day == 0 && sec > 0)) {
+                valid_certs++;
+
+                // we found 1 valid, stop reading the file
+                X509_free(cert);
+                break;
+            }
+        }
+
+        X509_free(cert);
+    }
+
+    fclose(fp);
+
+    ERR_clear_error();
+    return valid_certs > 0;
+}
+
+const char *detect_libcurl_default_ca() {
+#if LIBCURL_VERSION_NUM >= 0x074600  // 7.70.0 (CURLVERSION_SEVENTH)
+    curl_version_info_data *info = curl_version_info(CURLVERSION_NOW);
+    if (info) {
+        // Check built-in CA bundle
+        if (info->cainfo &&
+            access(info->cainfo, R_OK) == 0 &&
+            is_ca_bundle_valid(info->cainfo))
+            return info->cainfo;
+    }
+#endif
+    return NULL;
+}
+
+static const char *detect_ca_path(void) {
+    static const char *paths[] = {
+        "/etc/ssl/certs/ca-certificates.crt",               // Debian, Ubuntu, Arch
+        "/etc/pki/tls/certs/ca-bundle.crt",                 // RHEL, CentOS, Fedora
+        "/etc/ssl/ca-bundle.pem",                           // OpenSUSE
+        "/etc/ssl/cert.pem",                                // Alpine
+        "/opt/netdata/etc/ssl/certs/ca-certificates.crt",   // Netdata static build
+        "/opt/netdata/share/ssl/certs/ca-certificates.crt", // Netdata static build - fallback
+        NULL
+    };
+
+    for (int i = 0; paths[i] != NULL; i++) {
+        if (access(paths[i], R_OK) == 0 &&
+            is_ca_bundle_valid(paths[i]))
+            return paths[i];
+    }
+
+    return NULL;
+}
+
+void netdata_conf_ssl(void) {
+    netdata_ssl_initialize_openssl();
+
+#if 0
+    const char *p = getenv("CURL_CA_BUNDLE");
+    if(!p || !*p) p = getenv("SSL_CERT_FILE");
+    if(!p || !*p) {
+        p = X509_get_default_cert_file();
+        if(!p || !*p || !is_ca_bundle_valid(p))
+            p = NULL;
+    }
+    if(!p || !*p) p = detect_libcurl_default_ca();
+    if(!p || !*p) p = detect_ca_path();
+    setenv("CURL_CA_BUNDLE", inicfg_get(&netdata_config, CONFIG_SECTION_ENV_VARS, "CURL_CA_BUNDLE", p ? p : ""), 1);
+    setenv("SSL_CERT_FILE", inicfg_get(&netdata_config, CONFIG_SECTION_ENV_VARS, "SSL_CERT_FILE", p ? p : ""), 1);
+#endif
+
+}

src/daemon/config/netdata-conf-ssl.h

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_NETDATA_CONF_SSL_H
+#define NETDATA_NETDATA_CONF_SSL_H
+
+void netdata_conf_ssl(void);
+
+#endif //NETDATA_NETDATA_CONF_SSL_H

src/daemon/config/netdata-conf-web.c

@@ -159,7 +159,5 @@ void netdata_conf_web_security_init(void) {
 
     tls_version    = inicfg_get(&netdata_config, CONFIG_SECTION_WEB, "tls version",  "1.3");
     tls_ciphers    = inicfg_get(&netdata_config, CONFIG_SECTION_WEB, "tls ciphers",  "none");
-
-    netdata_ssl_initialize_openssl();
 }
 

src/daemon/config/netdata-conf.h

@@ -16,5 +16,6 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user);
 #include "netdata-conf-logs.h"
 #include "netdata-conf-web.h"
 #include "netdata-conf-cloud.h"
+#include "netdata-conf-ssl.h"
 
 #endif //NETDATA_DAEMON_NETDATA_CONF_H

src/daemon/environment.c

@@ -81,7 +81,8 @@ void set_environment_for_plugins_and_scripts(void) {
         freez((char *)default_port);
 
     // set the path we need
-    char path[4096], *p = getenv("PATH");
+    char path[4096];
+    const char *p = getenv("PATH");
     if (!p) p = "/bin:/usr/bin";
     snprintfz(path, sizeof(path), "%s:%s", p, "/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin");
     setenv("PATH", inicfg_get(&netdata_config, CONFIG_SECTION_ENV_VARS, "PATH", path), 1);

src/daemon/main.c

@@ -762,6 +762,7 @@ int netdata_main(int argc, char **argv) {
     // ----------------------------------------------------------------------------------------------------------------
     // global configuration
 
+    netdata_conf_ssl();
     netdata_conf_section_global();
 
     // Get execution path before switching user to avoid permission issues