Improve context load on startup (#16203)

* Retrieve last connected timestamp from the database (host->last_connected)

* Improve context load performance
Check for agent shutdown while context load in progress
Log information about host load start and finish

* Remove check for slot as it will only reach this part when a slot is found

database/sqlite/sqlite_aclk.c

@@ -84,6 +84,7 @@ enum {
     IDX_PROGRAM_VERSION,
     IDX_ENTRIES,
     IDX_HEALTH_ENABLED,
+    IDX_LAST_CONNECTED,
 };
 
 static int create_host_callback(void *data, int argc, char **argv, char **column)
@@ -127,8 +128,10 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
         , system_info
         , 1
     );
-    if (likely(host))
+    if (likely(host)) {
         host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]);
+        host->last_connected = (time_t) (argv[IDX_LAST_CONNECTED] ? str2uint64_t(argv[IDX_LAST_CONNECTED], NULL) : 0);
+    }
 
     (*number_of_chidren)++;
 
@@ -524,7 +527,7 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may
 
 #define SQL_FETCH_ALL_HOSTS "SELECT host_id, hostname, registry_hostname, update_every, os, " \
     "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \
-    "program_version, entries, health_enabled FROM host WHERE hops >0;"
+    "program_version, entries, health_enabled, last_connected FROM host WHERE hops >0;"
 
 #define SQL_FETCH_ALL_INSTANCES "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \
                                 "WHERE h.host_id = ni.host_id AND ni.node_id IS NOT NULL; "

database/sqlite/sqlite_context.c

@@ -92,7 +92,7 @@ void sql_close_context_database(void)
 // Fetching data
 //
 #define CTX_GET_CHART_LIST  "SELECT c.chart_id, c.type||'.'||c.id, c.name, c.context, c.title, c.unit, c.priority, " \
-        "c.update_every, c.chart_type, c.family FROM meta.chart c WHERE c.host_id = @host_id and c.chart_id is not null; "
+        "c.update_every, c.chart_type, c.family FROM chart c WHERE c.host_id = @host_id and c.chart_id is not null; "
 
 void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data)
 {
@@ -105,7 +105,7 @@ void ctx_get_chart_list(uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, voi
     }
 
     if (unlikely(!res)) {
-        rc = prepare_statement(db_context_meta, CTX_GET_CHART_LIST, &res);
+        rc = prepare_statement(db_meta, CTX_GET_CHART_LIST, &res);
         if (rc != SQLITE_OK) {
             error_report("Failed to prepare statement to fetch chart list");
             return;
@@ -141,14 +141,14 @@ skip_load:
 
 // Dimension list
 #define CTX_GET_DIMENSION_LIST  "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \
-        "FROM meta.dimension d WHERE d.chart_id = @id and d.dim_id is not null ORDER BY d.rowid ASC;"
+        "FROM dimension d WHERE d.chart_id = @id and d.dim_id is not null ORDER BY d.rowid ASC;"
 void ctx_get_dimension_list(uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
 {
     int rc;
     static __thread sqlite3_stmt *res = NULL;
 
     if (unlikely(!res)) {
-        rc = prepare_statement(db_context_meta, CTX_GET_DIMENSION_LIST, &res);
+        rc = prepare_statement(db_meta, CTX_GET_DIMENSION_LIST, &res);
         if (rc != SQLITE_OK) {
             error_report("Failed to prepare statement to fetch chart dimension data");
             return;

database/sqlite/sqlite_metadata.c

@@ -1270,7 +1270,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
     register_libuv_worker_jobs();
 
     struct scan_metadata_payload *data = req->data;
-    UNUSED(data);
+    struct metadata_wc *wc = data->wc;
 
     worker_is_busy(UV_EVENT_HOST_CONTEXT_LOAD);
     usec_t started_ut = now_monotonic_usec(); (void)started_ut;
@@ -1278,6 +1278,7 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
     RRDHOST *host;
 
     size_t max_threads = MIN(get_netdata_cpus() / 2, 6);
+    netdata_log_info("METADATA: Using %zu threads for context loading", max_threads);
     struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));
 
     size_t thread_index;
@@ -1289,25 +1290,28 @@ static void start_all_host_load_context(uv_work_t *req __maybe_unused)
        rrdhost_flag_set(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);
        internal_error(true, "METADATA: 'host:%s' loading context", rrdhost_hostname(host));
 
-       cleanup_finished_threads(hclt, max_threads, false);
+       bool found_slot = false;
-       bool found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);
+       do {
+           if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
+                break;
 
-       if (unlikely(!found_slot)) {
+           cleanup_finished_threads(hclt, max_threads, false);
-           struct host_context_load_thread hclt_sync = {.host = host};
+           found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);
-           restore_host_context(&hclt_sync);
+       } while (!found_slot);
-       }
+
-       else {
+       if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
-           __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
+           break;
-           hclt[thread_index].host = host;
+
-           assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));
+       __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
-       }
+       hclt[thread_index].host = host;
+       assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));
     }
     dfe_done(host);
 
     cleanup_finished_threads(hclt, max_threads, true);
     freez(hclt);
     usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
-    internal_error(true, "METADATA: 'host:ALL' contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);
+    netdata_log_info("METADATA: host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);
 
     worker_is_idle();
 }
@@ -1905,6 +1909,7 @@ void metadata_queue_load_host_context(RRDHOST *host)
     if (unlikely(!metasync_worker.loop))
         return;
     queue_metadata_cmd(METADATA_LOAD_HOST_CONTEXT, host, NULL);
+    netdata_log_info("Queued command to load host contexts");
 }
 
 //