0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-16 10:31:07 +00:00

Convert the ML database ()

* Convert a db to WAL with auto vacuum

* Use single sqlite configuration function

* Remove UNUSED statements
This commit is contained in:
Stelios Fragkakis 2023-09-28 19:40:02 +03:00 committed by GitHub
parent 5ff2ec1a29
commit f90c2a23e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 147 additions and 96 deletions

View file

@ -52,39 +52,8 @@ int sql_init_context_database(int memory)
if (likely(!memory))
target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION);
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
if(init_database_batch(db_context_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_synchronous
// PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
if(init_database_batch(db_context_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_mode
// PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
if(init_database_batch(db_context_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_temp_store
// PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
if(init_database_batch(db_context_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_size_limit
// PRAGMA schema.journal_size_limit = N ;
snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
if(init_database_batch(db_context_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_cache_size
// PRAGMA schema.cache_size = pages;
// PRAGMA schema.cache_size = -kibibytes;
snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
if(init_database_batch(db_context_meta, list)) return 1;
snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
if(init_database_batch(db_context_meta, list)) return 1;
if (configure_sqlite_database(db_context_meta, target_version))
return 1;
if (likely(!memory))
snprintfz(buf, 1024, "ATTACH DATABASE \"%s/netdata-meta.db\" as meta;", netdata_configured_cache_dir);

View file

@ -11,6 +11,24 @@ static int return_int_cb(void *data, int argc, char **argv, char **column)
return 0;
}
static int get_auto_vaccum(sqlite3 *database)
{
char *err_msg = NULL;
char sql[128];
int exists = 0;
snprintf(sql, 127, "PRAGMA auto_vacuum");
int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
if (rc != SQLITE_OK) {
netdata_log_info("Error checking database auto vacuum setting; %s", err_msg);
sqlite3_free(err_msg);
}
return exists;
}
int table_exists_in_database(const char *table)
{
char *err_msg = NULL;
@ -111,7 +129,6 @@ const char *database_migrate_v13_v14[] = {
static int do_migration_v1_v2(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running \"%s\" database migration", name);
if (table_exists_in_database("host") && !column_exists_in_table("host", "hops"))
@ -121,7 +138,6 @@ static int do_migration_v1_v2(sqlite3 *database, const char *name)
static int do_migration_v2_v3(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running \"%s\" database migration", name);
if (table_exists_in_database("host") && !column_exists_in_table("host", "memory_mode"))
@ -131,7 +147,6 @@ static int do_migration_v2_v3(sqlite3 *database, const char *name)
static int do_migration_v3_v4(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running database migration %s", name);
char sql[256];
@ -163,7 +178,6 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
static int do_migration_v4_v5(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running \"%s\" database migration", name);
return init_database_batch(database, &database_migrate_v4_v5[0]);
@ -171,7 +185,6 @@ static int do_migration_v4_v5(sqlite3 *database, const char *name)
static int do_migration_v5_v6(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running \"%s\" database migration", name);
return init_database_batch(database, &database_migrate_v5_v6[0]);
@ -179,7 +192,6 @@ static int do_migration_v5_v6(sqlite3 *database, const char *name)
static int do_migration_v6_v7(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running \"%s\" database migration", name);
char sql[256];
@ -213,7 +225,6 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
static int do_migration_v7_v8(sqlite3 *database, const char *name)
{
UNUSED(name);
netdata_log_info("Running database migration %s", name);
char sql[256];
@ -381,10 +392,27 @@ static int do_migration_v13_v14(sqlite3 *database, const char *name)
}
// Actions for ML migration
const char *database_ml_migrate_v1_v2[] = {
"PRAGMA journal_mode=delete",
"PRAGMA journal_mode=WAL",
"PRAGMA auto_vacuum=2",
"VACUUM",
NULL
};
static int do_ml_migration_v1_v2(sqlite3 *database, const char *name)
{
netdata_log_info("Running \"%s\" database migration", name);
if (get_auto_vaccum(database) != 2)
return init_database_batch(database, &database_ml_migrate_v1_v2[0]);
return 0;
}
static int do_migration_noop(sqlite3 *database, const char *name)
{
UNUSED(database);
UNUSED(name);
netdata_log_info("Running database migration %s", name);
return 0;
}
@ -448,6 +476,12 @@ DATABASE_FUNC_MIGRATION_LIST context_migration_action[] = {
{.name = NULL, .func = NULL}
};
DATABASE_FUNC_MIGRATION_LIST ml_migration_action[] = {
{.name = "v0 to v1", .func = do_migration_noop},
{.name = "v1 to v2", .func = do_ml_migration_v1_v2},
// the terminator of this array
{.name = NULL, .func = NULL}
};
int perform_database_migration(sqlite3 *database, int target_version)
{
@ -458,3 +492,8 @@ int perform_context_database_migration(sqlite3 *database, int target_version)
{
return migrate_database(database, target_version, "context", context_migration_action);
}
int perform_ml_database_migration(sqlite3 *database, int target_version)
{
return migrate_database(database, target_version, "ml", ml_migration_action);
}

View file

@ -9,5 +9,6 @@
int perform_database_migration(sqlite3 *database, int target_version);
int perform_context_database_migration(sqlite3 *database, int target_version);
int table_exists_in_database(const char *table);
int perform_ml_database_migration(sqlite3 *database, int target_version);
#endif //NETDATA_SQLITE_DB_MIGRATION_H

View file

@ -201,6 +201,55 @@ int execute_insert(sqlite3_stmt *res)
return rc;
}
int configure_sqlite_database(sqlite3 *database, int target_version)
{
char buf[1024 + 1] = "";
const char *list[2] = { buf, NULL };
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
if (init_database_batch(database, list))
return 1;
// https://www.sqlite.org/pragma.html#pragma_synchronous
// PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
if (init_database_batch(database, list))
return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_mode
// PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
if (init_database_batch(database, list))
return 1;
// https://www.sqlite.org/pragma.html#pragma_temp_store
// PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
if (init_database_batch(database, list))
return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_size_limit
// PRAGMA schema.journal_size_limit = N ;
snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
if (init_database_batch(database, list))
return 1;
// https://www.sqlite.org/pragma.html#pragma_cache_size
// PRAGMA schema.cache_size = pages;
// PRAGMA schema.cache_size = -kibibytes;
snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
if (init_database_batch(database, list))
return 1;
snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
if (init_database_batch(database, list))
return 1;
return 0;
}
#define MAX_OPEN_STATEMENTS (512)
static void add_stmt_to_list(sqlite3_stmt *res)
@ -382,9 +431,6 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
netdata_log_info("SQLite database %s initialization", sqlite_database);
char buf[1024 + 1] = "";
const char *list[2] = { buf, NULL };
rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
if (unlikely(rc != SQLITE_OK))
error_report("Failed to register internal u2h function");
@ -402,39 +448,8 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
if (likely(!memory))
target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
// PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
if(init_database_batch(db_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_synchronous
// PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
if(init_database_batch(db_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_mode
// PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
if(init_database_batch(db_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_temp_store
// PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
if(init_database_batch(db_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_journal_size_limit
// PRAGMA schema.journal_size_limit = N ;
snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
if(init_database_batch(db_meta, list)) return 1;
// https://www.sqlite.org/pragma.html#pragma_cache_size
// PRAGMA schema.cache_size = pages;
// PRAGMA schema.cache_size = -kibibytes;
snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
if(init_database_batch(db_meta, list)) return 1;
snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
if(init_database_batch(db_meta, list)) return 1;
if (configure_sqlite_database(db_meta, target_version))
return 1;
if (init_database_batch(db_meta, &database_config[0]))
return 1;

View file

@ -50,6 +50,7 @@ SQLITE_API int sqlite3_exec_monitored(
int init_database_batch(sqlite3 *database, const char *batch[]);
int sql_init_database(db_check_action_type_t rebuild, int memory);
void sql_close_database(void);
int configure_sqlite_database(sqlite3 *database, int target_version);
// Helpers
int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);

View file

@ -60,11 +60,12 @@
#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata
#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata
#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write
#define METADATA_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
#define METADATA_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)
#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop
#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
enum metadata_opcode {
METADATA_DATABASE_NOOP = 0,
METADATA_DATABASE_TIMER,
@ -1148,6 +1149,28 @@ static void timer_cb(uv_timer_t* handle)
}
}
void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc)
{
int free_pages = get_free_page_count(database);
int total_pages = get_database_page_count(database);
if (!threshold)
threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
if (!vacuum_pc)
vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
if (free_pages > (total_pages * threshold / 100)) {
int do_free_pages = (int) (free_pages * vacuum_pc / 100);
netdata_log_info("%s: Freeing %d database pages", db_alias, do_free_pages);
char sql[128];
snprintfz(sql, 127, "PRAGMA incremental_vacuum(%d)", do_free_pages);
(void) db_execute(database, sql);
}
}
void run_metadata_cleanup(struct metadata_wc *wc)
{
if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
@ -1161,18 +1184,7 @@ void run_metadata_cleanup(struct metadata_wc *wc)
if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
return;
int free_pages = get_free_page_count(db_meta);
int total_pages = get_database_page_count(db_meta);
if (free_pages > (total_pages * METADATA_FREE_PAGES_THRESHOLD_PC / 100)) {
int do_free_pages = (int) (free_pages * METADATA_FREE_PAGES_VACUUM_PC / 100);
netdata_log_info("METADATA: Freeing %d database pages", do_free_pages);
char sql[128];
snprintfz(sql, 127, "PRAGMA incremental_vacuum(%d)", do_free_pages);
(void) db_execute(db_meta, sql);
}
vacuum_database(db_meta, "METADATA", DATABASE_FREE_PAGES_THRESHOLD_PC, DATABASE_FREE_PAGES_VACUUM_PC);
(void) sqlite3_wal_checkpoint(db_meta, NULL);
}

View file

@ -17,6 +17,7 @@ void metaqueue_host_update_info(RRDHOST *host);
void metaqueue_ml_load_models(RRDDIM *rd);
void migrate_localhost(uuid_t *host_uuid);
void metadata_queue_load_host_context(RRDHOST *host);
void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);
// UNIT TEST
int metadata_unittest(void);

View file

@ -9,6 +9,8 @@
#include "ad_charts.h"
#include "database/sqlite/sqlite3.h"
#define ML_METADATA_VERSION 2
#define WORKER_TRAIN_QUEUE_POP 0
#define WORKER_TRAIN_ACQUIRE_DIMENSION 1
#define WORKER_TRAIN_QUERY 2
@ -1625,6 +1627,8 @@ static void ml_flush_pending_models(ml_training_thread_t *training_thread) {
training_thread->num_models_to_prune += training_thread->pending_model_info.size();
}
vacuum_database(db, "ML", 0, 0);
training_thread->pending_model_info.clear();
}
@ -1777,14 +1781,22 @@ void ml_init()
// create table
if (db) {
char *err = NULL;
int rc = sqlite3_exec(db, db_models_create_table, NULL, NULL, &err);
if (rc != SQLITE_OK) {
error_report("Failed to create models table (%s, %s)", sqlite3_errstr(rc), err ? err : "");
int target_version = perform_ml_database_migration(db, ML_METADATA_VERSION);
if (configure_sqlite_database(db, target_version)) {
error_report("Failed to setup ML database");
sqlite3_close(db);
sqlite3_free(err);
db = NULL;
}
else {
char *err = NULL;
int rc = sqlite3_exec(db, db_models_create_table, NULL, NULL, &err);
if (rc != SQLITE_OK) {
error_report("Failed to create models table (%s, %s)", sqlite3_errstr(rc), err ? err : "");
sqlite3_close(db);
sqlite3_free(err);
db = NULL;
}
}
}
}

View file

@ -9,6 +9,7 @@ extern "C" {
#include "daemon/common.h"
#include "web/api/queries/rrdr.h"
#include "database/sqlite/sqlite_db_migration.h"
bool ml_capable();
bool ml_enabled(RRDHOST *rh);