dbengine free from RRDSET and RRDDIM (#13772)

* dbengine free from RRDSET and RRDDIM

* fix for excess parameters to query ops

* add comment about ML

* update_every from int to uint32_t

* rrddim_mem storage engine working

* fixes for update_every_s

* working dbengine

* a lot of changes in dbengine regarding timestamps

* better logging of not sequential points

* rrdset_done() now gives aligned timestamps for higher tiers

* dont change the end_time of descriptors, because they cant be loaded back

* fixes for cmake

* fixes for db mode ram

* Global counters for dbengine loading errors.
Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data.

* update lgtm config

* fixes for 32-bit systems

* update unittests

* Don't try to find and create a host on the fly if not already in memory

* Remove unused functions

* print backtrace in case of fatal

* always set ctx to page_index

* detect ctx and metric uuid discrepancies

* use legacy uuid if multihost is not available

* fix for last commit

* prevent repeating log

* Do not try to access archived charts when executing a data query

* Remove unused function

* log inconsistent collections once every 10 mins

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

.lgtm.yml

@@ -13,10 +13,12 @@ path_classifiers:
     - collectors/python.d.plugin/python_modules/pyyaml2/
     - collectors/python.d.plugin/python_modules/pyyaml3/
     - ml/kmeans/dlib/
+    - ml/dlib/dlib/
     - ml/json/
     - web/gui/lib/
     - web/gui/src/
     - web/gui/css/
+    - web/gui/dashboard/lib/
     - libnetdata/libjudy/
   test:
     - tests/

CMakeLists.txt

@@ -9,31 +9,25 @@ project(netdata C CXX)
 find_package(Threads REQUIRED)
 find_package(PkgConfig REQUIRED)
 
+include(CheckFunctionExists)
+include(CheckLibraryExists)
+
 # default is "Debug"
 #set(CMAKE_BUILD_TYPE "Release")
 
 # set this to see the compilation commands
-# set(CMAKE_VERBOSE_MAKEFILE 1)
-
+#set(CMAKE_VERBOSE_MAKEFILE 1)
 
 # -----------------------------------------------------------------------------
 # Set compilation options according to build type
 
-IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
-    message(STATUS "building for: debugging")
+set(CMAKE_C_STANDARD 11)
 
-    ## unfortunately these produce errors
-    #include(CheckCXXCompilerFlag)
-    #CHECK_CXX_COMPILER_FLAG("-Wformat-signedness" CXX_FORMAT_SIGNEDNESS)
-    #CHECK_CXX_COMPILER_FLAG("-Werror=format-security" CXX_FORMAT_SECURITY)
-    #CHECK_CXX_COMPILER_FLAG("-fstack-protector-all" CXX_STACK_PROTECTOR)
-    set(CXX_FORMAT_SIGNEDNESS "-Wformat-signedness")
-    set(CXX_FORMAT_SECURITY "-Werror=format-security")
-    set(CXX_STACK_PROTECTOR "-fstack-protector-all")
-    set(CXX_FLAGS_DEBUG "-O0")
-    set(CMAKE_C_STANDARD 99)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}")
+IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
+    set(CXX_DEFAULT_CFLAGS "-O0 -g -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_DEV_MODE=1 -fstack-protector-all -fno-omit-frame-pointer")
+    message(STATUS "building for: debugging")
 ELSE()
+    set(CXX_DEFAULT_CFLAGS "-O2")
     message(STATUS "building for: release")
     cmake_policy(SET CMP0069 "NEW")
     include(CheckIPOSupported)
@@ -46,6 +40,7 @@ ELSE()
     ENDIF()
 ENDIF()
 
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -Wformat-signedness -Werror=format-security -Wunused-result ${CXX_DEFAULT_CFLAGS}")
 
 # -----------------------------------------------------------------------------
 # O/S Detection
@@ -70,6 +65,12 @@ message(STATUS "system name: ${CMAKE_SYSTEM_NAME}")
 set(GENERATED_CONFIG_H_DIR ${CMAKE_BINARY_DIR})
 set(GENERATED_CONFIG_H ${GENERATED_CONFIG_H_DIR}/config.h)
 
+# -----------------------------------------------------------------------------
+# Math
+
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m)
+set(NETDATA_REQUIRED_DEFINES "-DSTORAGE_WITH_MATH=1 ${NETDATA_REQUIRED_DEFINES}")
+
 # -----------------------------------------------------------------------------
 # Detect libuuid
 
@@ -85,6 +86,7 @@ pkg_check_modules(ZLIB REQUIRED zlib)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${ZLIB_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${ZLIB_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DNETDATA_WITH_ZLIB=1")
 
 # -----------------------------------------------------------------------------
 # libuv multi-platform support library with a focus on asynchronous I/O
@@ -101,6 +103,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1")
 
 # -----------------------------------------------------------------------------
 # Judy General purpose dynamic array
@@ -114,6 +117,7 @@ pkg_check_modules(OPENSSL REQUIRED openssl)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${OPENSSL_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${OPENSSL_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_HTTPS=1")
 
 # -----------------------------------------------------------------------------
 # JSON-C used to health
@@ -1023,7 +1027,11 @@ set(ML_FILES
     ml/ml-dummy.c
 )
 
+# -----------------------------------------------------------------------------
+# ML
+
 IF(ENABLE_ML)
+    message(STATUS "ML: enabled")
     list(APPEND ML_FILES
             ml/Config.h
             ml/Config.cc
@@ -1041,6 +1049,8 @@ IF(ENABLE_ML)
             ml/ml.cc
             ml/ml-private.h
     )
+ELSE()
+    message(STATUS "ML: disabled")
 ENDIF()
 
 set(NETDATA_FILES
@@ -1731,7 +1741,6 @@ if (HAVE_SYS_SYSMACROS_H)
 endif()
 
 if (CRYPTO_FOUND)
-    include(CheckLibraryExists)
     set(HAVE_CRYPTO True)
     FIND_LIBRARY(CRYPTO_LIBRARY_LOCATION NAMES crypto)
     check_library_exists(crypto X509_VERIFY_PARAM_set1_host ${CRYPTO_LIBRARY_LOCATION} HAVE_X509_VERIFY_PARAM_set1_host)
@@ -1763,4 +1772,23 @@ ELSE()
     SET(MAYBE_UNUSED_MACRO "")
 ENDIF()
 
+# -----------------------------------------------------------------------------
+
+check_library_exists(c clock_gettime "" HAVE_CLOCK_GETTIME)
+
+IF(NOT HAVE_CLOCK_GETTIME)
+    CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME)
+ENDIF()
+
+IF(HAVE_CLOCK_GETTIME)
+    message("-- clock_gettime(): found")
+    set(NETDATA_REQUIRED_DEFINES "-DHAVE_CLOCK_GETTIME=1 ${NETDATA_REQUIRED_DEFINES}")
+ELSE()
+    message("-- clock_gettime(): not found")
+ENDIF()
+
+# -----------------------------------------------------------------------------
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${NETDATA_REQUIRED_DEFINES}")
+message("CFLAGS=\"${CMAKE_C_FLAGS}\"")
+
 configure_file(config.cmake.h.in config.h)

aclk/aclk_query.c

@@ -99,7 +99,6 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
         }
     }
 
-    RRDHOST *temp_host = NULL;
     if (!strncmp(query->data.http_api_v2.query, NODE_ID_QUERY, strlen(NODE_ID_QUERY))) {
         char *node_uuid = query->data.http_api_v2.query + strlen(NODE_ID_QUERY);
         char nodeid[UUID_STR_LEN];
@@ -114,14 +113,11 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
 
         query_host = node_id_2_rrdhost(nodeid);
         if (!query_host) {
-            temp_host = sql_create_host_by_uuid(nodeid);
-            if (!temp_host) {
-                error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
-                retval = 1;
-                w->response.code = 404;
-                aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
-                goto cleanup;
-            }
+            error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
+            retval = 1;
+            w->response.code = 404;
+            aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
+            goto cleanup;
         }
     }
 
@@ -142,8 +138,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
     }
 
     // execute the query
-    t = aclk_web_api_v1_request(query_host ? query_host : temp_host, w, mysep ? mysep + 1 : "noop");
-    free_temporary_host(temp_host);
+    t = aclk_web_api_v1_request(query_host, w, mysep ? mysep + 1 : "noop");
     size = (w->mode == WEB_CLIENT_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;
     sent = size;
 

collectors/statsd.plugin/statsd.c

@@ -1421,7 +1421,10 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
                     app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
             }
             else if (!strcmp(name, "memory mode")) {
-                app->rrd_memory_mode = rrd_memory_mode_id(value);
+                // this is not supported anymore
+                // with the implementation of storage engines, all charts have the same storage engine always
+                // app->rrd_memory_mode = rrd_memory_mode_id(value);
+                ;
             }
             else if (!strcmp(name, "history")) {
                 app->rrd_history_entries = atol(value);

configure.ac

@@ -317,6 +317,10 @@ AM_CONDITIONAL([MACOS], [test "${build_target}" = "macos"])
 AM_CONDITIONAL([LINUX], [test "${build_target}" = "linux"])
 AC_MSG_RESULT([Host OS: ${build_target}])
 
+# -----------------------------------------------------------------------------
+# backtrace
+
+AC_SEARCH_LIBS([backtrace], [execinfo], [AC_DEFINE([HAVE_BACKTRACE], [1], [backtrace availability])])
 
 # -----------------------------------------------------------------------------
 # pthreads

daemon/unit_test.c

@@ -1816,6 +1816,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
     // feed it with the test data
     for (i = 0 ; i < CHARTS ; ++i) {
         for (j = 0 ; j < DIMS ; ++j) {
+            rd[i][j]->tiers[0]->collect_ops.change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
+
             rd[i][j]->last_collected_time.tv_sec =
             st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
             rd[i][j]->last_collected_time.tv_usec =
@@ -1859,7 +1861,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
         time_now = time_start + (c + 1) * update_every;
         for (i = 0 ; i < CHARTS ; ++i) {
             for (j = 0; j < DIMS; ++j) {
-                rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, TIER_QUERY_FETCH_SUM);
+                rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
                 for (k = 0; k < QUERY_BATCH; ++k) {
                     last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
                            j * REGION_POINTS[current_region] + c + k;
@@ -2316,7 +2318,7 @@ static void query_dbengine_chart(void *arg)
             time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
         }
 
-        rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, TIER_QUERY_FETCH_SUM);
+        rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
         ++thread_info->queries_nr;
         for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
             generatedv = generate_dbengine_chart_value(i, j, time_now);

database/engine/journalfile.c

@@ -311,20 +311,46 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
             }
             continue;
         }
-        uint64_t start_time = jf_metric_data->descr[i].start_time;
-        uint64_t end_time = jf_metric_data->descr[i].end_time;
+        uint64_t start_time_ut = jf_metric_data->descr[i].start_time_ut;
+        uint64_t end_time_ut = jf_metric_data->descr[i].end_time_ut;
+        size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
+        time_t update_every_s = (entries > 1) ? ((end_time_ut - start_time_ut) / USEC_PER_SEC / (entries - 1)) : 0;
 
-        if (unlikely(start_time > end_time)) {
-            error("Invalid page encountered, start time %"PRIu64" > end time %"PRIu64"", start_time , end_time);
+        if (unlikely(start_time_ut > end_time_ut)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut = end_time_ut;
             continue;
         }
 
-        if (unlikely(start_time == end_time)) {
-            size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
-            if (unlikely(entries > 1)) {
-                error("Invalid page encountered, start time %"PRIu64" = end time but %zu entries were found", start_time, entries);
-                continue;
-            }
+        if (unlikely(start_time_ut == end_time_ut && entries != 1)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut = end_time_ut;
+            continue;
+        }
+
+        if (unlikely(!entries)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut = end_time_ut;
+            continue;
+        }
+
+        if(entries > 1 && update_every_s == 0) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut = end_time_ut;
+            continue;
+        }
+
+        if(start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1) != end_time_ut) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut = end_time_ut;
+
+            // let this be
+            // end_time_ut = start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1);
         }
 
         temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
@@ -340,7 +366,7 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
             uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
             PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, temp_id, sizeof(uuid_t), PJE0);
             fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
-            *PValue = page_index = create_page_index(temp_id);
+            *PValue = page_index = create_page_index(temp_id, ctx);
             page_index->prev = pg_cache->metrics_index.last_page_index;
             pg_cache->metrics_index.last_page_index = page_index;
             uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
@@ -348,21 +374,32 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
 
         descr = pg_cache_create_descr();
         descr->page_length = jf_metric_data->descr[i].page_length;
-        descr->start_time = start_time;
-        descr->end_time = end_time;
+        descr->start_time_ut = start_time_ut;
+        descr->end_time_ut = end_time_ut;
+        descr->update_every_s = (update_every_s > 0) ? (uint32_t)update_every_s : (page_index->latest_update_every_s);
         descr->id = &page_index->id;
         descr->extent = extent;
         descr->type = page_type;
         extent->pages[valid_pages++] = descr;
         pg_cache_insert(ctx, page_index, descr);
+
+        if(page_index->latest_time_ut == descr->end_time_ut)
+            page_index->latest_update_every_s = descr->update_every_s;
+
+        if(descr->update_every_s == 0)
+            fatal(
+                "DBENGINE: page descriptor update every is zero, end_time_ut = %llu, start_time_ut = %llu, entries = %zu",
+                (unsigned long long)end_time_ut, (unsigned long long)start_time_ut, entries);
     }
 
     extent->number_of_pages = valid_pages;
 
     if (likely(valid_pages))
         df_extent_insert(extent);
-    else
+    else {
         freez(extent);
+        ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter++;
+    }
 }
 
 /*

database/engine/pagecache.c

@@ -127,12 +127,13 @@ struct rrdeng_page_descr *pg_cache_create_descr(void)
 
     descr = rrdeng_page_descr_mallocz();
     descr->page_length = 0;
-    descr->start_time = INVALID_TIME;
-    descr->end_time = INVALID_TIME;
+    descr->start_time_ut = INVALID_TIME;
+    descr->end_time_ut = INVALID_TIME;
     descr->id = NULL;
     descr->extent = NULL;
     descr->pg_cache_descr_state = 0;
     descr->pg_cache_descr = NULL;
+    descr->update_every_s = 0;
 
     return descr;
 }
@@ -476,7 +477,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
     uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
 
     uv_rwlock_wrlock(&page_index->lock);
-    ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
+    ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
     if (unlikely(0 == ret)) {
         uv_rwlock_wrunlock(&page_index->lock);
         if (unlikely(debug_flags & D_RRDENGINE)) {
@@ -506,7 +507,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
         while (!pg_cache_try_get_unsafe(descr, 1)) {
             debug(D_RRDENGINE, "%s: Waiting for locked page:", __func__);
             if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             pg_cache_wait_event_unsafe(descr);
         }
     }
@@ -517,7 +518,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
         while (unlikely(pg_cache_descr->flags & RRD_PAGE_DIRTY)) {
             debug(D_RRDENGINE, "%s: Found dirty page, waiting for it to be flushed:", __func__);
             if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             pg_cache_wait_event_unsafe(descr);
         }
     }
@@ -548,8 +549,8 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
 {
     usec_t pg_start, pg_end;
 
-    pg_start = descr->start_time;
-    pg_end = descr->end_time;
+    pg_start = descr->start_time_ut;
+    pg_end = descr->end_time_ut;
 
     return (pg_start < start_time && pg_end >= start_time) ||
            (pg_start >= start_time && pg_start <= end_time);
@@ -557,7 +558,7 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
 
 static inline int is_point_in_time_in_page(struct rrdeng_page_descr *descr, usec_t point_in_time)
 {
-    return (point_in_time >= descr->start_time && point_in_time <= descr->end_time);
+    return (point_in_time >= descr->start_time_ut && point_in_time <= descr->end_time_ut);
 }
 
 /* The caller must hold the page index lock */
@@ -592,14 +593,14 @@ static inline struct rrdeng_page_descr *
 /* Update metric oldest and latest timestamps efficiently when adding new values */
 void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr)
 {
-    usec_t oldest_time = page_index->oldest_time;
-    usec_t latest_time = page_index->latest_time;
+    usec_t oldest_time = page_index->oldest_time_ut;
+    usec_t latest_time = page_index->latest_time_ut;
 
-    if (unlikely(oldest_time == INVALID_TIME || descr->start_time < oldest_time)) {
-        page_index->oldest_time = descr->start_time;
+    if (unlikely(oldest_time == INVALID_TIME || descr->start_time_ut < oldest_time)) {
+        page_index->oldest_time_ut = descr->start_time_ut;
     }
-    if (likely(descr->end_time > latest_time || latest_time == INVALID_TIME)) {
-        page_index->latest_time = descr->end_time;
+    if (likely(descr->end_time_ut > latest_time || latest_time == INVALID_TIME)) {
+        page_index->latest_time_ut = descr->end_time_ut;
     }
 }
 
@@ -618,23 +619,23 @@ void pg_cache_update_metric_times(struct pg_cache_page_index *page_index)
     firstPValue = JudyLFirst(page_index->JudyL_array, &firstIndex, PJE0);
     if (likely(NULL != firstPValue)) {
         descr = *firstPValue;
-        oldest_time = descr->start_time;
+        oldest_time = descr->start_time_ut;
     }
     lastIndex = (Word_t)-1;
     lastPValue = JudyLLast(page_index->JudyL_array, &lastIndex, PJE0);
     if (likely(NULL != lastPValue)) {
         descr = *lastPValue;
-        latest_time = descr->end_time;
+        latest_time = descr->end_time_ut;
     }
     uv_rwlock_rdunlock(&page_index->lock);
 
     if (unlikely(NULL == firstPValue)) {
         fatal_assert(NULL == lastPValue);
-        page_index->oldest_time = page_index->latest_time = INVALID_TIME;
+        page_index->oldest_time_ut = page_index->latest_time_ut = INVALID_TIME;
         return;
     }
-    page_index->oldest_time = oldest_time;
-    page_index->latest_time = latest_time;
+    page_index->oldest_time_ut = oldest_time;
+    page_index->latest_time_ut = latest_time;
 }
 
 /* If index is NULL lookup by UUID (descr->id) */
@@ -669,7 +670,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
     }
 
     uv_rwlock_wrlock(&page_index->lock);
-    PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
+    PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
     *PValue = descr;
     ++page_index->page_count;
     pg_cache_add_new_metric_time(page_index, descr);
@@ -681,7 +682,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
     uv_rwlock_wrunlock(&pg_cache->pg_cache_rwlock);
 }
 
-usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time)
+usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -699,25 +700,25 @@ usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    descr = find_first_page_in_time_range(page_index, start_time, end_time);
+    descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
     if (NULL == descr) {
         uv_rwlock_rdunlock(&page_index->lock);
         return INVALID_TIME;
     }
     uv_rwlock_rdunlock(&page_index->lock);
-    return descr->start_time;
+    return descr->start_time_ut;
 }
 
 /**
  * Return page information for the first page before point_in_time that satisfies the filter.
  * @param ctx DB context
  * @param page_index page index of a metric
- * @param point_in_time the pages that are searched must be older than this timestamp
+ * @param point_in_time_ut the pages that are searched must be older than this timestamp
  * @param filter decides if the page satisfies the caller's criteria
  * @param page_info the result of the search is set in this pointer
  */
 void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
-                                     usec_t point_in_time, pg_cache_page_info_filter_t *filter,
+                                     usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
                                      struct rrdeng_page_info *page_info)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
@@ -728,7 +729,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
     (void)pg_cache;
     fatal_assert(NULL != page_index);
 
-    Index = (Word_t)(point_in_time / USEC_PER_SEC);
+    Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
     uv_rwlock_rdlock(&page_index->lock);
     do {
         PValue = JudyLPrev(page_index->JudyL_array, &Index, PJE0);
@@ -736,12 +737,12 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
     } while (descr != NULL && !filter(descr));
     if (unlikely(NULL == descr)) {
         page_info->page_length = 0;
-        page_info->start_time = INVALID_TIME;
-        page_info->end_time = INVALID_TIME;
+        page_info->start_time_ut = INVALID_TIME;
+        page_info->end_time_ut = INVALID_TIME;
     } else {
         page_info->page_length = descr->page_length;
-        page_info->start_time = descr->start_time;
-        page_info->end_time = descr->end_time;
+        page_info->start_time_ut = descr->start_time_ut;
+        page_info->end_time_ut = descr->end_time_ut;
     }
     uv_rwlock_rdunlock(&page_index->lock);
 }
@@ -750,7 +751,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
  * Searches for an unallocated page without triggering disk I/O. Attempts to reserve the page and get a reference.
  * @param ctx DB context
  * @param id lookup by UUID
- * @param start_time exact starting time in usec
+ * @param start_time_ut exact starting time in usec
  * @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
  * @return the page descriptor or NULL on failure. It can fail if:
  *         1. The page is already allocated to the page cache.
@@ -758,7 +759,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
  *         3. It did not succeed to reserve a spot in the page cache.
  */
 struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
-                                                               usec_t start_time)
+                                                               usec_t start_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -781,7 +782,7 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    Index = (Word_t)(start_time / USEC_PER_SEC);
+    Index = (Word_t)(start_time_ut / USEC_PER_SEC);
     PValue = JudyLGet(page_index->JudyL_array, Index, PJE0);
     if (likely(NULL != PValue)) {
         descr = *PValue;
@@ -818,15 +819,15 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
  * Does not get a reference.
  * @param ctx DB context
  * @param id UUID
- * @param start_time inclusive starting time in usec
- * @param end_time inclusive ending time in usec
+ * @param start_time_ut inclusive starting time in usec
+ * @param end_time_ut inclusive ending time in usec
  * @param page_info_arrayp It allocates (*page_arrayp) and populates it with information of pages that overlap
  *        with the time range [start_time,end_time]. The caller must free (*page_info_arrayp) with freez().
  *        If page_info_arrayp is set to NULL nothing was allocated.
  * @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
  * @return the number of pages that overlap with the time range [start_time,end_time].
  */
-unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
+unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
                           struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
@@ -854,14 +855,14 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    descr = find_first_page_in_time_range(page_index, start_time, end_time);
+    descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
     if (NULL == descr) {
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: No page was found to attempt preload.", __func__);
         *ret_page_indexp = NULL;
         return 0;
     } else {
-        Index = (Word_t)(descr->start_time / USEC_PER_SEC);
+        Index = (Word_t)(descr->start_time_ut / USEC_PER_SEC);
     }
     if (page_info_arrayp) {
         page_info_array_max_size = PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
@@ -869,7 +870,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
     }
 
     for (count = 0, preload_count = 0 ;
-         descr != NULL && is_page_in_time_range(descr, start_time, end_time) ;
+         descr != NULL && is_page_in_time_range(descr, start_time_ut, end_time_ut) ;
          PValue = JudyLNext(page_index->JudyL_array, &Index, PJE0),
          descr = unlikely(NULL == PValue) ? NULL : *PValue) {
         /* Iterate all pages in range */
@@ -881,8 +882,8 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
                 page_info_array_max_size += PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
                 *page_info_arrayp = reallocz(*page_info_arrayp, page_info_array_max_size);
             }
-            (*page_info_arrayp)[count].start_time = descr->start_time;
-            (*page_info_arrayp)[count].end_time = descr->end_time;
+            (*page_info_arrayp)[count].start_time_ut = descr->start_time_ut;
+            (*page_info_arrayp)[count].end_time_ut = descr->end_time_ut;
             (*page_info_arrayp)[count].page_length = descr->page_length;
         }
         ++count;
@@ -974,7 +975,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
  */
 struct rrdeng_page_descr *
         pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                        usec_t point_in_time)
+                        usec_t point_in_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -1003,15 +1004,15 @@ struct rrdeng_page_descr *
     page_not_in_cache = 0;
     uv_rwlock_rdlock(&page_index->lock);
     while (1) {
-        Index = (Word_t)(point_in_time / USEC_PER_SEC);
+        Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
         PValue = JudyLLast(page_index->JudyL_array, &Index, PJE0);
         if (likely(NULL != PValue)) {
             descr = *PValue;
         }
         if (NULL == PValue ||
             0 == descr->page_length ||
-            (INVALID_TIME != point_in_time &&
-             !is_point_in_time_in_page(descr, point_in_time))) {
+            (INVALID_TIME != point_in_time_ut &&
+             !is_point_in_time_in_page(descr, point_in_time_ut))) {
             /* non-empty page not found */
             uv_rwlock_rdunlock(&page_index->lock);
 
@@ -1038,7 +1039,7 @@ struct rrdeng_page_descr *
 
             debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
             if(unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
                 pg_cache_wait_event_unsafe(descr);
             }
@@ -1053,7 +1054,7 @@ struct rrdeng_page_descr *
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
         if(unlikely(debug_flags & D_RRDENGINE))
-            print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "", true);
         if (!(flags & RRD_PAGE_POPULATED))
             page_not_in_cache = 1;
         pg_cache_wait_event_unsafe(descr);
@@ -1081,7 +1082,7 @@ struct rrdeng_page_descr *
  */
 struct rrdeng_page_descr *
 pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                     usec_t start_time, usec_t end_time)
+                     usec_t start_time_ut, usec_t end_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -1110,7 +1111,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
     uv_rwlock_rdlock(&page_index->lock);
     int retry_count = 0;
     while (1) {
-        descr = find_first_page_in_time_range(page_index, start_time, end_time);
+        descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
         if (NULL == descr || 0 == descr->page_length || retry_count == default_rrdeng_page_fetch_retries) {
             /* non-empty page not found */
             if (retry_count == default_rrdeng_page_fetch_retries)
@@ -1140,7 +1141,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
 
             debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
             if(unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
                 pg_cache_wait_event_unsafe(descr);
             }
@@ -1155,7 +1156,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
         if(unlikely(debug_flags & D_RRDENGINE))
-            print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "", true);
         if (!(flags & RRD_PAGE_POPULATED))
             page_not_in_cache = 1;
 
@@ -1180,7 +1181,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
     return descr;
 }
 
-struct pg_cache_page_index *create_page_index(uuid_t *id)
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx)
 {
     struct pg_cache_page_index *page_index;
 
@@ -1188,11 +1189,15 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
     page_index->JudyL_array = (Pvoid_t) NULL;
     uuid_copy(page_index->id, *id);
     fatal_assert(0 == uv_rwlock_init(&page_index->lock));
-    page_index->oldest_time = INVALID_TIME;
-    page_index->latest_time = INVALID_TIME;
+    page_index->oldest_time_ut = INVALID_TIME;
+    page_index->latest_time_ut = INVALID_TIME;
     page_index->prev = NULL;
     page_index->page_count = 0;
+    page_index->refcount = 0;
     page_index->writers = 0;
+    page_index->ctx = ctx;
+    page_index->alignment = NULL;
+    page_index->latest_update_every_s = default_rrd_update_every;
 
     return page_index;
 }

database/engine/pagecache.h

@@ -60,18 +60,19 @@ struct rrdeng_page_descr {
     volatile unsigned long pg_cache_descr_state;
 
     /* page information */
-    usec_t start_time;
-    usec_t end_time;
-    uint32_t page_length;
+    usec_t start_time_ut;
+    usec_t end_time_ut;
+    uint32_t update_every_s:24;
     uint8_t type;
+    uint32_t page_length;
 };
 
 #define PAGE_INFO_SCRATCH_SZ (8)
 struct rrdeng_page_info {
     uint8_t scratch[PAGE_INFO_SCRATCH_SZ]; /* scratch area to be used by page-cache users */
 
-    usec_t start_time;
-    usec_t end_time;
+    usec_t start_time_ut;
+    usec_t end_time_ut;
     uint32_t page_length;
 };
 
@@ -80,6 +81,11 @@ typedef int pg_cache_page_info_filter_t(struct rrdeng_page_descr *);
 
 #define PAGE_CACHE_MAX_PRELOAD_PAGES    (256)
 
+struct pg_alignment {
+    uint32_t page_length;
+    uint32_t refcount;
+};
+
 /* maps time ranges to pages */
 struct pg_cache_page_index {
     uuid_t id;
@@ -89,6 +95,7 @@ struct pg_cache_page_index {
      */
     Pvoid_t JudyL_array;
     Word_t page_count;
+    unsigned short refcount;
     unsigned short writers;
     uv_rwlock_t lock;
 
@@ -96,13 +103,17 @@ struct pg_cache_page_index {
      * Only one effective writer, data deletion workqueue.
      * It's also written during the DB loading phase.
      */
-    usec_t oldest_time;
+    usec_t oldest_time_ut;
 
     /*
      * Only one effective writer, data collection thread.
      * It's also written by the data deletion workqueue when data collection is disabled for this metric.
      */
-    usec_t latest_time;
+    usec_t latest_time_ut;
+
+    struct rrdengine_instance *ctx;
+    struct pg_alignment *alignment;
+    uint32_t latest_update_every_s;
 
     struct pg_cache_page_index *prev;
 };
@@ -171,22 +182,22 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
 uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
                                    uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
 usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
-                                            usec_t start_time, usec_t end_time);
+                                            usec_t start_time_ut, usec_t end_time_ut);
 void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
-                                            usec_t point_in_time, pg_cache_page_info_filter_t *filter,
+                                            usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
                                             struct rrdeng_page_info *page_info);
 struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
-                                                                      usec_t start_time);
-extern unsigned
-        pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
+                                                                      usec_t start_time_ut);
+unsigned
+        pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
                          struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp);
-extern struct rrdeng_page_descr *
+struct rrdeng_page_descr *
         pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                        usec_t point_in_time);
-extern struct rrdeng_page_descr *
+                        usec_t point_in_time_ut);
+struct rrdeng_page_descr *
         pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                     usec_t start_time, usec_t end_time);
-struct pg_cache_page_index *create_page_index(uuid_t *id);
+                     usec_t start_time_ut, usec_t end_time_ut);
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx);
 void init_page_cache(struct rrdengine_instance *ctx);
 void free_page_cache(struct rrdengine_instance *ctx);
 void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr);
@@ -204,41 +215,41 @@ struct rrdeng_page_descr *rrdeng_page_descr_mallocz(void);
 void rrdeng_page_descr_freez(struct rrdeng_page_descr *descr);
 
 static inline void
-    pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_timep, uint32_t *page_lengthp)
+    pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_time_ut_p, uint32_t *page_lengthp)
 {
-    usec_t end_time, old_end_time;
+    usec_t end_time_ut, old_end_time_ut;
     uint32_t page_length;
 
     if (NULL == descr->extent) {
         /* this page is currently being modified, get consistent info locklessly */
         do {
-            end_time = descr->end_time;
+            end_time_ut = descr->end_time_ut;
             __sync_synchronize();
-            old_end_time = end_time;
+            old_end_time_ut = end_time_ut;
             page_length = descr->page_length;
             __sync_synchronize();
-            end_time = descr->end_time;
+            end_time_ut = descr->end_time_ut;
             __sync_synchronize();
-        } while ((end_time != old_end_time || (end_time & 1) != 0));
+        } while ((end_time_ut != old_end_time_ut || (end_time_ut & 1) != 0));
 
-        *end_timep = end_time;
+        *end_time_ut_p = end_time_ut;
         *page_lengthp = page_length;
     } else {
-        *end_timep = descr->end_time;
+        *end_time_ut_p = descr->end_time_ut;
         *page_lengthp = descr->page_length;
     }
 }
 
 /* The caller must hold a reference to the page and must have already set the new data */
-static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time, uint32_t page_length)
+static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time_ut, uint32_t page_length)
 {
-    fatal_assert(!(end_time & 1));
+    fatal_assert(!(end_time_ut & 1));
     __sync_synchronize();
-    descr->end_time |= 1; /* mark start of uncertainty period by adding 1 microsecond */
+    descr->end_time_ut |= 1; /* mark start of uncertainty period by adding 1 microsecond */
     __sync_synchronize();
     descr->page_length = page_length;
     __sync_synchronize();
-    descr->end_time = end_time; /* mark end of uncertainty period */
+    descr->end_time_ut = end_time_ut; /* mark end of uncertainty period */
 }
 
 #endif /* NETDATA_PAGECACHE_H */

database/engine/rrddiskprotocol.h

@@ -46,8 +46,8 @@ struct rrdeng_extent_page_descr {
 
     uint8_t uuid[UUID_SZ];
     uint32_t page_length;
-    uint64_t start_time;
-    uint64_t end_time;
+    uint64_t start_time_ut;
+    uint64_t end_time_ut;
 } __attribute__ ((packed));
 
 /*

database/engine/rrdengine.c

@@ -206,8 +206,8 @@ void read_cached_extent_cb(struct rrdengine_worker_config* wc, unsigned idx, str
             /* care, we don't hold the descriptor mutex */
             if (!uuid_compare(*extent->pages[j]->id, *descr->id) &&
                 extent->pages[j]->page_length == descr->page_length &&
-                extent->pages[j]->start_time == descr->start_time &&
-                extent->pages[j]->end_time == descr->end_time) {
+                extent->pages[j]->start_time_ut == descr->start_time_ut &&
+                extent->pages[j]->end_time_ut == descr->end_time_ut) {
                 break;
             }
             page_offset += extent->pages[j]->page_length;
@@ -378,8 +378,8 @@ after_crc_check:
             /* care, we don't hold the descriptor mutex */
             if (!uuid_compare(*(uuid_t *) header->descr[i].uuid, *descrj->id) &&
                 header->descr[i].page_length == descrj->page_length &&
-                header->descr[i].start_time == descrj->start_time &&
-                header->descr[i].end_time == descrj->end_time) {
+                header->descr[i].start_time_ut == descrj->start_time_ut &&
+                header->descr[i].end_time_ut == descrj->end_time_ut) {
                 descr = descrj;
                 break;
             }
@@ -387,7 +387,7 @@ after_crc_check:
         is_prefetched_page = 0;
         if (!descr) { /* This extent page has not been requested. Try populating it for locality (best effort). */
             descr = pg_cache_lookup_unpopulated_and_lock(ctx, (uuid_t *)header->descr[i].uuid,
-                                                         header->descr[i].start_time);
+                                                         header->descr[i].start_time_ut);
             if (!descr)
                 continue; /* Failed to reserve a suitable page */
             is_prefetched_page = 1;
@@ -820,8 +820,8 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct
         header->descr[i].type = descr->type;
         uuid_copy(*(uuid_t *)header->descr[i].uuid, *descr->id);
         header->descr[i].page_length = descr->page_length;
-        header->descr[i].start_time = descr->start_time;
-        header->descr[i].end_time = descr->end_time;
+        header->descr[i].start_time_ut = descr->start_time_ut;
+        header->descr[i].end_time_ut = descr->end_time_ut;
         pos += sizeof(header->descr[i]);
     }
     for (i = 0 ; i < count ; ++i) {
@@ -1043,7 +1043,70 @@ static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
 /* return 0 on success */
 int init_rrd_files(struct rrdengine_instance *ctx)
 {
-    return init_data_files(ctx);
+    int ret = init_data_files(ctx);
+
+    BUFFER *wb = buffer_create(1000);
+    size_t all_errors = 0;
+    usec_t now = now_realtime_usec();
+
+    if(ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter) {
+        buffer_sprintf(wb, "%s%zu pages had start time > end time (latest: %llu secs ago)"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter
+                       , (now - ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut) / USEC_PER_SEC
+                       );
+        all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter;
+    }
+
+    if(ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter) {
+        buffer_sprintf(wb, "%s%zu pages had start time = end time with more than 1 entries (latest: %llu secs ago)"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter
+                       , (now - ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut) / USEC_PER_SEC
+        );
+        all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter;
+    }
+
+    if(ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter) {
+        buffer_sprintf(wb, "%s%zu pages had zero points (latest: %llu secs ago)"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter
+                       , (now - ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut) / USEC_PER_SEC
+        );
+        all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter;
+    }
+
+    if(ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter) {
+        buffer_sprintf(wb, "%s%zu pages had update every == 0 with entries > 1 (latest: %llu secs ago)"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter
+                       , (now - ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut) / USEC_PER_SEC
+        );
+        all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter;
+    }
+
+    if(ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter) {
+        buffer_sprintf(wb, "%s%zu pages had a different number of points compared to their timestamps (latest: %llu secs ago; these page have been loaded)"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter
+                       , (now - ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut) / USEC_PER_SEC
+        );
+        all_errors += ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter;
+    }
+
+    if(ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter) {
+        buffer_sprintf(wb, "%s%zu extents have been dropped because they didn't have any valid pages"
+                       , (all_errors)?", ":""
+                       , ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter
+        );
+        all_errors += ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter;
+    }
+
+    if(all_errors)
+        info("DBENGINE: tier %d: %s", ctx->tier, buffer_tostring(wb));
+
+    buffer_free(wb);
+    return ret;
 }
 
 void finalize_rrd_files(struct rrdengine_instance *ctx)

database/engine/rrdengine.h

@@ -37,29 +37,25 @@ struct rrdengine_instance;
 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
 
 struct rrdeng_collect_handle {
-    struct rrdeng_metric_handle *metric_handle;
+    struct pg_cache_page_index *page_index;
     struct rrdeng_page_descr *descr;
     unsigned long page_correlation_id;
-    struct rrdengine_instance *ctx;
     // set to 1 when this dimension is not page aligned with the other dimensions in the chart
     uint8_t unaligned_page;
 };
 
 struct rrdeng_query_handle {
-    struct rrdeng_metric_handle *metric_handle;
     struct rrdeng_page_descr *descr;
     struct rrdengine_instance *ctx;
     struct pg_cache_page_index *page_index;
-    time_t next_page_time;
-    time_t now;
+    time_t wanted_start_time_s;
+    time_t now_s;
     unsigned position;
     unsigned entries;
-    TIER_QUERY_FETCH tier_query_fetch_type;
     storage_number *page;
-    usec_t page_end_time;
+    usec_t page_end_time_ut;
     uint32_t page_length;
-    usec_t dt;
-    time_t dt_sec;
+    time_t dt_s;
 };
 
 typedef enum {
@@ -230,6 +226,15 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele
 #define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
 #define QUIESCED    (2) /* is set after all threads have finished running */
 
+typedef enum {
+    LOAD_ERRORS_PAGE_FLIPPED_TIME = 0,
+    LOAD_ERRORS_PAGE_EQUAL_TIME = 1,
+    LOAD_ERRORS_PAGE_ZERO_ENTRIES = 2,
+    LOAD_ERRORS_PAGE_UPDATE_ZERO = 3,
+    LOAD_ERRORS_PAGE_FLEXY_TIME = 4,
+    LOAD_ERRORS_DROPPED_EXTENT = 5,
+} INVALID_PAGE_ID;
+
 struct rrdengine_instance {
     struct metalog_instance *metalog_ctx;
     struct rrdengine_worker_config worker_config;
@@ -254,6 +259,11 @@ struct rrdengine_instance {
     uint8_t page_type; /* Default page type for this context */
 
     struct rrdengine_statistics stats;
+
+    struct {
+        size_t counter;
+        usec_t latest_end_time_ut;
+    } load_errors[6];
 };
 
 void *dbengine_page_alloc(void);

database/engine/rrdengineapi.c

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 #include "rrdengine.h"
+#include "../storage_engine.h"
 
 /* Default global database instance */
 struct rrdengine_instance multidb_ctx_storage_tier0;
@@ -35,14 +36,41 @@ int default_multidb_disk_quota_mb = 256;
 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
 
+
+// ----------------------------------------------------------------------------
+// helpers
+
 static inline struct rrdengine_instance *get_rrdeng_ctx_from_host(RRDHOST *host, int tier) {
     if(tier < 0 || tier >= RRD_STORAGE_TIERS) tier = 0;
     if(!host->storage_instance[tier]) tier = 0;
     return (struct rrdengine_instance *)host->storage_instance[tier];
 }
 
+// ----------------------------------------------------------------------------
+// metrics groups
+
+STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid __maybe_unused) {
+    return callocz(1, sizeof(struct pg_alignment));
+}
+
+void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
+    if(!smg) return;
+
+    struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
+    struct pg_alignment *pa = (struct pg_alignment *)smg;
+    struct page_cache *pg_cache = &ctx->pg_cache;
+
+    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+    if(pa->refcount == 0)
+        freez(pa);
+    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
+}
+
+// ----------------------------------------------------------------------------
+// metric handle for legacy dbs
+
 /* This UUID is not unique across hosts */
-void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
+void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
 {
     EVP_MD_CTX *evpctx;
     unsigned char hash_value[EVP_MAX_MD_SIZE];
@@ -75,97 +103,134 @@ void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uu
     memcpy(ret_uuid, hash_value, sizeof(uuid_t));
 }
 
-struct rrdeng_metric_handle {
-    RRDDIM *rd;
-    struct rrdengine_instance *ctx;
-    uuid_t *rrdeng_uuid;                            // database engine metric UUID
-    struct pg_cache_page_index *page_index;
-};
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, const char *rd_id, const char *st_id, STORAGE_METRICS_GROUP *smg) {
+    uuid_t legacy_uuid;
+    rrdeng_generate_legacy_uuid(rd_id, st_id, &legacy_uuid);
+    return rrdeng_metric_get(db_instance, &legacy_uuid, smg);
+}
+
+// ----------------------------------------------------------------------------
+// metric handle
+
+void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle) {
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+    struct rrdengine_instance *ctx = page_index->ctx;
+    struct page_cache *pg_cache = &ctx->pg_cache;
+
+    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
+    page_index->refcount--;
+    if(page_index->alignment && page_index->refcount == 0) {
+        page_index->alignment->refcount--;
+        page_index->alignment = NULL;
+    }
+    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
 
-void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle) {
     freez(db_metric_handle);
 }
 
-STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance) {
+STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg) {
     struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
-    struct page_cache *pg_cache;
-    uuid_t legacy_uuid;
-    uuid_t multihost_legacy_uuid;
-    Pvoid_t *PValue;
+    struct pg_alignment *pa = (struct pg_alignment *)smg;
+    struct page_cache *pg_cache = &ctx->pg_cache;
     struct pg_cache_page_index *page_index = NULL;
-    int is_multihost_child = 0;
-    RRDHOST *host = rd->rrdset->rrdhost;
-
-    pg_cache = &ctx->pg_cache;
-
-    rrdeng_generate_legacy_uuid(rrddim_id(rd), (char *)rrdset_id(rd->rrdset), &legacy_uuid);
-    if (host != localhost && is_storage_engine_shared((STORAGE_INSTANCE *)ctx))
-        is_multihost_child = 1;
 
     uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
-    PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &legacy_uuid, sizeof(uuid_t));
+    Pvoid_t *PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, uuid, sizeof(uuid_t));
     if (likely(NULL != PValue)) {
         page_index = *PValue;
+        page_index->refcount++;
+
+        if(pa) {
+            if(page_index->alignment && page_index->alignment != pa)
+                fatal("DBENGINE: page_index has a different alignment.");
+
+            if(!page_index->alignment) {
+                page_index->alignment = pa;
+                pa->refcount++;
+            }
+        }
     }
     uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
-    if (is_multihost_child || NULL == PValue) {
-        /* First time we see the legacy UUID or metric belongs to child host in multi-host DB.
-         * Drop legacy support, normal path */
 
-        uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
-        PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t));
-        if (likely(NULL != PValue)) {
-            page_index = *PValue;
+    return (STORAGE_METRIC_HANDLE *)page_index;
+}
+
+STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg) {
+    internal_fatal(!db_instance, "DBENGINE: db_instance is NULL");
+
+    struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
+    struct pg_alignment *pa = (struct pg_alignment *)smg;
+    struct pg_cache_page_index *page_index;
+    struct page_cache *pg_cache = &ctx->pg_cache;
+
+    uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
+    Pvoid_t *PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, uuid, sizeof(uuid_t), PJE0);
+    fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
+    *PValue = page_index = create_page_index(uuid, ctx);
+    page_index->prev = pg_cache->metrics_index.last_page_index;
+    pg_cache->metrics_index.last_page_index = page_index;
+    page_index->alignment = pa;
+    if(pa)
+        pa->refcount++;
+    uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
+
+    return (STORAGE_METRIC_HANDLE *)page_index;
+}
+
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
+    STORAGE_METRIC_HANDLE *db_metric_handle;
+
+    db_metric_handle = rrdeng_metric_get(db_instance, &rd->metric_uuid, smg);
+    if(!db_metric_handle) {
+        db_metric_handle = rrdeng_metric_get_legacy(db_instance, rrddim_id(rd), rrdset_id(rd->rrdset), smg);
+        if(db_metric_handle) {
+            struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+            uuid_copy(rd->metric_uuid, page_index->id);
         }
-        uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
-        if (NULL == PValue) {
-            uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
-            PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
-            fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
-            *PValue = page_index = create_page_index(&rd->metric_uuid);
-            page_index->prev = pg_cache->metrics_index.last_page_index;
-            pg_cache->metrics_index.last_page_index = page_index;
-            uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
-        }
-    } else {
-        /* There are legacy UUIDs in the database, implement backward compatibility */
+    }
+    if(!db_metric_handle)
+        db_metric_handle = rrdeng_metric_create(db_instance, &rd->metric_uuid, smg);
 
-        rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
-                                                &multihost_legacy_uuid);
+#ifdef NETDATA_INTERNAL_CHECKS
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+    if(uuid_compare(rd->metric_uuid, page_index->id) != 0) {
+        char uuid1[UUID_STR_LEN + 1];
+        char uuid2[UUID_STR_LEN + 1];
 
-        int need_to_store = uuid_compare(rd->metric_uuid, multihost_legacy_uuid);
-
-        uuid_copy(rd->metric_uuid, multihost_legacy_uuid);
-
-        if (unlikely(need_to_store && !ctx->tier))
-            (void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
+        uuid_unparse(rd->metric_uuid, uuid1);
+        uuid_unparse(page_index->id, uuid2);
+        fatal("DBENGINE: uuids do not match, asked for metric '%s', but got page_index of metric '%s'", uuid1, uuid2);
     }
 
-    struct rrdeng_metric_handle *mh = mallocz(sizeof(struct rrdeng_metric_handle));
-    mh->rd = rd;
-    mh->ctx = ctx;
-    mh->rrdeng_uuid = &page_index->id;
-    mh->page_index = page_index;
-    return (STORAGE_METRIC_HANDLE *)mh;
+    struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
+    if(page_index->ctx != ctx)
+        fatal("DBENGINE: mixed up rrdengine instances, asked for metric from %p, got from %p", ctx, page_index->ctx);
+#endif
+
+    return db_metric_handle;
 }
 
+
+// ----------------------------------------------------------------------------
+// collect ops
+
 /*
  * Gets a handle for storing metrics to the database.
  * The handle must be released with rrdeng_store_metric_final().
  */
-STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
-
+STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every) {
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
     struct rrdeng_collect_handle *handle;
-    struct pg_cache_page_index *page_index;
+
+    if(!page_index->alignment)
+        fatal("DBENGINE: metric group is required for collect operations");
 
     handle = callocz(1, sizeof(struct rrdeng_collect_handle));
-    handle->metric_handle = metric_handle;
-    handle->ctx = metric_handle->ctx;
+    handle->page_index = page_index;
     handle->descr = NULL;
     handle->unaligned_page = 0;
+    page_index->latest_update_every_s = update_every;
 
-    page_index = metric_handle->page_index;
     uv_rwlock_wrlock(&page_index->lock);
     ++page_index->writers;
     uv_rwlock_wrunlock(&page_index->lock);
@@ -213,7 +278,7 @@ static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr)
 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle) {
     struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
     // struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
-    struct rrdengine_instance *ctx = handle->ctx;
+    struct rrdengine_instance *ctx = handle->page_index->ctx;
     struct rrdeng_page_descr *descr = handle->descr;
 
     if (unlikely(!ctx)) return;
@@ -226,9 +291,7 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_h
 
         page_is_empty = page_has_only_empty_metrics(descr);
         if (page_is_empty) {
-            debug(D_RRDENGINE, "Page has empty metrics only, deleting:");
-            if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "Page has empty metrics only, deleting", true);
             pg_cache_put(ctx, descr);
             pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
         } else
@@ -241,8 +304,8 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_h
     handle->descr = NULL;
 }
 
-void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
-                              usec_t point_in_time,
+static void rrdeng_store_metric_next_internal(STORAGE_COLLECT_HANDLE *collection_handle,
+                              usec_t point_in_time_ut,
                               NETDATA_DOUBLE n,
                               NETDATA_DOUBLE min_value,
                               NETDATA_DOUBLE max_value,
@@ -251,11 +314,10 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
                               SN_FLAGS flags)
 {
     struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
-    struct rrdengine_instance *ctx = handle->ctx;
+    struct pg_cache_page_index *page_index = handle->page_index;
+    struct rrdengine_instance *ctx = handle->page_index->ctx;
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = handle->descr;
-    RRDDIM *rd = metric_handle->rd;
 
     void *page;
     uint8_t must_flush_unaligned_page = 0, perfect_page_alignment = 0;
@@ -263,21 +325,33 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
     if (descr) {
         /* Make alignment decisions */
 
-        if (descr->page_length == rd->rrdset->rrddim_page_alignment) {
+#ifdef NETDATA_INTERNAL_CHECKS
+        if(descr->end_time_ut + page_index->latest_update_every_s * USEC_PER_SEC != point_in_time_ut) {
+            char buffer[200 + 1];
+            snprintfz(buffer, 200,
+                      "metrics collected are %s, end_time_ut = %llu, point_in_time_ut = %llu, update_every = %u, delta = %llu",
+                      (point_in_time_ut / USEC_PER_SEC - descr->end_time_ut / USEC_PER_SEC > page_index->latest_update_every_s)?"far apart":"not aligned",
+                      descr->end_time_ut / USEC_PER_SEC,
+                      point_in_time_ut / USEC_PER_SEC,
+                      page_index->latest_update_every_s,
+                      point_in_time_ut / USEC_PER_SEC - descr->end_time_ut / USEC_PER_SEC);
+            print_page_cache_descr(descr, buffer, false);
+        }
+#endif
+
+        if (descr->page_length == page_index->alignment->page_length) {
             /* this is the leading dimension that defines chart alignment */
             perfect_page_alignment = 1;
         }
         /* is the metric far enough out of alignment with the others? */
-        if (unlikely(descr->page_length + PAGE_POINT_SIZE_BYTES(descr) < rd->rrdset->rrddim_page_alignment)) {
+        if (unlikely(descr->page_length + PAGE_POINT_SIZE_BYTES(descr) < page_index->alignment->page_length)) {
             handle->unaligned_page = 1;
-            debug(D_RRDENGINE, "Metric page is not aligned with chart:");
-            if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "Metric page is not aligned with chart", true);
         }
         if (unlikely(handle->unaligned_page &&
                      /* did the other metrics change page? */
-                     rd->rrdset->rrddim_page_alignment <= PAGE_POINT_SIZE_BYTES(descr))) {
-            debug(D_RRDENGINE, "Flushing unaligned metric page.");
+                     page_index->alignment->page_length <= PAGE_POINT_SIZE_BYTES(descr))) {
+            print_page_cache_descr(descr, "must_flush_unaligned_page = 1", true);
             must_flush_unaligned_page = 1;
             handle->unaligned_page = 0;
         }
@@ -285,16 +359,21 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
     if (unlikely(NULL == descr ||
                  descr->page_length + PAGE_POINT_SIZE_BYTES(descr) > RRDENG_BLOCK_SIZE ||
                  must_flush_unaligned_page)) {
-        rrdeng_store_metric_flush_current_page(collection_handle);
 
-        page = rrdeng_create_page(ctx, &metric_handle->page_index->id, &descr);
+        if(descr) {
+            print_page_cache_descr(descr, "flushing metric", true);
+            rrdeng_store_metric_flush_current_page(collection_handle);
+        }
+
+        page = rrdeng_create_page(ctx, &page_index->id, &descr);
         fatal_assert(page);
 
+        descr->update_every_s = page_index->latest_update_every_s;
         handle->descr = descr;
 
         handle->page_correlation_id = rrd_atomic_fetch_add(&pg_cache->committed_page_index.latest_corr_id, 1);
 
-        if (0 == rd->rrdset->rrddim_page_alignment) {
+        if (0 == page_index->alignment->page_length) {
             /* this is the leading dimension that defines chart alignment */
             perfect_page_alignment = 1;
         }
@@ -329,13 +408,13 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
         break;
     }
 
-    pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + PAGE_POINT_SIZE_BYTES(descr));
+    pg_cache_atomic_set_pg_info(descr, point_in_time_ut, descr->page_length + PAGE_POINT_SIZE_BYTES(descr));
 
     if (perfect_page_alignment)
-        rd->rrdset->rrddim_page_alignment = descr->page_length;
-    if (unlikely(INVALID_TIME == descr->start_time)) {
+        page_index->alignment->page_length = descr->page_length;
+    if (unlikely(INVALID_TIME == descr->start_time_ut)) {
         unsigned long new_metric_API_producers, old_metric_API_max_producers, ret_metric_API_max_producers;
-        descr->start_time = point_in_time;
+        descr->start_time_ut = point_in_time_ut;
 
         new_metric_API_producers = rrd_atomic_add_fetch(&ctx->stats.metric_API_producers, 1);
         while (unlikely(new_metric_API_producers > (old_metric_API_max_producers = ctx->metric_API_max_producers))) {
@@ -349,20 +428,111 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
             }
         }
 
-        pg_cache_insert(ctx, metric_handle->page_index, descr);
+        pg_cache_insert(ctx, page_index, descr);
     } else {
-        pg_cache_add_new_metric_time(metric_handle->page_index, descr);
+        pg_cache_add_new_metric_time(page_index, descr);
     }
+
+//    {
+//        unsigned char u[16] = { 0x0C, 0x0A, 0x40, 0xD6, 0x2A, 0x43, 0x4A, 0x7C, 0x95, 0xF7, 0xD1, 0x1E, 0x0C, 0x9E, 0x8A, 0xE7 };
+//        if(uuid_compare(u, page_index->id) == 0) {
+//            char buffer[100];
+//            snprintfz(buffer, 100, "store system.cpu, collect:%u, page_index first:%u, last:%u",
+//                      (uint32_t)(point_in_time / USEC_PER_SEC),
+//                      (uint32_t)(page_index->oldest_time / USEC_PER_SEC),
+//                      (uint32_t)(page_index->latest_time / USEC_PER_SEC));
+//
+//            print_page_cache_descr(descr, buffer, false);
+//        }
+//    }
 }
 
+void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
+                              usec_t point_in_time_ut,
+                              NETDATA_DOUBLE n,
+                              NETDATA_DOUBLE min_value,
+                              NETDATA_DOUBLE max_value,
+                              uint16_t count,
+                              uint16_t anomaly_count,
+                              SN_FLAGS flags)
+{
+    struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
+    struct pg_cache_page_index *page_index = handle->page_index;
+    struct rrdeng_page_descr *descr = handle->descr;
+
+    if(likely(descr)) {
+        usec_t last_point_in_time_ut = descr->end_time_ut;
+        usec_t update_every_ut = page_index->latest_update_every_s * USEC_PER_SEC;
+        size_t points_gap = (point_in_time_ut <= last_point_in_time_ut) ?
+                                (size_t)0 :
+                                (size_t)((point_in_time_ut - last_point_in_time_ut) / update_every_ut);
+
+        if(unlikely(points_gap != 1)) {
+            if (unlikely(points_gap <= 0)) {
+                time_t now = now_realtime_sec();
+                static __thread size_t counter = 0;
+                static __thread time_t last_time_logged = 0;
+                counter++;
+
+                if(now - last_time_logged > 600) {
+                    error("DBENGINE: collected point is in the past (repeated %zu times in the last %zu secs). Ignoring these data collection points.",
+                          counter, (size_t)(last_time_logged?(now - last_time_logged):0));
+
+                    last_time_logged = now;
+                    counter = 0;
+                }
+                return;
+            }
+
+            size_t point_size = PAGE_POINT_SIZE_BYTES(descr);
+            size_t page_size_in_points = RRDENG_BLOCK_SIZE / point_size;
+            size_t used_points = descr->page_length / point_size;
+            size_t remaining_points_in_page = page_size_in_points - used_points;
+
+            bool new_point_is_aligned = true;
+            if(unlikely((point_in_time_ut - last_point_in_time_ut) / points_gap != update_every_ut))
+                new_point_is_aligned = false;
+
+            if(unlikely(points_gap > remaining_points_in_page || !new_point_is_aligned)) {
+//                char buffer[200];
+//                snprintfz(buffer, 200, "data collection skipped %zu points, last stored point %llu, new point %llu, update every %d. Cutting page.",
+//                      points_gap, last_point_in_time_ut / USEC_PER_SEC, point_in_time_ut / USEC_PER_SEC, page_index->latest_update_every_s);
+//                print_page_cache_descr(descr, buffer, false);
+
+                rrdeng_store_metric_flush_current_page(collection_handle);
+            }
+            else {
+//                char buffer[200];
+//                snprintfz(buffer, 200, "data collection skipped %zu points, last stored point %llu, new point %llu, update every %d. Filling the gap.",
+//                      points_gap, last_point_in_time_ut / USEC_PER_SEC, point_in_time_ut / USEC_PER_SEC, page_index->latest_update_every_s);
+//                print_page_cache_descr(descr, buffer, false);
+
+                // loop to fill the gap
+                usec_t step_ut = page_index->latest_update_every_s * USEC_PER_SEC;
+                usec_t last_point_filled_ut = last_point_in_time_ut + step_ut;
+
+                while (last_point_filled_ut < point_in_time_ut) {
+                    rrdeng_store_metric_next_internal(
+                        collection_handle, last_point_filled_ut, NAN, NAN, NAN,
+                        1, 0, SN_EMPTY_SLOT);
+
+                    last_point_filled_ut += step_ut;
+                }
+            }
+        }
+    }
+
+    rrdeng_store_metric_next_internal(collection_handle, point_in_time_ut, n, min_value, max_value, count, anomaly_count, flags);
+}
+
+
 /*
  * Releases the database reference from the handle for storing metrics.
  * Returns 1 if it's safe to delete the dimension.
  */
 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
     struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)handle->metric_handle;
-    struct pg_cache_page_index *page_index = metric_handle->page_index;
+    struct pg_cache_page_index *page_index = handle->page_index;
 
     uint8_t can_delete_metric = 0;
 
@@ -377,6 +547,18 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
     return can_delete_metric;
 }
 
+void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every) {
+    struct rrdeng_collect_handle *handle = (struct rrdeng_collect_handle *)collection_handle;
+    struct pg_cache_page_index *page_index = handle->page_index;
+    rrdeng_store_metric_flush_current_page(collection_handle);
+    uv_rwlock_rdlock(&page_index->lock);
+    page_index->latest_update_every_s = update_every;
+    uv_rwlock_rdunlock(&page_index->lock);
+}
+
+// ----------------------------------------------------------------------------
+// query ops
+
 //static inline uint32_t *pginfo_to_dt(struct rrdeng_page_info *page_info)
 //{
 //    return (uint32_t *)&page_info->scratch[0];
@@ -391,49 +573,45 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
  * Gets a handle for loading metrics from the database.
  * The handle must be released with rrdeng_load_metric_final().
  */
-void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
+void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle, time_t start_time_s, time_t end_time_s)
 {
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
-    struct rrdengine_instance *ctx = metric_handle->ctx;
-    RRDDIM *rd = metric_handle->rd;
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+    struct rrdengine_instance *ctx = page_index->ctx;
 
     // fprintf(stderr, "%s: %s/%s start time %ld, end time %ld\n", __FUNCTION__ , rd->rrdset->name, rd->name, start_time, end_time);
 
     struct rrdeng_query_handle *handle;
     unsigned pages_nr;
 
-    rrdimm_handle->start_time = start_time;
-    rrdimm_handle->end_time = end_time;
+    if(!page_index->latest_update_every_s)
+        page_index->latest_update_every_s = default_rrd_update_every;
+
+    rrdimm_handle->start_time_s = start_time_s;
+    rrdimm_handle->end_time_s = end_time_s;
 
     handle = callocz(1, sizeof(struct rrdeng_query_handle));
-    handle->next_page_time = start_time;
-    handle->now = start_time;
-    handle->tier_query_fetch_type = tier_query_fetch_type;
-    // TODO we should store the dt of each page in each page
-    // this will produce wrong values for dt in case the user changes
-    // the update every of the charts or the tier grouping iterations
-    handle->dt_sec = get_tier_grouping(ctx->tier) * (time_t)rd->update_every;
-    handle->dt = handle->dt_sec * USEC_PER_SEC;
+    handle->wanted_start_time_s = start_time_s;
+    handle->now_s = start_time_s;
     handle->position = 0;
     handle->ctx = ctx;
-    handle->metric_handle = metric_handle;
     handle->descr = NULL;
+    handle->dt_s = page_index->latest_update_every_s;
     rrdimm_handle->handle = (STORAGE_QUERY_HANDLE *)handle;
-    pages_nr = pg_cache_preload(ctx, metric_handle->rrdeng_uuid, start_time * USEC_PER_SEC, end_time * USEC_PER_SEC,
+    pages_nr = pg_cache_preload(ctx, &page_index->id, start_time_s * USEC_PER_SEC, end_time_s * USEC_PER_SEC,
                                 NULL, &handle->page_index);
     if (unlikely(NULL == handle->page_index || 0 == pages_nr))
         // there are no metrics to load
-        handle->next_page_time = INVALID_TIME;
+        handle->wanted_start_time_s = INVALID_TIME;
 }
 
-static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
+static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle, bool debug_this __maybe_unused) {
     struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
 
     struct rrdengine_instance *ctx = handle->ctx;
     struct rrdeng_page_descr *descr = handle->descr;
 
     uint32_t page_length;
-    usec_t page_end_time;
+    usec_t page_end_time_ut;
     unsigned position;
 
     if (likely(descr)) {
@@ -445,14 +623,15 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
 
         pg_cache_put(ctx, descr);
         handle->descr = NULL;
-        handle->next_page_time = (handle->page_end_time / USEC_PER_SEC) + 1;
+        handle->wanted_start_time_s = (time_t)((handle->page_end_time_ut / USEC_PER_SEC) + handle->dt_s);
 
-        if (unlikely(handle->next_page_time > rrdimm_handle->end_time))
+        if (unlikely(handle->wanted_start_time_s > rrdimm_handle->end_time_s))
             return 1;
     }
 
-    usec_t next_page_time = handle->next_page_time * USEC_PER_SEC;
-    descr = pg_cache_lookup_next(ctx, handle->page_index, &handle->page_index->id, next_page_time, rrdimm_handle->end_time * USEC_PER_SEC);
+    usec_t wanted_start_time_ut = handle->wanted_start_time_s * USEC_PER_SEC;
+    descr = pg_cache_lookup_next(ctx, handle->page_index, &handle->page_index->id,
+        wanted_start_time_ut, rrdimm_handle->end_time_s * USEC_PER_SEC);
     if (NULL == descr)
         return 1;
 
@@ -461,77 +640,116 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
 #endif
 
     handle->descr = descr;
-    pg_cache_atomic_get_pg_info(descr, &page_end_time, &page_length);
-    if (unlikely(INVALID_TIME == descr->start_time || INVALID_TIME == page_end_time))
+    pg_cache_atomic_get_pg_info(descr, &page_end_time_ut, &page_length);
+    if (unlikely(INVALID_TIME == descr->start_time_ut || INVALID_TIME == page_end_time_ut || 0 == descr->update_every_s)) {
+        error("DBENGINE: discarding invalid page descriptor (start_time = %llu, end_time = %llu, update_every_s = %d)",
+              descr->start_time_ut, page_end_time_ut, descr->update_every_s);
         return 1;
+    }
 
-    if (unlikely(descr->start_time != page_end_time && next_page_time > descr->start_time)) {
+    if (unlikely(descr->start_time_ut != page_end_time_ut && wanted_start_time_ut > descr->start_time_ut)) {
         // we're in the middle of the page somewhere
         unsigned entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
-        position = ((uint64_t)(next_page_time - descr->start_time)) * (entries - 1) /
-                   (page_end_time - descr->start_time);
+        position = ((uint64_t)(wanted_start_time_ut - descr->start_time_ut)) * (entries - 1) /
+                   (page_end_time_ut - descr->start_time_ut);
     }
     else
         position = 0;
 
-    handle->page_end_time = page_end_time;
+    handle->page_end_time_ut = page_end_time_ut;
     handle->page_length = page_length;
+    handle->entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
     handle->page = descr->pg_cache_descr->page;
-    usec_t entries = handle->entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
-    if (likely(entries > 1))
-        handle->dt = (page_end_time - descr->start_time) / (entries - 1);
-    else {
-        // TODO we should store the dt of each page in each page
-        // now we keep the dt of whatever was before
-        ;
-    }
-
-    handle->dt_sec = (time_t)(handle->dt / USEC_PER_SEC);
+    handle->dt_s = descr->update_every_s;
     handle->position = position;
 
+//    if(debug_this)
+//        info("DBENGINE: rrdeng_load_page_next(), "
+//             "position:%d, "
+//             "start_time_ut:%llu, "
+//             "page_end_time_ut:%llu, "
+//             "next_page_time_ut:%llu, "
+//             "in_out:%s"
+//             , position
+//             , descr->start_time_ut
+//             , page_end_time_ut
+//             ,
+//            wanted_start_time_ut, in_out?"true":"false"
+//             );
+
     return 0;
 }
 
 // Returns the metric and sets its timestamp into current_time
 // IT IS REQUIRED TO **ALWAYS** SET ALL RETURN VALUES (current_time, end_time, flags)
 // IT IS REQUIRED TO **ALWAYS** KEEP TRACK OF TIME, EVEN OUTSIDE THE DATABASE BOUNDARIES
-STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle) {
-    struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
+STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle) {
+    struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrddim_handle->handle;
     // struct rrdeng_metric_handle *metric_handle = handle->metric_handle;
 
-    STORAGE_POINT sp;
     struct rrdeng_page_descr *descr = handle->descr;
+    time_t now = handle->now_s + handle->dt_s;
+
+//    bool debug_this = false;
+//    {
+//        unsigned char u[16] = { 0x0C, 0x0A, 0x40, 0xD6, 0x2A, 0x43, 0x4A, 0x7C, 0x95, 0xF7, 0xD1, 0x1E, 0x0C, 0x9E, 0x8A, 0xE7 };
+//        if(uuid_compare(u, handle->page_index->id) == 0) {
+//            char buffer[100];
+//            snprintfz(buffer, 100, "load system.cpu, now:%u, dt:%u, position:%u page_index first:%u, last:%u",
+//                      (uint32_t)(now),
+//                      (uint32_t)(handle->dt_s),
+//                      (uint32_t)(handle->position),
+//                      (uint32_t)(handle->page_index->oldest_time / USEC_PER_SEC),
+//                      (uint32_t)(handle->page_index->latest_time / USEC_PER_SEC));
+//
+//            print_page_cache_descr(descr, buffer, false);
+//            debug_this = true;
+//        }
+//    }
+
+    STORAGE_POINT sp;
     unsigned position = handle->position + 1;
-    time_t now = handle->now + handle->dt_sec;
     storage_number_tier1_t tier1_value;
 
-    if (unlikely(INVALID_TIME == handle->next_page_time)) {
-        handle->next_page_time = INVALID_TIME;
-        handle->now = now;
-        storage_point_empty(sp, now - handle->dt_sec, now);
+    if (unlikely(INVALID_TIME == handle->wanted_start_time_s)) {
+        handle->wanted_start_time_s = INVALID_TIME;
+        handle->now_s = now;
+        storage_point_empty(sp, now - handle->dt_s, now);
         return sp;
     }
 
     if (unlikely(!descr || position >= handle->entries)) {
         // We need to get a new page
-        if(rrdeng_load_page_next(rrdimm_handle)) {
+        if(rrdeng_load_page_next(rrddim_handle, false)) {
             // next calls will not load any more metrics
-            handle->next_page_time = INVALID_TIME;
-            handle->now = now;
-            storage_point_empty(sp, now - handle->dt_sec, now);
+            handle->wanted_start_time_s = INVALID_TIME;
+            handle->now_s = now;
+            storage_point_empty(sp, now - handle->dt_s, now);
             return sp;
         }
 
         descr = handle->descr;
         position = handle->position;
-        now = (time_t)((descr->start_time + position * handle->dt) / USEC_PER_SEC);
+        now = (time_t)((descr->start_time_ut / USEC_PER_SEC) + position * descr->update_every_s);
+
+//        if(debug_this) {
+//            char buffer[100];
+//            snprintfz(buffer, 100, "NEW PAGE system.cpu, now:%u, dt:%u, position:%u page_index first:%u, last:%u",
+//                      (uint32_t)(now),
+//                      (uint32_t)(handle->dt_s),
+//                      (uint32_t)(handle->position),
+//                      (uint32_t)(handle->page_index->oldest_time / USEC_PER_SEC),
+//                      (uint32_t)(handle->page_index->latest_time / USEC_PER_SEC));
+//
+//            print_page_cache_descr(descr, buffer, false);
+//        }
     }
 
-    sp.start_time = now - handle->dt_sec;
+    sp.start_time = now - handle->dt_s;
     sp.end_time = now;
 
     handle->position = position;
-    handle->now = now;
+    handle->now_s = now;
 
     switch(descr->type) {
         case PAGE_METRICS: {
@@ -566,18 +784,26 @@ STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle)
         break;
     }
 
-    if (unlikely(now >= rrdimm_handle->end_time)) {
+    if (unlikely(now >= rrddim_handle->end_time_s)) {
         // next calls will not load any more metrics
-        handle->next_page_time = INVALID_TIME;
+        handle->wanted_start_time_s = INVALID_TIME;
     }
 
+//    if(debug_this)
+//        info("DBENGINE: returning point: "
+//             "time from %ld to %ld   //  query from %ld to %ld   //   wanted_start_time_s %ld"
+//             , sp.start_time, sp.end_time
+//             , rrddim_handle->start_time_s, rrddim_handle->end_time_s
+//             , handle->wanted_start_time_s
+//             );
+
     return sp;
 }
 
 int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle)
 {
     struct rrdeng_query_handle *handle = (struct rrdeng_query_handle *)rrdimm_handle->handle;
-    return (INVALID_TIME == handle->next_page_time);
+    return (INVALID_TIME == handle->wanted_start_time_s);
 }
 
 /*
@@ -602,46 +828,12 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
 }
 
 time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
-
-    struct pg_cache_page_index *page_index = metric_handle->page_index;
-    return page_index->latest_time / USEC_PER_SEC;
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+    return (time_t)(page_index->latest_time_ut / USEC_PER_SEC);
 }
 time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle) {
-    struct rrdeng_metric_handle *metric_handle = (struct rrdeng_metric_handle *)db_metric_handle;
-
-    struct pg_cache_page_index *page_index = metric_handle->page_index;
-    return page_index->oldest_time / USEC_PER_SEC;
-}
-
-int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier)
-{
-    struct page_cache *pg_cache;
-    struct rrdengine_instance *ctx;
-    Pvoid_t *PValue;
-    struct pg_cache_page_index *page_index = NULL;
-
-    ctx = get_rrdeng_ctx_from_host(localhost, tier);
-    if (unlikely(!ctx)) {
-        error("Failed to fetch multidb context");
-        return 1;
-    }
-    pg_cache = &ctx->pg_cache;
-
-    uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
-    PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t));
-    if (likely(NULL != PValue)) {
-        page_index = *PValue;
-    }
-    uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
-
-    if (likely(page_index)) {
-        *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
-        *last_entry_t = page_index->latest_time / USEC_PER_SEC;
-        return 0;
-    }
-
-    return 1;
+    struct pg_cache_page_index *page_index = (struct pg_cache_page_index *)db_metric_handle;
+    return (time_t)(page_index->oldest_time_ut / USEC_PER_SEC);
 }
 
 int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t)
@@ -666,8 +858,8 @@ int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time
     uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
 
     if (likely(page_index)) {
-        *first_entry_t = page_index->oldest_time / USEC_PER_SEC;
-        *last_entry_t = page_index->latest_time / USEC_PER_SEC;
+        *first_entry_t = page_index->oldest_time_ut / USEC_PER_SEC;
+        *last_entry_t = page_index->latest_time_ut / USEC_PER_SEC;
         return 0;
     }
 
@@ -694,7 +886,7 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
 
     debug(D_RRDENGINE, "Created new page:");
     if (unlikely(debug_flags & D_RRDENGINE))
-        print_page_cache_descr(descr);
+        print_page_cache_descr(descr, "", true);
     rrdeng_page_descr_mutex_unlock(ctx, descr);
     *ret_descr = descr;
     return page;
@@ -766,13 +958,13 @@ void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **
 }
 
 /* Gets a reference for the page */
-void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle)
+void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time_ut, void **handle)
 {
     struct rrdeng_page_descr *descr;
     struct page_cache_descr *pg_cache_descr;
 
     debug(D_RRDENGINE, "Reading existing page:");
-    descr = pg_cache_lookup(ctx, NULL, id, point_in_time);
+    descr = pg_cache_lookup(ctx, NULL, id, point_in_time_ut);
     if (NULL == descr) {
         *handle = NULL;
 
@@ -1009,13 +1201,13 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
                 size_t points = descr->page_length / PAGE_POINT_SIZE_BYTES(descr);
 
                 if(likely(points > 1))
-                    update_every_usec = (descr->end_time - descr->start_time) / (points - 1);
+                    update_every_usec = (descr->end_time_ut - descr->start_time_ut) / (points - 1);
                 else {
                     update_every_usec = default_rrd_update_every * get_tier_grouping(ctx->tier) * USEC_PER_SEC;
                     stats.single_point_pages++;
                 }
 
-                time_t duration_secs = (time_t)((descr->end_time - descr->start_time + update_every_usec)/USEC_PER_SEC);
+                time_t duration_secs = (time_t)((descr->end_time_ut - descr->start_time_ut + update_every_usec)/USEC_PER_SEC);
 
                 stats.extents_pages++;
                 stats.pages_uncompressed_bytes += descr->page_length;
@@ -1027,11 +1219,11 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
                 stats.page_types[descr->type].pages_duration_secs += duration_secs;
                 stats.page_types[descr->type].points += points;
 
-                if(!stats.first_t || (descr->start_time - update_every_usec) < stats.first_t)
-                    stats.first_t = (descr->start_time - update_every_usec) / USEC_PER_SEC;
+                if(!stats.first_t || (descr->start_time_ut - update_every_usec) < stats.first_t)
+                    stats.first_t = (descr->start_time_ut - update_every_usec) / USEC_PER_SEC;
 
-                if(!stats.last_t || descr->end_time > stats.last_t)
-                    stats.last_t = descr->end_time / USEC_PER_SEC;
+                if(!stats.last_t || descr->end_time_ut > stats.last_t)
+                    stats.last_t = descr->end_time_ut / USEC_PER_SEC;
             }
         }
     }
@@ -1071,7 +1263,7 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx) {
         }
     }
 
-    stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index));
+    stats.sizeof_metric = struct_natural_alignment(sizeof(struct pg_cache_page_index) + sizeof(struct pg_alignment));
     stats.sizeof_page = struct_natural_alignment(sizeof(struct rrdeng_page_descr));
     stats.sizeof_datafile = struct_natural_alignment(sizeof(struct rrdengine_datafile)) + struct_natural_alignment(sizeof(struct rrdengine_journalfile));
     stats.sizeof_page_in_cache = struct_natural_alignment(sizeof(struct page_cache_descr));

database/engine/rrdengineapi.h

@@ -25,7 +25,7 @@ extern size_t page_type_size[];
 #define PAGE_POINT_SIZE_BYTES(x) page_type_size[(x)->type]
 
 struct rrdeng_region_info {
-    time_t start_time;
+    time_t start_time_s;
     int update_every;
     unsigned points;
 };
@@ -34,20 +34,24 @@ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrde
 void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
                                Word_t page_correlation_id);
 void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **handle);
-void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle);
+void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time_ut, void **handle);
 void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle);
 
-void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid);
+void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
 void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
                                                     uuid_t *ret_uuid);
 
 
-STORAGE_METRIC_HANDLE *rrdeng_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
-void rrdeng_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
+STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
+STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
+STORAGE_METRIC_HANDLE *rrdeng_metric_get_legacy(STORAGE_INSTANCE *db_instance, const char *rd_id, const char *st_id, STORAGE_METRICS_GROUP *smg);
+void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
 
-STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle);
+STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
 void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
-void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE n,
+void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
+void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
                                      NETDATA_DOUBLE min_value,
                                      NETDATA_DOUBLE max_value,
                                      uint16_t count,
@@ -55,12 +59,13 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
                                      SN_FLAGS flags);
 int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
 
-unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
+unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time_s, time_t end_time_s,
                                     struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
 
 void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *rrdimm_handle,
-                                    time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
-STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle);
+                                    time_t start_time_s, time_t end_time_s);
+STORAGE_POINT rrdeng_load_metric_next(struct rrddim_query_handle *rrddim_handle);
+
 
 int rrdeng_load_metric_is_finished(struct rrddim_query_handle *rrdimm_handle);
 void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle);
@@ -75,9 +80,11 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
 
 int rrdeng_exit(struct rrdengine_instance *ctx);
 void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
-int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t, int tier);
 int rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *si, uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t);
 
+extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
+extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
+
 typedef struct rrdengine_size_statistics {
     size_t default_granularity_secs;
 

database/engine/rrdenginelib.c

@@ -4,28 +4,45 @@
 #define BUFSIZE (512)
 
 /* Caller must hold descriptor lock */
-void print_page_cache_descr(struct rrdeng_page_descr *descr)
+void print_page_cache_descr(struct rrdeng_page_descr *descr, const char *msg, bool log_debug)
 {
-    struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
-    char uuid_str[UUID_STR_LEN];
-    char str[BUFSIZE + 1];
-    int pos = 0;
+    if(log_debug && !(debug_flags & D_RRDENGINE))
+        return;
 
-    uuid_unparse_lower(*descr->id, uuid_str);
-    pos += snprintfz(str, BUFSIZE - pos, "page(%p) id=%s\n"
-                                    "--->len:%"PRIu32" time:%"PRIu64"->%"PRIu64" xt_offset:",
-                    pg_cache_descr->page, uuid_str,
-                    descr->page_length,
-                    (uint64_t)descr->start_time,
-                    (uint64_t)descr->end_time);
-    if (!descr->extent) {
-        pos += snprintfz(str + pos, BUFSIZE - pos, "N/A");
-    } else {
-        pos += snprintfz(str + pos, BUFSIZE - pos, "%"PRIu64, descr->extent->offset);
+    BUFFER *wb = buffer_create(512);
+
+    if(!descr) {
+        buffer_sprintf(wb, "DBENGINE: %s : descr is NULL", msg);
+    }
+    else {
+        struct page_cache_descr *pg_cache_descr = descr->pg_cache_descr;
+        char uuid_str[UUID_STR_LEN];
+
+        uuid_unparse_lower(*descr->id, uuid_str);
+        buffer_sprintf(wb, "DBENGINE: %s : page(%p) metric:%s, len:%"PRIu32", time:%"PRIu64"->%"PRIu64", update_every:%u, type:%u, xt_offset:",
+                       msg,
+                       pg_cache_descr->page, uuid_str,
+                       descr->page_length,
+                       (uint64_t)descr->start_time_ut,
+                       (uint64_t)descr->end_time_ut,
+                       (uint32_t)descr->update_every_s,
+                       (uint32_t)descr->type
+                       );
+        if (!descr->extent) {
+            buffer_strcat(wb, "N/A");
+        } else {
+            buffer_sprintf(wb, "%"PRIu64, descr->extent->offset);
+        }
+
+        buffer_sprintf(wb, ", flags:0x%2.2lX refcnt:%u", pg_cache_descr->flags, pg_cache_descr->refcnt);
     }
 
-    snprintfz(str + pos, BUFSIZE - pos, " flags:0x%2.2lX refcnt:%u\n\n", pg_cache_descr->flags, pg_cache_descr->refcnt);
-    debug(D_RRDENGINE, "%s", str);
+    if(log_debug)
+        debug(D_RRDENGINE, "%s", buffer_tostring(wb));
+    else
+        internal_error(true, "%s", buffer_tostring(wb));
+
+    buffer_free(wb);
 }
 
 void print_page_descr(struct rrdeng_page_descr *descr)
@@ -39,8 +56,8 @@ void print_page_descr(struct rrdeng_page_descr *descr)
                                      "--->len:%"PRIu32" time:%"PRIu64"->%"PRIu64" xt_offset:",
                      uuid_str,
                      descr->page_length,
-                     (uint64_t)descr->start_time,
-                     (uint64_t)descr->end_time);
+                     (uint64_t)descr->start_time_ut,
+                     (uint64_t)descr->end_time_ut);
     if (!descr->extent) {
         pos += snprintfz(str + pos, BUFSIZE - pos, "N/A");
     } else {

database/engine/rrdenginelib.h

@@ -83,7 +83,7 @@ static inline void crc32set(void *crcp, uLong crc)
     *(uint32_t *)crcp = crc;
 }
 
-void print_page_cache_descr(struct rrdeng_page_descr *page_cache_descr);
+void print_page_cache_descr(struct rrdeng_page_descr *descr, const char *msg, bool log_debug);
 void print_page_descr(struct rrdeng_page_descr *descr);
 int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size);
 int open_file_for_io(char *path, int flags, uv_file *file, int direct);

database/ram/rrddim_mem.c

@@ -2,18 +2,67 @@
 
 #include "rrddim_mem.h"
 
-// ----------------------------------------------------------------------------
-// RRDDIM legacy data collection functions
+static Pvoid_t rrddim_JudyHS_array = NULL;
+static netdata_rwlock_t rrddim_JudyHS_rwlock = NETDATA_RWLOCK_INITIALIZER;
 
-STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_unused) {
-    return (STORAGE_METRIC_HANDLE *)rd;
+
+// ----------------------------------------------------------------------------
+// metrics groups
+
+STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid __maybe_unused) {
+    return NULL;
 }
 
-void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
+void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
+    // if(!smg) return; // smg may be NULL
     ;
 }
 
-STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle) {
+// ----------------------------------------------------------------------------
+// RRDDIM legacy data collection functions
+
+STORAGE_METRIC_HANDLE *
+rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance __maybe_unused, STORAGE_METRICS_GROUP *smg __maybe_unused) {
+    STORAGE_METRIC_HANDLE *t = rrddim_metric_get(db_instance, &rd->metric_uuid, smg);
+    if(!t) {
+        netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);
+        Pvoid_t *PValue = JudyHSIns(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
+        fatal_assert(NULL == *PValue);
+        *PValue = rd;
+        t = (STORAGE_METRIC_HANDLE *)rd;
+        netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
+    }
+
+    if((RRDDIM *)t != rd)
+        fatal("RRDDIM_MEM: incorrect pointer returned from index.");
+
+    return (STORAGE_METRIC_HANDLE *)rd;
+}
+
+STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid, STORAGE_METRICS_GROUP *smg __maybe_unused) {
+    RRDDIM *rd = NULL;
+    netdata_rwlock_rdlock(&rrddim_JudyHS_rwlock);
+    Pvoid_t *PValue = JudyHSGet(rrddim_JudyHS_array, uuid, sizeof(uuid_t));
+    if (likely(NULL != PValue))
+        rd = *PValue;
+    netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
+
+    return (STORAGE_METRIC_HANDLE *)rd;
+}
+
+void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle __maybe_unused) {
+    RRDDIM *rd = (RRDDIM *)db_metric_handle;
+
+    netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);
+    JudyHSDel(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(uuid_t), PJE0);
+    netdata_rwlock_unlock(&rrddim_JudyHS_rwlock);
+}
+
+void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every __maybe_unused) {
+    rrddim_store_metric_flush(collection_handle);
+}
+
+STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every __maybe_unused) {
     RRDDIM *rd = (RRDDIM *)db_metric_handle;
     rd->db[rd->rrdset->current_entry] = pack_storage_number(NAN, SN_FLAG_NONE);
     struct mem_collect_handle *ch = callocz(1, sizeof(struct mem_collect_handle));
@@ -41,8 +90,11 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
 
 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
     struct mem_collect_handle *ch = (struct mem_collect_handle *)collection_handle;
+
     RRDDIM *rd = ch->rd;
-    memset(rd->db, 0, rd->rrdset->entries * sizeof(storage_number));
+    for(int i = 0; i < rd->rrdset->entries ;i++)
+        rd->db[i] = SN_EMPTY_SLOT;
+
 }
 
 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
@@ -134,14 +186,12 @@ static inline time_t rrddim_slot2time(RRDDIM *rd, size_t slot) {
 // ----------------------------------------------------------------------------
 // RRDDIM legacy database query functions
 
-void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type) {
-    UNUSED(tier_query_fetch_type);
-
+void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) {
     RRDDIM *rd = (RRDDIM *)db_metric_handle;
 
     handle->rd = rd;
-    handle->start_time = start_time;
-    handle->end_time = end_time;
+    handle->start_time_s = start_time;
+    handle->end_time_s = end_time;
     struct mem_query_handle* h = mallocz(sizeof(struct mem_query_handle));
     h->slot           = rrddim_time2slot(rd, start_time);
     h->last_slot      = rrddim_time2slot(rd, end_time);
@@ -200,7 +250,7 @@ STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle) {
 
 int rrddim_query_is_finished(struct rrddim_query_handle *handle) {
     struct mem_query_handle* h = (struct mem_query_handle*)handle->handle;
-    return (h->next_timestamp > handle->end_time);
+    return (h->next_timestamp > handle->end_time_s);
 }
 
 void rrddim_query_finalize(struct rrddim_query_handle *handle) {

database/ram/rrddim_mem.h

@@ -20,10 +20,15 @@ struct mem_query_handle {
     size_t last_slot;
 };
 
-STORAGE_METRIC_HANDLE *rrddim_metric_init(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
-void rrddim_metric_free(STORAGE_METRIC_HANDLE *db_metric_handle);
+STORAGE_METRIC_HANDLE *rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
+STORAGE_METRIC_HANDLE *rrddim_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
+void rrddim_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
 
-STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle);
+STORAGE_METRICS_GROUP *rrddim_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
+void rrddim_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
+
+STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
+void rrddim_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
 void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number,
                                  NETDATA_DOUBLE min_value,
                                  NETDATA_DOUBLE max_value,
@@ -33,7 +38,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle, usec
 void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *collection_handle);
 
-void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
+void rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
 STORAGE_POINT rrddim_query_next_metric(struct rrddim_query_handle *handle);
 int rrddim_query_is_finished(struct rrddim_query_handle *handle);
 void rrddim_query_finalize(struct rrddim_query_handle *handle);

database/rrd.h

@@ -11,6 +11,7 @@ extern "C" {
 // to enable type checking at compile time
 typedef struct storage_instance STORAGE_INSTANCE;
 typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
+typedef struct storage_alignment STORAGE_METRICS_GROUP;
 
 // forward typedefs
 typedef struct rrdhost RRDHOST;
@@ -356,8 +357,8 @@ typedef struct storage_query_handle STORAGE_QUERY_HANDLE;
 // iterator state for RRD dimension data queries
 struct rrddim_query_handle {
     RRDDIM *rd;
-    time_t start_time;
-    time_t end_time;
+    time_t start_time_s;
+    time_t end_time_s;
     STORAGE_QUERY_HANDLE* handle;
 };
 
@@ -401,7 +402,7 @@ typedef struct storage_point {
 // function pointers that handle data collection
 struct rrddim_collect_ops {
     // an initialization function to run before starting collection
-    STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle);
+    STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every);
 
     // run this to store each metric into the database
     void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
@@ -413,12 +414,17 @@ struct rrddim_collect_ops {
     // a finalization function to run after collection is over
     // returns 1 if it's safe to delete the dimension
     int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
+
+    void (*change_collection_frequency)(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
+
+    STORAGE_METRICS_GROUP *(*metrics_group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
+    void (*metrics_group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *sa);
 };
 
 // function pointers that handle database queries
 struct rrddim_query_ops {
     // run this before starting a series of next_metric() database queries
-    void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
+    void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
 
     // run this to load each metric number from the database
     STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
@@ -557,6 +563,8 @@ struct rrdset {
                                                     // netdata will interpolate values for gaps lower than this
                                                     // TODO - use the global - all charts have the same value
 
+    STORAGE_METRICS_GROUP *storage_metrics_groups[RRD_STORAGE_TIERS];
+
     // ------------------------------------------------------------------------
     // linking to siblings and parents
 

database/rrddim.c

@@ -101,7 +101,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
             rd->tiers[tier]->mode = eng->id;
             rd->tiers[tier]->collect_ops = eng->api.collect_ops;
             rd->tiers[tier]->query_ops = eng->api.query_ops;
-            rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]);
+            rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->storage_instance[tier], rd->rrdset->storage_metrics_groups[tier]);
             storage_point_unset(rd->tiers[tier]->virtual_point);
             initialized++;
 
@@ -120,7 +120,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
         size_t initialized = 0;
         for (int tier = 0; tier < storage_tiers; tier++) {
             if (rd->tiers[tier]) {
-                rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
+                rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, st->update_every * storage_tiers_grouping_iterations[tier]);
                 initialized++;
             }
         }
@@ -217,7 +217,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
         STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode);
         if(eng)
-            eng->api.free(rd->tiers[tier]->db_metric_handle);
+            eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
 
         freez(rd->tiers[tier]);
         rd->tiers[tier] = NULL;
@@ -253,7 +253,7 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
         for(int tier = 0; tier < storage_tiers ;tier++) {
             if (rd->tiers[tier])
                 rd->tiers[tier]->db_collection_handle =
-                    rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
+                    rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle, st->update_every * storage_tiers_grouping_iterations[tier]);
         }
 
         rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);

database/rrdset.c

@@ -3,6 +3,7 @@
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
 #include <sched.h>
+#include "storage_engine.h"
 
 // ----------------------------------------------------------------------------
 
@@ -109,7 +110,7 @@ struct rrdset_constructor {
     enum {
         RRDSET_REACT_NONE                   = 0,
         RRDSET_REACT_NEW                    = (1 << 0),
-        RRDSET_REACT_CHART_ARCHIVED_TO_LIVE = (1 << 1),
+        RRDSET_REACT_UPDATED                = (1 << 1),
         RRDSET_REACT_PLUGIN_UPDATED         = (1 << 2),
         RRDSET_REACT_MODULE_UPDATED         = (1 << 3),
         RRDSET_REACT_CHART_ACTIVATED        = (1 << 4),
@@ -176,6 +177,17 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
         uuid_generate(st->chart_uuid);
     update_chart_metadata(&st->chart_uuid, st, string2str(st->parts.id), string2str(st->parts.name));
 
+    // initialize the db tiers
+    {
+        RRD_MEMORY_MODE wanted_mode = ctr->memory_mode;
+        for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
+            STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
+            if(!eng) continue;
+
+            st->storage_metrics_groups[tier] = eng->api.collect_ops.metrics_group_get(host->storage_instance[tier], &st->chart_uuid);
+        }
+    }
+
     rrddim_index_init(st);
 
     // chart variables - we need this for data collection to work (collector given chart variables) - not only health
@@ -203,6 +215,17 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
     rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_ID);
 
+    // cleanup storage engines
+    {
+        RRD_MEMORY_MODE wanted_mode = st->rrd_memory_mode;
+        for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
+            STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
+            if(!eng) continue;
+
+            eng->api.collect_ops.metrics_group_release(host->storage_instance[tier], st->storage_metrics_groups[tier]);
+        }
+    }
+
     // remove it from the name index
     rrdset_index_del_name(host, st);
 
@@ -281,15 +304,27 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
     }
 
     if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
-        ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+        ctr->react_action |= RRDSET_REACT_UPDATED;
 
     if (unlikely(st->priority != ctr->priority)) {
         st->priority = ctr->priority;
-        ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+        ctr->react_action |= RRDSET_REACT_UPDATED;
     }
-    if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != ctr->update_every)) {
+
+    if (unlikely(st->update_every != ctr->update_every)) {
         st->update_every = ctr->update_every;
-        ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+
+        // switch update every to the storage engine
+        RRDDIM *rd;
+        rrddim_foreach_read(rd, st) {
+            for (int tier = 0; tier < storage_tiers; tier++) {
+                if (rd->tiers[tier] && rd->tiers[tier]->db_collection_handle)
+                    rd->tiers[tier]->collect_ops.change_collection_frequency(rd->tiers[tier]->db_collection_handle, st->update_every);
+            }
+        }
+        rrddim_foreach_done(rd);
+
+        ctr->react_action |= RRDSET_REACT_UPDATED;
     }
 
     if(ctr->plugin && *ctr->plugin) {
@@ -312,7 +347,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
         STRING *old_title = st->title;
         st->title = rrd_string_strdupz(ctr->title);
         if(old_title != st->title)
-            ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+            ctr->react_action |= RRDSET_REACT_UPDATED;
         string_freez(old_title);
     }
 
@@ -320,7 +355,7 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
         STRING *old_units = st->units;
         st->units = rrd_string_strdupz(ctr->units);
         if(old_units != st->units)
-            ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+            ctr->react_action |= RRDSET_REACT_UPDATED;
         string_freez(old_units);
     }
 
@@ -328,13 +363,13 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
         STRING *old_context = st->context;
         st->context = rrd_string_strdupz(ctr->context);
         if(old_context != st->context)
-            ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+            ctr->react_action |= RRDSET_REACT_UPDATED;
         string_freez(old_context);
     }
 
     if(st->chart_type != ctr->chart_type) {
         st->chart_type = ctr->chart_type;
-        ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
+        ctr->react_action |= RRDSET_REACT_UPDATED;
     }
 
     rrdset_update_permanent_labels(st);
@@ -357,7 +392,7 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
         rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
     }
 
-    if(ctr->react_action & (RRDSET_REACT_CHART_ARCHIVED_TO_LIVE | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
+    if(ctr->react_action & (RRDSET_REACT_UPDATED | RRDSET_REACT_PLUGIN_UPDATED | RRDSET_REACT_MODULE_UPDATED)) {
         debug(D_METADATALOG, "CHART [%s] metadata updated", rrdset_id(st));
         if(unlikely(update_chart_metadata(&st->chart_uuid, st, ctr->id, ctr->name)))
             error_report("Failed to update chart metadata in the database");
@@ -998,10 +1033,38 @@ static inline time_t tier_next_point_time(RRDDIM *rd, struct rrddim_tier *t, tim
     return now + loop - ((now + loop) % loop);
 }
 
-void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut) {
+void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {
     if (unlikely(!t->next_point_time))
         t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
 
+    if(unlikely(sp.start_time > t->next_point_time)) {
+        if (likely(!storage_point_is_unset(t->virtual_point))) {
+
+            t->collect_ops.store_metric(
+                t->db_collection_handle,
+                t->next_point_time * USEC_PER_SEC,
+                t->virtual_point.sum,
+                t->virtual_point.min,
+                t->virtual_point.max,
+                t->virtual_point.count,
+                t->virtual_point.anomaly_count,
+                t->virtual_point.flags);
+        }
+        else {
+            t->collect_ops.store_metric(
+                t->db_collection_handle,
+                t->next_point_time * USEC_PER_SEC,
+                NAN,
+                NAN,
+                NAN,
+                0,
+                0, SN_FLAG_NONE);
+        }
+
+        t->virtual_point.count = 0; // make the point unset
+        t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
+    }
+
     // merge the dates into our virtual point
     if (unlikely(sp.start_time < t->virtual_point.start_time))
         t->virtual_point.start_time = sp.start_time;
@@ -1027,34 +1090,6 @@ void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, u
             t->virtual_point = sp;
         }
     }
-
-    if(unlikely(sp.end_time >= t->next_point_time)) {
-        if (likely(!storage_point_is_unset(t->virtual_point))) {
-
-            t->collect_ops.store_metric(
-                t->db_collection_handle,
-                now_ut,
-                t->virtual_point.sum,
-                t->virtual_point.min,
-                t->virtual_point.max,
-                t->virtual_point.count,
-                t->virtual_point.anomaly_count,
-                t->virtual_point.flags);
-        }
-        else {
-            t->collect_ops.store_metric(
-                t->db_collection_handle,
-                now_ut,
-                NAN,
-                NAN,
-                NAN,
-                0,
-                0, SN_FLAG_NONE);
-        }
-
-        t->virtual_point.count = 0;
-        t->next_point_time = tier_next_point_time(rd, t, sp.end_time);
-    }
 }
 
 static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
@@ -1062,30 +1097,30 @@ static void store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
     // store the metric on tier 0
     rd->tiers[0]->collect_ops.store_metric(rd->tiers[0]->db_collection_handle, point_end_time_ut, n, 0, 0, 1, 0, flags);
 
+    time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
+
+    STORAGE_POINT sp = {
+        .start_time = now - rd->update_every,
+        .end_time = now,
+        .min = n,
+        .max = n,
+        .sum = n,
+        .count = 1,
+        .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
+        .flags = flags
+    };
+
     for(int tier = 1; tier < storage_tiers ;tier++) {
         if(unlikely(!rd->tiers[tier])) continue;
 
         struct rrddim_tier *t = rd->tiers[tier];
 
-        time_t now = (time_t)(point_end_time_ut / USEC_PER_SEC);
-
         if(!t->last_collected_ut) {
             // we have not collected this tier before
             // let's fill any gap that may exist
             rrdr_fill_tier_gap_from_smaller_tiers(rd, tier, now);
         }
 
-        STORAGE_POINT sp = {
-            .start_time = now - rd->update_every,
-            .end_time = now,
-            .min = n,
-            .max = n,
-            .sum = n,
-            .count = 1,
-            .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,
-            .flags = flags
-        };
-
         t->last_collected_ut = point_end_time_ut;
         store_metric_at_tier(rd, t, sp, point_end_time_ut);
     }

database/sqlite/sqlite_functions.c

@@ -1330,98 +1330,6 @@ failed:
     return;
 }
 
-void free_temporary_host(RRDHOST *host)
-{
-    if (host) {
-        string_freez(host->hostname);
-        string_freez(host->os);
-        string_freez(host->tags);
-        string_freez(host->timezone);
-        string_freez(host->program_name);
-        string_freez(host->program_version);
-        string_freez(host->registry_hostname);
-        freez(host->system_info);
-        freez(host);
-    }
-}
-
-#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;"
-#define SELECT_HOST_BY_UUID "select h.host_id, h.registry_hostname, h.update_every, h.os, h.timezone, h.tags from host h, node_instance ni " \
-    "where (ni.host_id = @host_id or ni.node_id = @host_id) AND ni.host_id = h.host_id;"
-
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    int rc;
-    RRDHOST *host = NULL;
-    uuid_t host_uuid;
-
-    sqlite3_stmt *res = NULL;
-
-    rc = uuid_parse(hostname, host_uuid);
-    if (!rc) {
-        rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to prepare statement to fetch host by uuid");
-            return NULL;
-        }
-        rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to bind host_id parameter to fetch host information");
-            goto failed;
-        }
-    }
-    else {
-        rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to prepare statement to fetch host by hostname");
-            return NULL;
-        }
-        rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to bind hostname parameter to fetch host information");
-            goto failed;
-        }
-    }
-
-    rc = sqlite3_step_monitored(res);
-    if (unlikely(rc != SQLITE_ROW)) {
-        error_report("Failed to find hostname %s", hostname);
-        goto failed;
-    }
-
-    char uuid_str[GUID_LEN + 1];
-    uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 0)), uuid_str);
-
-    host = callocz(1, sizeof(RRDHOST));
-
-    set_host_properties(host, sqlite3_column_int(res, 2),
-                        RRD_MEMORY_MODE_DBENGINE,
-                        (char *) sqlite3_column_text(res, 1),
-                        (char *) sqlite3_column_text(res, 3),
-                        (char *) sqlite3_column_text(res, 5),
-                        (char *) sqlite3_column_text(res, 4),
-                        NULL, 0, NULL, NULL);
-
-    uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
-
-    host->system_info = callocz(1, sizeof(*host->system_info));;
-    rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED);
-
-#ifdef ENABLE_DBENGINE
-    if(dbengine_enabled) {
-        for (int tier = 0; tier < storage_tiers; tier++)
-            host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier];
-    }
-#endif
-
-failed:
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when reading host information");
-
-    return host;
-}
-
 void db_execute(const char *cmd)
 {
     int rc;
@@ -1586,221 +1494,6 @@ static void sql_store_host_label(uuid_t *host_uuid, int source_type, const char
     (void) sql_store_label(res, host_uuid, source_type, label, value);
 }
 
-int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id,
-                                uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid, int tier)
-{
-#ifdef ENABLE_DBENGINE
-    int rc;
-    uuid_t  legacy_uuid;
-    uuid_t  multihost_legacy_uuid;
-    time_t dim_first_entry_t, dim_last_entry_t;
-
-    rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
-    if (unlikely(rc)) {
-        rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid);
-        rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
-        if (likely(rc)) {
-            rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid);
-            rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t, tier);
-            if (likely(!rc))
-                uuid_copy(*rrdeng_uuid, multihost_legacy_uuid);
-        }
-        else
-            uuid_copy(*rrdeng_uuid, legacy_uuid);
-    }
-    else
-        uuid_copy(*rrdeng_uuid, *uuid);
-
-    if (likely(!rc)) {
-        *first_entry_t = MIN(*first_entry_t, dim_first_entry_t);
-        *last_entry_t = MAX(*last_entry_t, dim_last_entry_t);
-    }
-    return rc;
-#else
-    UNUSED(machine_guid);
-    UNUSED(chart_id);
-    UNUSED(dim_id);
-    UNUSED(uuid);
-    UNUSED(first_entry_t);
-    UNUSED(last_entry_t);
-    UNUSED(rrdeng_uuid);
-    return 1;
-#endif
-}
-#include "../storage_engine.h"
-#ifdef ENABLE_DBENGINE
-static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
-{
-    STORAGE_ENGINE *eng = storage_engine_get(RRD_MEMORY_MODE_DBENGINE);
-
-    if (unlikely(!eng))
-        return NULL;
-
-    RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd));
-    rd->rrdset = st;
-    rd->update_every = st->update_every;
-    rd->last_stored_value = NAN;
-    rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
-
-    uuid_copy(rd->metric_uuid, *metric_uuid);
-    rd->id = string_strdupz(id);
-    rd->name = string_strdupz(name);
-
-    for(int tier = 0; tier < storage_tiers ;tier++) {
-        rd->tiers[tier] = onewayalloc_callocz(owa, 1, sizeof(*rd->tiers[tier]));
-        rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
-        rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
-        rd->tiers[tier]->mode = RRD_MEMORY_MODE_DBENGINE;
-        rd->tiers[tier]->query_ops.init = rrdeng_load_metric_init;
-        rd->tiers[tier]->query_ops.next_metric = rrdeng_load_metric_next;
-        rd->tiers[tier]->query_ops.is_finished = rrdeng_load_metric_is_finished;
-        rd->tiers[tier]->query_ops.finalize = rrdeng_load_metric_finalize;
-        rd->tiers[tier]->query_ops.latest_time = rrdeng_metric_latest_time;
-        rd->tiers[tier]->query_ops.oldest_time = rrdeng_metric_oldest_time;
-        rd->tiers[tier]->db_metric_handle = eng->api.init(rd, st->rrdhost->storage_instance[tier]);
-    }
-
-    return rd;
-}
-#endif
-
-#define SELECT_CHART_CONTEXT  "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, " \
-    "c.context, CASE WHEN d.options = 'hidden' THEN 1 else 0 END from chart c, " \
-    "dimension d, host h " \
-    "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \
-    "order by c.chart_id asc, c.type||c.id desc;"
-
-#define SELECT_CHART_SINGLE  "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, " \
-    "c.context, CASE WHEN d.options = 'hidden' THEN 1 else 0 END from chart c, " \
-    "dimension d, host h " \
-    "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
-    "order by c.chart_id asc, c.type||'.'||c.id desc;"
-
-void sql_build_context_param_list(ONEWAYALLOC  *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart)
-{
-#ifdef ENABLE_DBENGINE
-    int rc;
-
-    if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
-        return;
-
-    if (unlikely(!(*param_list))) {
-        *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
-        (*param_list)->first_entry_t = LONG_MAX;
-        (*param_list)->last_entry_t = 0;
-        (*param_list)->rd = NULL;
-        (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE;
-        if (chart)
-            (*param_list)->flags |= CONTEXT_FLAGS_CHART;
-        else
-            (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT;
-    }
-
-    sqlite3_stmt *res = NULL;
-
-    if (context)
-        rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0);
-    else
-        rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to fetch host archived charts");
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host parameter to fetch archived charts");
-        goto failed;
-    }
-
-    if (context)
-        rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC);
-    else
-        rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host parameter to fetch archived charts");
-        goto failed;
-    }
-
-    RRDSET *st = NULL;
-    char machine_guid[GUID_LEN + 1];
-    uuid_unparse_lower(host->host_uuid, machine_guid);
-    uuid_t rrdeng_uuid;
-    uuid_t chart_id;
-
-    while (sqlite3_step_monitored(res) == SQLITE_ROW) {
-        char id[512];
-        sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1));
-
-        if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
-            if (unlikely(st && !st->counter)) {
-                string_freez(st->context);
-                string_freez(st->name);
-                string_freez(st->id);
-                onewayalloc_freez(owa, st);
-            }
-            st = onewayalloc_callocz(owa, 1, sizeof(*st));
-            char n[RRD_ID_LENGTH_MAX + 1];
-
-            snprintfz(
-                n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
-                (char *)sqlite3_column_text(res, 3));
-            st->name = string_strdupz(n);
-            st->update_every = sqlite3_column_int(res, 6);
-            st->counter = 0;
-            if (chart) {
-                st->context = string_strdupz((char *)sqlite3_column_text(res, 8));
-                st->id = string_strdupz(chart);
-            } else
-                st->id = string_strdupz(n);
-
-            uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
-            st->last_entry_t = 0;
-            st->rrdhost = host;
-        }
-
-        if (unlikely(find_dimension_first_last_t(machine_guid, (char *)rrdset_name(st), (char *)sqlite3_column_text(res, 1),
-                (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t,
-                &rrdeng_uuid, 0)))
-            continue;
-
-        st->counter++;
-        st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
-
-        RRDDIM *rd = create_rrdim_entry(owa, st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
-        if (unlikely(!rd))
-            continue;
-        if (sqlite3_column_int(res, 9) == 1)
-            rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
-        rd->next = (*param_list)->rd;
-        (*param_list)->rd = rd;
-    }
-    if (st) {
-        if (!st->counter) {
-            string_freez(st->context);
-            string_freez(st->name);
-            string_freez(st->id);
-            onewayalloc_freez(owa,st);
-        }
-        else
-            if (!st->context && context)
-                st->context = string_strdupz(context);
-    }
-
-failed:
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when reading archived charts");
-#else
-    UNUSED(param_list);
-    UNUSED(host);
-    UNUSED(context);
-    UNUSED(chart);
-#endif
-    return;
-}
-
-
 #define SQL_STORE_CLAIM_ID  "insert into node_instance " \
     "(host_id, claim_id, date_created) values (@host_id, @claim_id, unixepoch()) " \
     "on conflict(host_id) do update set claim_id = excluded.claim_id;"

database/sqlite/sqlite_functions.h

@@ -85,7 +85,6 @@ int find_uuid_type(uuid_t *uuid);
 
 void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
 
-RRDHOST *sql_create_host_by_uuid(char *guid);
 int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
 int execute_insert(sqlite3_stmt *res);
 void db_execute(const char *cmd);
@@ -94,7 +93,6 @@ void add_migrated_file(char *path, uint64_t file_size);
 void db_unlock(void);
 void db_lock(void);
 void delete_dimension_uuid(uuid_t *dimension_uuid);
-void sql_build_context_param_list(ONEWAYALLOC  *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart);
 void store_claim_id(uuid_t *host_id, uuid_t *claim_id);
 int update_node_id(uuid_t *host_id, uuid_t *node_id);
 int get_node_id(uuid_t *host_id, uuid_t *node_id);
@@ -104,7 +102,6 @@ struct node_instance_list *get_node_list(void);
 void sql_load_node_id(RRDHOST *host);
 int sql_set_dimension_option(uuid_t *dim_uuid, char *option);
 char *get_hostname_by_node_id(char *node_id);
-void free_temporary_host(RRDHOST *host);
 int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
 void migrate_localhost(uuid_t *host_uuid);
 void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info);

database/storage_engine.c

@@ -10,7 +10,10 @@
     .init = rrddim_collect_init,\
     .store_metric = rrddim_collect_store_metric,\
     .flush = rrddim_store_metric_flush,\
-    .finalize = rrddim_collect_finalize\
+    .finalize = rrddim_collect_finalize, \
+    .change_collection_frequency = rrddim_store_metric_change_collection_frequency, \
+    .metrics_group_get = rrddim_metrics_group_get, \
+    .metrics_group_release = rrddim_metrics_group_release, \
 }
 
 #define im_query_ops { \
@@ -27,8 +30,11 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_NONE,
         .name = RRD_MEMORY_MODE_NONE_NAME,
         .api = {
-            .init = rrddim_metric_init,
-            .free = rrddim_metric_free,
+            .metric_get = rrddim_metric_get,
+            .metric_get_or_create = rrddim_metric_get_or_create,
+            .metric_release = rrddim_metric_release,
+            .group_get = rrddim_metrics_group_get,
+            .group_release = rrddim_metrics_group_release,
             .collect_ops = im_collect_ops,
             .query_ops = im_query_ops
         }
@@ -37,8 +43,11 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_RAM,
         .name = RRD_MEMORY_MODE_RAM_NAME,
         .api = {
-            .init = rrddim_metric_init,
-            .free = rrddim_metric_free,
+            .metric_get = rrddim_metric_get,
+            .metric_get_or_create = rrddim_metric_get_or_create,
+            .metric_release = rrddim_metric_release,
+            .group_get = rrddim_metrics_group_get,
+            .group_release = rrddim_metrics_group_release,
             .collect_ops = im_collect_ops,
             .query_ops = im_query_ops
         }
@@ -47,8 +56,11 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_MAP,
         .name = RRD_MEMORY_MODE_MAP_NAME,
         .api = {
-            .init = rrddim_metric_init,
-            .free = rrddim_metric_free,
+            .metric_get = rrddim_metric_get,
+            .metric_get_or_create = rrddim_metric_get_or_create,
+            .metric_release = rrddim_metric_release,
+            .group_get = rrddim_metrics_group_get,
+            .group_release = rrddim_metrics_group_release,
             .collect_ops = im_collect_ops,
             .query_ops = im_query_ops
         }
@@ -57,8 +69,11 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_SAVE,
         .name = RRD_MEMORY_MODE_SAVE_NAME,
         .api = {
-            .init = rrddim_metric_init,
-            .free = rrddim_metric_free,
+            .metric_get = rrddim_metric_get,
+            .metric_get_or_create = rrddim_metric_get_or_create,
+            .metric_release = rrddim_metric_release,
+            .group_get = rrddim_metrics_group_get,
+            .group_release = rrddim_metrics_group_release,
             .collect_ops = im_collect_ops,
             .query_ops = im_query_ops
         }
@@ -67,8 +82,11 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_ALLOC,
         .name = RRD_MEMORY_MODE_ALLOC_NAME,
         .api = {
-            .init = rrddim_metric_init,
-            .free = rrddim_metric_free,
+            .metric_get = rrddim_metric_get,
+            .metric_get_or_create = rrddim_metric_get_or_create,
+            .metric_release = rrddim_metric_release,
+            .group_get = rrddim_metrics_group_get,
+            .group_release = rrddim_metrics_group_release,
             .collect_ops = im_collect_ops,
             .query_ops = im_query_ops
         }
@@ -78,13 +96,19 @@ static STORAGE_ENGINE engines[] = {
         .id = RRD_MEMORY_MODE_DBENGINE,
         .name = RRD_MEMORY_MODE_DBENGINE_NAME,
         .api = {
-            .init = rrdeng_metric_init,
-            .free = rrdeng_metric_free,
+            .metric_get = rrdeng_metric_get,
+            .metric_get_or_create = rrdeng_metric_get_or_create,
+            .metric_release = rrdeng_metric_release,
+            .group_get = rrdeng_metrics_group_get,
+            .group_release = rrdeng_metrics_group_release,
             .collect_ops = {
                 .init = rrdeng_store_metric_init,
                 .store_metric = rrdeng_store_metric_next,
                 .flush = rrdeng_store_metric_flush_current_page,
-                .finalize = rrdeng_store_metric_finalize
+                .finalize = rrdeng_store_metric_finalize,
+                .change_collection_frequency = rrdeng_store_metric_change_collection_frequency,
+                .metrics_group_get = rrdeng_metrics_group_get,
+                .metrics_group_release = rrdeng_metrics_group_release,
             },
             .query_ops = {
                 .init = rrdeng_load_metric_init,

database/storage_engine.h

@@ -10,8 +10,16 @@ typedef struct storage_engine STORAGE_ENGINE;
 // ------------------------------------------------------------------------
 // function pointers for all APIs provided by a storge engine
 typedef struct storage_engine_api {
-    STORAGE_METRIC_HANDLE *(*init)(RRDDIM *rd, STORAGE_INSTANCE *instance);
-    void (*free)(STORAGE_METRIC_HANDLE *);
+    // metric management
+    STORAGE_METRIC_HANDLE *(*metric_get)(STORAGE_INSTANCE *instance, uuid_t *uuid, STORAGE_METRICS_GROUP *smg);
+    STORAGE_METRIC_HANDLE *(*metric_get_or_create)(RRDDIM *rd, STORAGE_INSTANCE *instance, STORAGE_METRICS_GROUP *smg);
+    void (*metric_release)(STORAGE_METRIC_HANDLE *);
+
+    // metrics groups management
+    STORAGE_METRICS_GROUP *(*group_get)(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
+    void (*group_release)(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);
+
+    // operations
     struct rrddim_collect_ops collect_ops;
     struct rrddim_query_ops query_ops;
 } STORAGE_ENGINE_API;

exporting/process_data.c

@@ -125,7 +125,7 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
     size_t counter = 0;
     NETDATA_DOUBLE sum = 0;
 
-    for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) {
+    for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before); !rd->tiers[0]->query_ops.is_finished(&handle);) {
         STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
 
         if (unlikely(storage_point_is_empty(sp))) {

libnetdata/clocks/clocks.c

@@ -11,14 +11,14 @@ usec_t clock_monotonic_resolution = 1000;
 usec_t clock_realtime_resolution = 1000;
 
 #ifndef HAVE_CLOCK_GETTIME
-inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
+inline int clock_gettime(clockid_t clk_id __maybe_unused, struct timespec *ts) {
     struct timeval tv;
     if(unlikely(gettimeofday(&tv, NULL) == -1)) {
         error("gettimeofday() failed.");
         return -1;
     }
     ts->tv_sec = tv.tv_sec;
-    ts->tv_nsec = (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC;
+    ts->tv_nsec = (long)((tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC);
     return 0;
 }
 #endif

libnetdata/log/log.c

@@ -3,6 +3,10 @@
 #include <daemon/main.h>
 #include "../libnetdata.h"
 
+#ifdef HAVE_BACKTRACE
+#include <execinfo.h>
+#endif
+
 int web_server_is_multithreaded = 1;
 
 const char *program_name = "";
@@ -857,6 +861,18 @@ static void crash_netdata(void) {
 }
 #endif
 
+#ifdef HAVE_BACKTRACE
+#define BT_BUF_SIZE 100
+static void print_call_stack(void) {
+    int nptrs;
+    void *buffer[BT_BUF_SIZE];
+
+    nptrs = backtrace(buffer, BT_BUF_SIZE);
+    if(nptrs)
+        backtrace_symbols_fd(buffer, nptrs, fileno(stderr));
+}
+#endif
+
 void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
     // save a copy of errno - just in case this function generates a new error
     int __errno = errno;
@@ -904,6 +920,10 @@ void fatal_int( const char *file, const char *function, const unsigned long line
     snprintfz(action_result, 60, "%s:%s", program_name, strncmp(thread_tag, "STREAM_RECEIVER", strlen("STREAM_RECEIVER")) ? thread_tag : "[x]");
     send_statistics("FATAL", action_result, action_data);
 
+#ifdef HAVE_BACKTRACE
+    print_call_stack();
+#endif
+
 #ifdef NETDATA_INTERNAL_CHECKS
     crash_netdata();
 #endif

libnetdata/storage_number/storage_number.c

@@ -176,7 +176,8 @@ int print_netdata_double(char *str, NETDATA_DOUBLE value) {
 #ifdef STORAGE_WITH_MATH
     fractional = modfndd(value, &integral) * 10000000.0;
 #else
-    fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
+    integral = (NETDATA_DOUBLE)((unsigned long long)(value * 10000000ULL) / 10000000ULL);
+    fractional = (NETDATA_DOUBLE)((unsigned long long)(value * 10000000ULL) % 10000000ULL);
 #endif
 
     unsigned long long integral_int = (unsigned long long)integral;

ml/Query.h

@@ -20,7 +20,7 @@ public:
     }
 
     void init(time_t AfterT, time_t BeforeT) {
-        Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM);
+        Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT);
         Initialized = true;
     }
 

streaming/receiver.c

@@ -126,18 +126,15 @@ PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins
     return PARSER_RC_ERROR;
 }
 
-#define CLAIMED_ID_MIN_WORDS 3
 PARSER_RC streaming_claimed_id(char **words, void *user, PLUGINSD_ACTION *plugins_action)
 {
     UNUSED(plugins_action);
 
-    int i;
     uuid_t uuid;
     RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
 
-    for (i = 0; words[i]; i++) ;
-    if (i != CLAIMED_ID_MIN_WORDS) {
-        error("Command CLAIMED_ID came malformed %d parameters are expected but %d received", CLAIMED_ID_MIN_WORDS - 1, i - 1);
+    if (!words[1] || !words[2]) {
+        error("Command CLAIMED_ID came malformed, uuid = '%s', claim_id = '%s'", words[1]?words[1]:"[unset]", words[2]?words[2]:"[unset]");
         return PARSER_RC_ERROR;
     }
 

web/api/formatters/rrd2json.c

@@ -27,7 +27,7 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
         if(archive_mode) {
             STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
             if (eng)
-                eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
+                eng->api.metric_release(temp_rd->tiers[tier]->db_metric_handle);
         }
 
         onewayalloc_freez(owa, temp_rd->tiers[tier]);

web/api/queries/query.c

@@ -966,7 +966,7 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
 
     ops->tier = ops->plan.data[plan_id].tier;
     ops->tier_ptr = ops->rd->tiers[ops->tier];
-    ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
+    ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
     ops->next_metric = ops->tier_ptr->query_ops.next_metric;
     ops->is_finished = ops->tier_ptr->query_ops.is_finished;
     ops->finalize = ops->tier_ptr->query_ops.finalize;
@@ -1156,6 +1156,10 @@ static inline void rrd2rrdr_do_dimension(
     , time_t after_wanted
     , time_t before_wanted
 ){
+//    bool debug_this = false;
+//    if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
+//        debug_this = true;
+
     time_t max_date = 0,
            min_date = 0;
 
@@ -1210,6 +1214,9 @@ static inline void rrd2rrdr_do_dimension(
                 new_point = QUERY_POINT_EMPTY;
                 new_point.start_time = last1_point.end_time;
                 new_point.end_time   = now_end_time;
+//
+//                if(debug_this) info("QUERY: is finished() returned true");
+//
                 break;
             }
 
@@ -1225,6 +1232,10 @@ static inline void rrd2rrdr_do_dimension(
                 new_point.anomaly    = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
                 query_point_set_id(new_point, ops.db_total_points_read);
 
+//                if(debug_this)
+//                    info("QUERY: got point %zu, from time %ld to %ld   //   now from %ld to %ld   //   query from %ld to %ld",
+//                         new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
+//
                 // set the right value to the point we got
                 if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
 
@@ -1268,7 +1279,7 @@ static inline void rrd2rrdr_do_dimension(
 
             // check if the db is advancing the query
             if(unlikely(new_point.end_time <= last1_point.end_time)) {
-                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
+                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
                                rrdset_name(rd->rrdset), rrddim_name(rd), new_point.id, new_point.start_time, new_point.end_time,
                                last1_point.id, last1_point.end_time, now_start_time, now_end_time);
 
@@ -1466,7 +1477,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
         long before_wanted = smaller_tier_last_time;
 
         struct rrddim_tier *tmp = rd->tiers[tr];
-        tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
+        tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
 
         size_t points = 0;
 

web/api/tests/valid_urls.c

@@ -19,19 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;

web/api/tests/web_api.c

@@ -19,18 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;

web/api/web_api_v1.c

@@ -721,10 +721,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
 
         if (likely(context_param_list && context_param_list->rd))  // Just set the first one
             st = context_param_list->rd->rrdset;
-        else {
-            if (!chart_label_key && !chart_labels_filter)
-                sql_build_context_param_list(owa, &context_param_list, host, context, NULL);
-        }
     }
     else {
         st = rrdset_find(host, chart);
@@ -732,8 +728,6 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
             st = rrdset_find_byname(host, chart);
         if (likely(st))
             st->last_accessed_time = now_realtime_sec();
-        else
-            sql_build_context_param_list(owa, &context_param_list, host, NULL, chart);
     }
 
     if (!st) {

web/server/web_client.c

@@ -1308,14 +1308,6 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
         host = rrdhost_find_by_hostname(tok);
         if (!host)
             host = rrdhost_find_by_guid(tok);
-        if (!host) {
-            host = sql_create_host_by_uuid(tok);
-            if (likely(host)) {
-                int rc = web_client_process_url(host, w, url);
-                free_temporary_host(host);
-                return rc;
-            }
-        }
         if (host) return web_client_process_url(host, w, url);
     }