Improve estimated disk space usage for data file rotation (#20019)

* Add locks to protect the datafile disk space calculation

* Avoid queueing RRDENG_OPCODE_DATABASE_ROTATE operations needlessly

src/daemon/pulse/pulse-db-dbengine-retention.c

@@ -56,7 +56,7 @@ void dbengine_retention_statistics(bool extended __maybe_unused) {
         //       get_used_disk_space is used to determine if database cleanup (file rotation should happen)
         //                           and adds to the disk space used the desired file size of the active
         //                           datafile
-        uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier]);
+        uint64_t disk_space = rrdeng_get_used_disk_space(multidb_ctx[tier], false);
         //uint64_t disk_space = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si);
 
         uint64_t config_disk_space = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si);

src/database/engine/rrdengine.c

@@ -913,7 +913,7 @@ static void *extent_write_tp_worker(
 
     extent_flush_to_open(ctx, xt_io_descr, df_write_error);
 
-     if(ctx_is_available_for_queries(ctx))
+     if(ctx_is_available_for_queries(ctx) && rrdeng_ctx_tier_cap_exceeded(ctx))
         rrdeng_enq_cmd(ctx, RRDENG_OPCODE_DATABASE_ROTATE, NULL, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL, NULL);
 done:
     if(completion)
@@ -1655,13 +1655,22 @@ static void *cleanup_tp_worker(struct rrdengine_instance *ctx __maybe_unused, vo
     return data;
 }
 
-uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx)
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx, bool having_lock)
 {
     uint64_t active_space = 0;
 
+    if (!having_lock)
+        uv_rwlock_rdlock(&ctx->datafiles.rwlock);
+
     if (ctx->datafiles.first && ctx->datafiles.first->prev)
         active_space = ctx->datafiles.first->prev->pos;
 
+    if (!having_lock)
+        uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
+
+    // calculate the estimated disk space based on the expected final size of the datafile
+    // We cant know the final v1/v2 journal size -- we let the current v1 size be part of the calculation by not
+    // including it in the active_space
     uint64_t estimated_disk_space = ctx_current_disk_space_get(ctx) + rrdeng_target_data_file_size(ctx) - active_space;
 
     uint64_t database_space = get_total_database_space();
@@ -1688,15 +1697,17 @@ static time_t get_tier_retention(struct rrdengine_instance *ctx)
 // Check if disk or retention time cap reached
 bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx)
 {
-    if(!ctx->datafiles.first)
-        // no datafiles available
-        return false;
 
-    if(!ctx->datafiles.first->next)
-        // only 1 datafile available
+    uv_rwlock_rdlock(&ctx->datafiles.rwlock);
+    if (!ctx->datafiles.first || !ctx->datafiles.first->next) {
+        uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
         return false;
+    }
+
+    uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx, true);
+
+    uv_rwlock_rdunlock(&ctx->datafiles.rwlock);
 
-    uint64_t estimated_disk_space = rrdeng_get_used_disk_space(ctx);
     time_t retention = get_tier_retention(ctx);
 
     if (ctx->config.max_retention_s && retention > ctx->config.max_retention_s)

src/database/engine/rrdengine.h

@@ -546,7 +546,7 @@ static inline int journal_metric_uuid_compare(const void *key, const void *metri
 }
 
 // --------------------------------------------------------------------------------------------------------------------
-uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx);
+uint64_t rrdeng_get_used_disk_space(struct rrdengine_instance *ctx, bool having_lock);
 void rrdeng_calculate_tier_disk_space_percentage(void);
 uint64_t rrdeng_get_directory_free_bytes_space(struct rrdengine_instance *ctx);
 void dbengine_shutdown();