archive/netdata_netdata
0
0
Fork 0
mirror of https://github.com/netdata/netdata.git synced 2025-04-16 10:31:07 +00:00
Code Issues Projects Releases Wiki Activity

procfile: more comfortable initial settings and faster/fewer reallocs ()

Browse source
This commit is contained in:
Costa Tsaousis 2022-05-02 14:33:41 +03:00 • committed by GitHub
parent f389f8c7f0
commit 43b9fdc213
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 53 additions and 51 deletions

71
libnetdata/libnetdata.c
View file

@ -31,6 +31,8 @@ const char *program_version = VERSION;
// routines.
#ifdef NETDATA_LOG_ALLOCATIONS
#warning NETDATA_LOG_ALLOCATIONS ENABLED - set log_thread_memory_allocations=1 on any thread to log all its allocations - or use log_allocations() to log them on demand
static __thread struct memory_statistics {
volatile ssize_t malloc_calls_made;
volatile ssize_t calloc_calls_made;
@ -44,23 +46,14 @@ static __thread struct memory_statistics {
__thread size_t log_thread_memory_allocations = 0;
static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) {
inline void log_allocations_int(const char *file, const char *function, const unsigned long line) {
static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n",
fprintf(stderr, "%s MEMORY ALLOCATIONS: (%04lu@%s:%s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
netdata_thread_tag(),
log_thread_memory_allocations,
line, file, function,
type, size
);
fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
netdata_thread_tag(),
log_thread_memory_allocations,
line, file, function,
(memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
(memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
type,
memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
@ -79,12 +72,12 @@ static inline void mmap_accounting(size_t size) {
}
void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
if(log_thread_memory_allocations) {
memory_statistics.memory_calls_made++;
memory_statistics.malloc_calls_made++;
memory_statistics.allocated_memory += size;
print_allocations(file, function, line, "malloc()", size);
}
memory_statistics.memory_calls_made++;
memory_statistics.malloc_calls_made++;
memory_statistics.allocated_memory += size;
if(log_thread_memory_allocations)
log_allocations_int(file, function, line);
size_t *n = (size_t *)malloc(sizeof(size_t) + size);
if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
@ -95,12 +88,11 @@ void *mallocz_int(const char *file, const char *function, const unsigned long li
void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
size = nmemb * size;
if(log_thread_memory_allocations) {
memory_statistics.memory_calls_made++;
memory_statistics.calloc_calls_made++;
memory_statistics.allocated_memory += size;
print_allocations(file, function, line, "calloc()", size);
}
memory_statistics.memory_calls_made++;
memory_statistics.calloc_calls_made++;
memory_statistics.allocated_memory += size;
if(log_thread_memory_allocations)
log_allocations_int(file, function, line);
size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
@ -118,12 +110,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
n = realloc(n, sizeof(size_t) + size);
if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
if(log_thread_memory_allocations) {
memory_statistics.memory_calls_made++;
memory_statistics.realloc_calls_made++;
memory_statistics.allocated_memory += (size - old_size);
print_allocations(file, function, line, "realloc()", size - old_size);
}
memory_statistics.memory_calls_made++;
memory_statistics.realloc_calls_made++;
memory_statistics.allocated_memory += (size - old_size);
if(log_thread_memory_allocations)
log_allocations_int(file, function, line);
*n = size;
return (void *)&n[1];
@ -132,12 +123,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
size_t size = strlen(s) + 1;
if(log_thread_memory_allocations) {
memory_statistics.memory_calls_made++;
memory_statistics.strdup_calls_made++;
memory_statistics.allocated_memory += size;
print_allocations(file, function, line, "strdup()", size);
}
memory_statistics.memory_calls_made++;
memory_statistics.strdup_calls_made++;
memory_statistics.allocated_memory += size;
if(log_thread_memory_allocations)
log_allocations_int(file, function, line);
size_t *n = (size_t *)malloc(sizeof(size_t) + size);
if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size);
@ -155,12 +145,11 @@ void freez_int(const char *file, const char *function, const unsigned long line,
n--;
size_t size = *n;
if(log_thread_memory_allocations) {
memory_statistics.memory_calls_made++;
memory_statistics.free_calls_made++;
memory_statistics.allocated_memory -= size;
print_allocations(file, function, line, "free()", size);
}
memory_statistics.memory_calls_made++;
memory_statistics.free_calls_made++;
memory_statistics.allocated_memory -= size;
if(log_thread_memory_allocations)
log_allocations_int(file, function, line);
free(n);
}

3
libnetdata/libnetdata.h
View file

@ -233,12 +233,15 @@ extern __thread size_t log_thread_memory_allocations;
#define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size)
#define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size)
#define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr)
#define log_allocations() log_allocations_int(__FILE__, __FUNCTION__, __LINE__)
extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s);
extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size);
extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size);
extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size);
extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr);
extern void log_allocations_int(const char *file, const char *function, const unsigned long line);
#else // NETDATA_LOG_ALLOCATIONS
extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;

30
libnetdata/procfile/procfile.c
View file

@ -4,9 +4,9 @@
#define PF_PREFIX "PROCFILE"
#define PFWORDS_INCREASE_STEP 200
#define PFLINES_INCREASE_STEP 10
#define PROCFILE_INCREMENT_BUFFER 512
#define PFWORDS_INCREASE_STEP 2000
#define PFLINES_INCREASE_STEP 200
#define PROCFILE_INCREMENT_BUFFER 4096
int procfile_open_flags = O_RDONLY;
@ -48,9 +48,12 @@ static inline void pfwords_add(procfile *ff, char *str) {
pfwords *fw = ff->words;
if(unlikely(fw->len == fw->size)) {
// debug(D_PROCFILE, PF_PREFIX ": expanding words");
size_t minimum = PFWORDS_INCREASE_STEP;
size_t optimal = fw->size / 2;
size_t wanted = (optimal > minimum)?optimal:minimum;
ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *));
fw->size += PFWORDS_INCREASE_STEP;
ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *));
fw->size += wanted;
}
fw->words[fw->len++] = str;
@ -90,9 +93,12 @@ static inline size_t *pflines_add(procfile *ff) {
pflines *fl = ff->lines;
if(unlikely(fl->len == fl->size)) {
// debug(D_PROCFILE, PF_PREFIX ": expanding lines");
size_t minimum = PFLINES_INCREASE_STEP;
size_t optimal = fl->size / 2;
size_t wanted = (optimal > minimum)?optimal:minimum;
ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline));
fl->size += PFLINES_INCREASE_STEP;
ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline));
fl->size += wanted;
}
ffline *ffl = &fl->lines[fl->len++];
@ -272,9 +278,13 @@ procfile *procfile_readall(procfile *ff) {
ssize_t x = ff->size - s;
if(unlikely(!x)) {
debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff));
ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER);
ff->size += PROCFILE_INCREMENT_BUFFER;
size_t minimum = PROCFILE_INCREMENT_BUFFER;
size_t optimal = ff->size / 2;
size_t wanted = (optimal > minimum)?optimal:minimum;
debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted);
ff = reallocz(ff, sizeof(procfile) + ff->size + wanted);
ff->size += wanted;
}
debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));