Add noise stats, autolevel, and squelch (#1763)

conf/rtl_433.example.conf

@@ -85,12 +85,32 @@ sample_rate   250k
 # see "decoder" section below.
 
 # as command line option:
-#   [-Y level=<dB level>] Manual detection level used to determine pulses (-1.0 to -30.0) (0=auto)
+#   [-Y auto | classic | minmax] FSK pulse detector mode.
+#pulse_detect auto
+
+# as command line option:
+#   [-Y level=<dB level>] Manual detection level used to determine pulses (-1.0 to -30.0) (0=auto).
 #pulse_detect level=0
 
 # as command line option:
-#   [-Y auto | classic | minmax] FSK pulse detector mode.
+#   [-Y minlevel=<dB level>] Manual minimum detection level used to determine pulses (-1.0 to -99.0).
-#pulse_detect auto
+#pulse_detect minlevel=-12
+
+# as command line option:
+#   [-Y minsnr=<dB level>] Minimum SNR to determine pulses (1.0 to 99.0).
+#pulse_detect minsnr=9
+
+# as command line option:
+#   [-Y autolevel] Set minlevel automatically based on average estimated noise.
+#pulse_detect autolevel
+
+# as command line option:
+#   [-Y squelch] Skip frames below estimated noise level to lower cpu load.
+#pulse_detect squelch
+
+# as command line option:
+#   [-Y ampest | magest] Choose amplitude or magnitude level estimator.
+#pulse_detect magest
 
 # as command line option:
 #   [-n <value>] Specify number of samples to take (each sample is 2 bytes: 1 each of I & Q)
@@ -111,21 +131,25 @@ analyze_pulses false
 #out_block_size
 
 # as command line option:
-#   [-M time[:<options>]|protocol|level|stats|bits] Add various metadata to every output line.
+#   [-M time[:<options>]|protocol|level|noise[:<secs>]|stats|bits] Add various metadata to every output line.
 # Use "time" to add current date and time meta data (preset for live inputs).
 # Use "time:rel" to add sample position meta data (preset for read-file and stdin).
 # Use "time:unix" to show the seconds since unix epoch as time meta data.
 # Use "time:iso" to show the time with ISO-8601 format (YYYY-MM-DD"T"hh:mm:ss).
 # Use "time:off" to remove time meta data.
 # Use "time:usec" to add microseconds to date time meta data.
+# Use "time:tz" to output time with timezone offset.
 # Use "time:utc" to output time in UTC.
 #   (this may also be accomplished by invocation with TZ environment variable set).
 #   "usec" and "utc" can be combined with other options, eg. "time:unix:utc:usec".
 # Use "protocol" / "noprotocol" to output the decoder protocol number meta data.
 # Use "level" to add Modulation, Frequency, RSSI, SNR, and Noise meta data.
+# Use "noise[:secs]" to report estimated noise level at intervals (default: 10 seconds).
 # Use "stats[:[<level>][:<interval>]]" to report statistics (default: 600 seconds).
 #   level 0: no report, 1: report successful devices, 2: report active devices, 3: report all
+# Use "bits" to add bit representation to code outputs (for debug).
 report_meta level
+report_meta noise
 report_meta stats
 report_meta time:usec
 report_meta protocol
@@ -172,6 +196,22 @@ signal_grabber none
 # default is "kv", multiple outputs can be used.
 output json
 
+# as command line option:
+#   [-K FILE | PATH | <tag> | <key>=<tag>] Add an expanded token or fixed tag to every output line.
+# If <tag> is "FILE" or "PATH" an expanded token will be added.
+# The <tag> can also be a GPSd URL, e.g.
+#   -K gpsd,lat,lon" (report lat and lon keys from local gpsd)
+#   -K loc=gpsd,lat,lon" (report lat and lon in loc object)
+#   -K gpsd" (full json TPV report, in default "gps" object)
+#   -K foo=gpsd://127.0.0.1:2947" (with key and address)
+#   -K bar=gpsd,nmea" (NMEA deault GPGGA report)
+#   -K rmc=gpsd,nmea,filter='$GPRMC'" (NMEA GPRMC report)
+# Also <tag> can be a generic tcp address, e.g.
+#   -K foo=tcp:localhost:4000" (read lines as TCP client)
+#   -K bar=tcp://127.0.0.1:3000,init='subscribe tags\\r\\n'"
+#   -K baz=tcp://127.0.0.1:5000,filter='a prefix to match'"
+#output_tag mytag
+
 # as command line option:
 #   [-C] native|si|customary Convert units in decoded output.
 # default is "native"

include/baseband.h

@@ -22,15 +22,16 @@
     @param iq_buf input samples (I/Q samples in interleaved uint8)
     @param[out] y_buf output buffer
     @param len number of samples to process
+    @return the average level in dB
 */
-void envelope_detect(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float envelope_detect(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
 
 // for evaluation
-void envelope_detect_nolut(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float envelope_detect_nolut(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
-void magnitude_est_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float magnitude_est_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
-void magnitude_true_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float magnitude_true_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len);
-void magnitude_est_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float magnitude_est_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len);
-void magnitude_true_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len);
+float magnitude_true_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len);
 
 #define AMP_TO_DB(x) (10.0f * ((x) > 0 ? log10f(x) : 0) - 42.1442f)  // 10*log10f(16384.0f)
 #define MAG_TO_DB(x) (20.0f * ((x) > 0 ? log10f(x) : 0) - 84.2884f)  // 20*log10f(16384.0f)

include/optparse.h

@@ -106,7 +106,7 @@ uint32_t atouint32_metric(char const *str, char const *error_hint);
 ///
 /// @param str character string to parse
 /// @param error_hint prepended to error output
-/// @return parsed number value
+/// @return parsed number value in seconds
 int atoi_time(char const *str, char const *error_hint);
 
 /// Similar to strsep.

include/r_private.h

@@ -17,7 +17,11 @@
 #include "compat_time.h"
 
 struct dm_state {
+    float auto_level;
+    float squelch_offset;
     float level_limit;
+    float noise_level;
+    float min_level_auto;
     float min_level;
     float min_snr;
     float low_pass;

include/rtl_433.h

@@ -77,6 +77,7 @@ typedef struct r_cfg {
     int verbose_bits;
     conversion_mode_t conversion_mode;
     int report_meta;
+    int report_noise;
     int report_protocol;
     time_mode_t report_time;
     int report_time_hires;

src/baseband.c

@@ -32,32 +32,39 @@ static void calc_squares()
 
 // This will give a noisy envelope of OOK/ASK signals.
 // Subtract the bias (-128) and get an envelope estimation.
-void envelope_detect(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float envelope_detect(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         y_buf[i] = scaled_squares[iq_buf[2 * i ]] + scaled_squares[iq_buf[2 * i + 1]];
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? AMP_TO_DB((float)sum / len) : AMP_TO_DB(1);
 }
 
 /// This will give a noisy envelope of OOK/ASK signals.
 /// Subtracts the bias (-128) and calculates the norm (scaled by 16384).
 /// Using a LUT is slower for O1 and above.
-void envelope_detect_nolut(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float envelope_detect_nolut(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         int16_t x = 127 - iq_buf[2 * i];
         int16_t y = 127 - iq_buf[2 * i + 1];
         y_buf[i]  = x * x + y * y; // max 32768, fs 16384
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? AMP_TO_DB((float)sum / len) : AMP_TO_DB(1);
 }
 
 /// 122/128, 51/128 Magnitude Estimator for CU8 (SIMD has min/max).
 /// Note that magnitude emphasizes quiet signals / deemphasizes loud signals.
-void magnitude_est_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float magnitude_est_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         uint16_t x = abs(iq_buf[2 * i] - 128);
         uint16_t y = abs(iq_buf[2 * i + 1] - 128);
@@ -65,24 +72,30 @@ void magnitude_est_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
         uint16_t mx = x > y ? x : y;
         uint16_t mag_est = 122 * mx + 51 * mi;
         y_buf[i] = mag_est; // max 22144, fs 16384
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? MAG_TO_DB((float)sum / len) : MAG_TO_DB(1);
 }
 
 /// True Magnitude for CU8 (sqrt can SIMD but float is slow).
-void magnitude_true_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float magnitude_true_cu8(uint8_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         int16_t x = iq_buf[2 * i] - 128;
         int16_t y = iq_buf[2 * i + 1] - 128;
         y_buf[i]  = (uint16_t)(sqrt(x * x + y * y) * 128.0); // max 181, scaled 23170, fs 16384
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? MAG_TO_DB((float)sum / len) : MAG_TO_DB(1);
 }
 
 /// 122/128, 51/128 Magnitude Estimator for CS16 (SIMD has min/max).
-void magnitude_est_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float magnitude_est_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         uint32_t x = abs(iq_buf[2 * i]);
         uint32_t y = abs(iq_buf[2 * i + 1]);
@@ -90,18 +103,23 @@ void magnitude_est_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len)
         uint32_t mx = x > y ? x : y;
         uint32_t mag_est = 122 * mx + 51 * mi;
         y_buf[i] = mag_est >> 8; // max 5668864, scaled 22144, fs 16384
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? MAG_TO_DB((float)sum / len) : MAG_TO_DB(1);
 }
 
 /// True Magnitude for CS16 (sqrt can SIMD but float is slow).
-void magnitude_true_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len)
+float magnitude_true_cs16(int16_t const *iq_buf, uint16_t *y_buf, uint32_t len)
 {
     unsigned long i;
+    uint32_t sum = 0;
     for (i = 0; i < len; i++) {
         int32_t x = iq_buf[2 * i];
         int32_t y = iq_buf[2 * i + 1];
         y_buf[i]  = (int)sqrt(x * x + y * y) >> 1; // max 46341, scaled 23170, fs 16384
+        sum += y_buf[i];
     }
+    return len > 0 && sum >= len ? MAG_TO_DB((float)sum / len) : MAG_TO_DB(1);
 }
 
 

src/rtl_433.c

@@ -116,9 +116,13 @@ static void usage(int exit_code)
             "       Specify a negative number to disable a device decoding protocol (can be used multiple times)\n"
             "  [-G] Enable blacklisted device decoding protocols, for testing only.\n"
             "  [-X <spec> | help] Add a general purpose decoder (prepend -R 0 to disable all decoders)\n"
-            "  [-Y level=<dB level>] Manual detection level used to determine pulses (-1.0 to -30.0) (0=auto)\n"
-            "  [-n <value>] Specify number of samples to take (each sample is 2 bytes: 1 each of I & Q)\n"
             "  [-Y auto | classic | minmax] FSK pulse detector mode.\n"
+            "  [-Y level=<dB level>] Manual detection level used to determine pulses (-1.0 to -30.0) (0=auto).\n"
+            "  [-Y minlevel=<dB level>] Manual minimum detection level used to determine pulses (-1.0 to -99.0).\n"
+            "  [-Y minsnr=<dB level>] Minimum SNR to determine pulses (1.0 to 99.0).\n"
+            "  [-Y autolevel] Set minlevel automatically based on average estimated noise.\n"
+            "  [-Y squelch] Skip frames below estimated noise level to reduce cpu load.\n"
+            "  [-Y ampest | magest] Choose amplitude or magnitude level estimator.\n"
             "\t\t= Analyze/Debug options =\n"
             "  [-a] Analyze mode. Print a textual description of the signal.\n"
             "  [-A] Pulse Analyzer. Enable pulse analysis and decode attempt.\n"
@@ -134,9 +138,10 @@ static void usage(int exit_code)
             "  [-F kv | json | csv | mqtt | influx | syslog | null | help] Produce decoded output in given format.\n"
             "       Append output to file with :<filename> (e.g. -F csv:log.csv), defaults to stdout.\n"
             "       Specify host/port for syslog with e.g. -F syslog:127.0.0.1:1514\n"
-            "  [-M time[:<options>] | protocol | level | stats | bits | help] Add various meta data to each output.\n"
+            "  [-M time[:<options>] | protocol | level | noise[:secs] | stats | bits | help] Add various meta data to each output.\n"
             "  [-K FILE | PATH | <tag> | <key>=<tag>] Add an expanded token or fixed tag to every output line.\n"
             "  [-C native | si | customary] Convert units in decoded output.\n"
+            "  [-n <value>] Specify number of samples to take (each sample is an I/Q pair)\n"
             "  [-T <seconds>] Specify number of seconds to run, also 12:34 or 1h23m45s\n"
             "  [-E hop | quit] Hop/Quit after outputting successful event(s)\n"
             "  [-h] Output this usage help and exit\n"
@@ -253,7 +258,7 @@ static void help_meta(void)
 {
     term_help_printf(
             "\t\t= Meta information option =\n"
-            "  [-M time[:<options>]|protocol|level|stats|bits] Add various metadata to every output line.\n"
+            "  [-M time[:<options>]|protocol|level|noise[:<secs>]|stats|bits] Add various metadata to every output line.\n"
             "\tUse \"time\" to add current date and time meta data (preset for live inputs).\n"
             "\tUse \"time:rel\" to add sample position meta data (preset for read-file and stdin).\n"
             "\tUse \"time:unix\" to show the seconds since unix epoch as time meta data.\n"
@@ -266,6 +271,7 @@ static void help_meta(void)
             "\t\t\"usec\" and \"utc\" can be combined with other options, eg. \"time:unix:utc:usec\".\n"
             "\tUse \"protocol\" / \"noprotocol\" to output the decoder protocol number meta data.\n"
             "\tUse \"level\" to add Modulation, Frequency, RSSI, SNR, and Noise meta data.\n"
+            "\tUse \"noise[:secs]\" to report estimated noise level at intervals (default: 10 seconds).\n"
             "\tUse \"stats[:[<level>][:<interval>]]\" to report statistics (default: 600 seconds).\n"
             "\t  level 0: no report, 1: report successful devices, 2: report active devices, 3: report all\n"
             "\tUse \"bits\" to add bit representation to code outputs (for debug).\n");
@@ -325,6 +331,8 @@ static void sdr_callback(unsigned char *iq_buf, uint32_t len, void *ctx)
         cfg->exit_async = 1;
     }
 
+    // save last frame time to see if a new second started
+    time_t last_frame_sec = demod->now.tv_sec;
     get_time_now(&demod->now);
 
     n_samples = len / demod->sample_size;
@@ -349,18 +357,51 @@ static void sdr_callback(unsigned char *iq_buf, uint32_t len, void *ctx)
     }
 
     // AM demodulation
+    float avg_db;
     if (demod->sample_size == 2) { // CU8
         if (demod->use_mag_est) {
             //magnitude_true_cu8(iq_buf, demod->buf.temp, n_samples);
-            magnitude_est_cu8(iq_buf, demod->buf.temp, n_samples);
+            avg_db = magnitude_est_cu8(iq_buf, demod->buf.temp, n_samples);
         }
         else { // amp est
-            envelope_detect(iq_buf, demod->buf.temp, n_samples);
+            avg_db = envelope_detect(iq_buf, demod->buf.temp, n_samples);
         }
     } else { // CS16
         //magnitude_true_cs16((int16_t *)iq_buf, demod->buf.temp, n_samples);
-        magnitude_est_cs16((int16_t *)iq_buf, demod->buf.temp, n_samples);
+        avg_db = magnitude_est_cs16((int16_t *)iq_buf, demod->buf.temp, n_samples);
     }
+
+    //fprintf(stderr, "noise level: %.1f dB current: %.1f dB min level: %.1f dB\n", demod->noise_level, avg_db, demod->min_level_auto);
+    if (demod->min_level_auto == 0.0f) {
+        demod->min_level_auto = demod->min_level;
+    }
+    if (demod->noise_level == 0.0f) {
+        demod->noise_level = demod->min_level_auto - 3.0f;
+    }
+    int noise_only = avg_db < demod->noise_level + 3.0f; // or demod->min_level_auto?
+    // always process frames if loader, dumper, or analyzers are in use, otherwise skip silent frames
+    int process_frame = demod->squelch_offset <= 0 || !noise_only || demod->load_info.format || demod->analyze_pulses || demod->dumper.len || demod->samp_grab;
+    if (noise_only) {
+        demod->noise_level = (demod->noise_level * 7 + avg_db) / 8; // average over 8 frames
+        // If there is a significant change in noise level
+        if (fabsf(demod->min_level_auto - demod->noise_level - 3.0f) > 1.0f) {
+            // If auto_level is on and the noise level is well below min_level
+            if (demod->auto_level > 0 && demod->noise_level < demod->min_level - 3.0f) {
+                demod->min_level_auto = demod->noise_level + 3.0f;
+                fprintf(stderr, "Estimated noise level is %.1f dB, adjusting minimum detection level to %.1f dB\n", demod->noise_level, demod->min_level_auto);
+                pulse_detect_set_levels(demod->pulse_detect, demod->use_mag_est, demod->level_limit, demod->min_level_auto, demod->min_snr, demod->detect_verbosity);
+            }
+            else {
+                fprintf(stderr, "Estimated noise level is %.1f dB, minimum detection level is %.1f dB\n", demod->noise_level, demod->min_level);
+            }
+        }
+    }
+    // Report noise every report_noise seconds, but only for the first frame that second
+    if (cfg->report_noise && last_frame_sec != demod->now.tv_sec && demod->now.tv_sec % cfg->report_noise == 0) {
+        fprintf(stderr, "Current level %.1f dB, estimated noise %.1f dB\n", avg_db, demod->noise_level);
+    }
+
+    if (process_frame)
     baseband_low_pass_filter(demod->buf.temp, demod->am_buf, n_samples, &demod->lowpass_filter_state);
 
     // FM demodulation
@@ -373,7 +414,7 @@ static void sdr_callback(unsigned char *iq_buf, uint32_t len, void *ctx)
             fpdm = FSK_PULSE_DETECT_OLD;
     }
 
-    if (demod->enable_FM_demod) {
+    if (demod->enable_FM_demod && process_frame) {
         float low_pass = demod->low_pass != 0.0f ? demod->low_pass : fpdm ? 0.2f : 0.1f;
         if (demod->sample_size == 2) { // CU8
             baseband_demod_FM(iq_buf, demod->buf.fm, n_samples, cfg->samp_rate, low_pass, &demod->demod_FM_state);
@@ -405,7 +446,7 @@ static void sdr_callback(unsigned char *iq_buf, uint32_t len, void *ctx)
                 break;
             }
         }
-        while (package_type) {
+        while (package_type && process_frame) {
             int p_events = 0; // Sensor events successfully detected per package
             package_type = pulse_detect_package(demod->pulse_detect, demod->am_buf, demod->buf.fm, n_samples, cfg->samp_rate, cfg->input_pos, &demod->pulse_data, &demod->fsk_pulse_data, fpdm);
             if (package_type) {
@@ -964,6 +1005,8 @@ static void parse_conf_option(r_cfg_t *cfg, int opt, char *arg)
             cfg->report_protocol = 0;
         else if (!strcasecmp(arg, "level"))
             cfg->report_meta = 1;
+        else if (!strcasecmp(arg, "noise"))
+            cfg->report_noise = atoiv(arg_param(arg), 10); // atoi_time_default()
         else if (!strcasecmp(arg, "bits"))
             cfg->verbose_bits = 1;
         else if (!strcasecmp(arg, "description"))
@@ -976,7 +1019,7 @@ static void parse_conf_option(r_cfg_t *cfg, int opt, char *arg)
             // there also should be options to set wether to flush on report
             char *p = arg_param(arg);
             cfg->report_stats = atoiv(p, 1);
-            cfg->stats_interval = atoiv(arg_param(p), 600);
+            cfg->stats_interval = atoiv(arg_param(p), 600); // atoi_time_default()
             time(&cfg->stats_time);
             cfg->stats_time += cfg->stats_interval;
         }
@@ -1105,7 +1148,11 @@ static void parse_conf_option(r_cfg_t *cfg, int opt, char *arg)
             usage(1);
         char *p = arg;
         while (p && *p) {
-            if (!strncmp(p, "auto", 4))
+            if (!strncasecmp(p, "autolevel", 9))
+                cfg->demod->auto_level = atoiv(arg_param(arg), 1); // arg_float_default(p + 9, "-Y autolevel: ");
+            else if (!strncasecmp(p, "squelch", 7))
+                cfg->demod->squelch_offset = atoiv(arg_param(arg), 1); // arg_float_default(p + 7, "-Y squelch: ");
+            else if (!strncmp(p, "auto", 4))
                 cfg->fsk_pulse_detect_mode = FSK_PULSE_DETECT_AUTO;
             else if (!strncmp(p, "classic", 7))
                 cfg->fsk_pulse_detect_mode = FSK_PULSE_DETECT_OLD;

src/term_ctl.c

@@ -377,7 +377,7 @@ int term_help_puts(void *ctx, char const *buf)
             state = 1;
             next_color = 5;
         }
-        else if ((state == 1 || state == 2) && *p == ']' && (p[1] == ' ' || p[1] == '\n' || p[1] == '\0')) {
+        else if ((state == 1 || state == 2) && *p == ']' && ((p[1] == ' ' && p[2] != '|') || p[1] == '\n' || p[1] == '\0')) {
             state = 0;
             set_color = 0;
         }