Add "Filter by keywords in the message body" feature

cc: #653

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 - Implement the "Add Check" dialog
 - Include last ping type in Slack, Mattermost, Discord notifications
 - Upgrade to cron-descriptor 1.2.30
+- Add "Filter by keywords in the message body" feature (#653)
 
 ### Bug Fixes
 - Fix the display of ignored pings with non-zero exit status

hc/api/management/commands/smtpd.py

@@ -7,6 +7,7 @@ from smtpd import SMTPServer
 from django.core.management.base import BaseCommand
 from django.db import connections
 from hc.api.models import Check
+from hc.lib.html import html2text
 
 RE_UUID = re.compile(
     "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$"
@@ -22,6 +23,23 @@ def _match(subject, keywords):
     return False
 
 
+def _to_text(message, with_subject, with_body):
+    chunks = []
+    if with_subject:
+        chunks.append(message.get("subject", ""))
+    if with_body:
+        plain_mime_part = message.get_body(("plain",))
+        if plain_mime_part:
+            chunks.append(plain_mime_part.get_content())
+
+        html_mime_part = message.get_body(("html",))
+        if html_mime_part:
+            html = html_mime_part.get_content()
+            chunks.append(html2text(html))
+
+    return "\n".join(chunks)
+
+
 def _process_message(remote_addr, mailfrom, mailto, data):
     to_parts = mailto.split("@")
     code = to_parts[0]
@@ -35,15 +53,16 @@ def _process_message(remote_addr, mailfrom, mailto, data):
         return f"Check not found: {code}"
 
     action = "success"
-    if check.subject or check.subject_fail:
-        action = "ign"
-        # Specify policy, the default policy does not decode encoded headers:
+    if check.filter_subject or check.filter_body:
         data_str = data.decode(errors="replace")
-        parsed = email.message_from_string(data_str, policy=email.policy.SMTP)
-        subject = parsed.get("subject", "")
-        if check.subject_fail and _match(subject, check.subject_fail):
+        # Specify policy, the default policy does not decode encoded headers:
+        message = email.message_from_string(data_str, policy=email.policy.SMTP)
+        text = _to_text(message, check.filter_subject, check.filter_body)
+
+        action = "ign"
+        if check.failure_kw and _match(text, check.failure_kw):
             action = "fail"
-        elif check.subject and _match(subject, check.subject):
+        elif check.success_kw and _match(text, check.success_kw):
             action = "success"
 
     ua = "Email from %s" % mailfrom

hc/api/migrations/0087_check_failure_kw_check_filter_body_and_more.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.0.6 on 2022-07-12 10:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0086_remove_check_last_ping_was_fail_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='check',
+            name='failure_kw',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name='check',
+            name='filter_body',
+            field=models.BooleanField(null=True),
+        ),
+        migrations.AddField(
+            model_name='check',
+            name='filter_subject',
+            field=models.BooleanField(null=True),
+        ),
+        migrations.AddField(
+            model_name='check',
+            name='success_kw',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+        migrations.AlterField(
+            model_name='channel',
+            name='kind',
+            field=models.CharField(choices=[('email', 'Email'), ('webhook', 'Webhook'), ('hipchat', 'HipChat'), ('slack', 'Slack'), ('pd', 'PagerDuty'), ('pagertree', 'PagerTree'), ('pagerteam', 'Pager Team'), ('po', 'Pushover'), ('pushbullet', 'Pushbullet'), ('opsgenie', 'Opsgenie'), ('victorops', 'Splunk On-Call'), ('discord', 'Discord'), ('telegram', 'Telegram'), ('sms', 'SMS'), ('zendesk', 'Zendesk'), ('trello', 'Trello'), ('matrix', 'Matrix'), ('whatsapp', 'WhatsApp'), ('apprise', 'Apprise'), ('mattermost', 'Mattermost'), ('msteams', 'Microsoft Teams'), ('shell', 'Shell Command'), ('zulip', 'Zulip'), ('spike', 'Spike'), ('call', 'Phone Call'), ('linenotify', 'LINE Notify'), ('signal', 'Signal'), ('gotify', 'Gotify')], max_length=20),
+        ),
+    ]

hc/api/migrations/0088_fill_kw.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.0.6 on 2022-07-12 10:02
+
+from django.db import migrations
+from django.db.models import Case, F, When
+
+
+def fill_kw(apps, schema_editor):
+    Check = apps.get_model("api", "Check")
+    Check.objects.update(
+        success_kw=F("subject"),
+        failure_kw=F("subject_fail"),
+        filter_subject=Case(
+            When(subject__gt="", then=True),
+            When(subject_fail__gt="", then=True),
+            default=False,
+        ),
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0087_check_failure_kw_check_filter_body_and_more"),
+    ]
+
+    operations = [migrations.RunPython(fill_kw, migrations.RunPython.noop)]

hc/api/models.py

@@ -98,6 +98,10 @@ class Check(models.Model):
     tz = models.CharField(max_length=36, default="UTC")
     subject = models.CharField(max_length=200, blank=True)
     subject_fail = models.CharField(max_length=200, blank=True)
+    filter_subject = models.BooleanField(null=True)
+    filter_body = models.BooleanField(null=True)
+    success_kw = models.CharField(max_length=200, blank=True, null=True)
+    failure_kw = models.CharField(max_length=200, blank=True, null=True)
     methods = models.CharField(max_length=30, blank=True)
     manual_resume = models.BooleanField(default=False)
 

hc/api/tests/test_smtpd.py

@@ -12,6 +12,28 @@ Subject: %s
 ...
 """.strip()
 
+HTML_PAYLOAD_TMPL = """
+From: "User Name" <username@gmail.com>
+To: "John Smith" <john@example.com>
+Subject: %s
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="=-eZCfVHHsxj32NZLHPpkbTCXb9C529OcJ23WKzQ=="
+
+--=-eZCfVHHsxj32NZLHPpkbTCXb9C529OcJ23WKzQ==
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+Plain text here
+
+--=-eZCfVHHsxj32NZLHPpkbTCXb9C529OcJ23WKzQ==
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+%s
+
+--=-eZCfVHHsxj32NZLHPpkbTCXb9C529OcJ23WKzQ==--
+""".strip()
+
 
 @override_settings(S3_BUCKET=None)
 class SmtpdTestCase(BaseTestCase):
@@ -30,7 +52,8 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, None)
 
     def test_it_handles_subject_filter_match(self):
-        self.check.subject = "SUCCESS"
+        self.check.filter_subject = True
+        self.check.success_kw = "SUCCESS"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[SUCCESS] Backup completed"
@@ -41,8 +64,43 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.ua, "Email from foo@example.org")
         self.assertEqual(ping.kind, None)
 
+    def test_it_handles_body_filter_match(self):
+        self.check.filter_body = True
+        self.check.success_kw = "SUCCESS"
+        self.check.save()
+
+        body = PAYLOAD_TMPL % "Subject goes here"
+        body += "\nBody goes here, SUCCESS.\n"
+        _process_message("1.2.3.4", "foo@example.org", self.email, body.encode("utf8"))
+
+        ping = Ping.objects.latest("id")
+        self.assertEqual(ping.kind, None)
+
+    def test_it_handles_html_body_filter_match(self):
+        self.check.filter_body = True
+        self.check.success_kw = "SUCCESS"
+        self.check.save()
+
+        body = HTML_PAYLOAD_TMPL % ("Subject", "<b>S</b>UCCESS")
+        _process_message("1.2.3.4", "foo@example.org", self.email, body.encode("utf8"))
+
+        ping = Ping.objects.latest("id")
+        self.assertEqual(ping.kind, None)
+
+    def test_it_handles_body_filter_miss(self):
+        self.check.filter_body = True
+        self.check.success_kw = "SUCCESS"
+        self.check.save()
+
+        body = PAYLOAD_TMPL % "Subject goes here"
+        _process_message("1.2.3.4", "foo@example.org", self.email, body.encode("utf8"))
+
+        ping = Ping.objects.latest("id")
+        self.assertEqual(ping.kind, "ign")
+
     def test_it_handles_subject_filter_miss(self):
-        self.check.subject = "SUCCESS"
+        self.check.filter_subject = True
+        self.check.success_kw = "SUCCESS"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[FAIL] Backup did not complete"
@@ -54,7 +112,8 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, "ign")
 
     def test_it_handles_subject_fail_filter_match(self):
-        self.check.subject_fail = "FAIL"
+        self.check.filter_subject = True
+        self.check.failure_kw = "FAIL"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[FAIL] Backup did not complete"
@@ -66,7 +125,8 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, "fail")
 
     def test_it_handles_subject_fail_filter_miss(self):
-        self.check.subject_fail = "FAIL"
+        self.check.filter_subject = True
+        self.check.failure_kw = "FAIL"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[SUCCESS] Backup completed"
@@ -78,7 +138,8 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, "ign")
 
     def test_it_handles_multiple_subject_keywords(self):
-        self.check.subject = "SUCCESS, OK"
+        self.check.filter_subject = True
+        self.check.success_kw = "SUCCESS, OK"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[OK] Backup completed"
@@ -90,7 +151,8 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, None)
 
     def test_it_handles_multiple_subject_fail_keywords(self):
-        self.check.subject_fail = "FAIL, WARNING"
+        self.check.filter_subject = True
+        self.check.failure_kw = "FAIL, WARNING"
         self.check.save()
 
         body = PAYLOAD_TMPL % "[WARNING] Backup did not complete"
@@ -102,8 +164,9 @@ class SmtpdTestCase(BaseTestCase):
         self.assertEqual(ping.kind, "fail")
 
     def test_it_handles_subject_fail_before_success(self):
-        self.check.subject = "SUCCESS"
-        self.check.subject_fail = "FAIL"
+        self.check.filter_subject = True
+        self.check.success_kw = "SUCCESS"
+        self.check.failure_kw = "FAIL"
         self.check.save()
 
         subject = "[SUCCESS] 1 Backup completed, [FAIL] 1 Backup did not complete"

hc/front/forms.py

@@ -90,9 +90,10 @@ class AddCheckForm(NameTagsForm):
 
 
 class FilteringRulesForm(forms.Form):
-    filter_by_subject = forms.ChoiceField(choices=(("no", "no"), ("yes", "yes")))
-    subject = forms.CharField(required=False, max_length=200)
-    subject_fail = forms.CharField(required=False, max_length=200)
+    filter_subject = forms.BooleanField(required=False)
+    filter_body = forms.BooleanField(required=False)
+    success_kw = forms.CharField(required=False, max_length=200)
+    failure_kw = forms.CharField(required=False, max_length=200)
     methods = forms.ChoiceField(required=False, choices=(("", "Any"), ("POST", "POST")))
     manual_resume = forms.BooleanField(required=False)
 

hc/front/tests/test_filtering_rules.py

@@ -12,8 +12,10 @@ class FilteringRulesTestCase(BaseTestCase):
 
     def test_it_works(self):
         payload = {
-            "subject": "SUCCESS",
-            "subject_fail": "ERROR",
+            "filter_subject": "on",
+            "filter_body": "on",
+            "success_kw": "SUCCESS",
+            "failure_kw": "ERROR",
             "methods": "POST",
             "manual_resume": "1",
             "filter_by_subject": "yes",
@@ -24,8 +26,10 @@ class FilteringRulesTestCase(BaseTestCase):
         self.assertRedirects(r, self.redirect_url)
 
         self.check.refresh_from_db()
-        self.assertEqual(self.check.subject, "SUCCESS")
-        self.assertEqual(self.check.subject_fail, "ERROR")
+        self.assertTrue(self.check.filter_subject)
+        self.assertTrue(self.check.filter_body)
+        self.assertEqual(self.check.success_kw, "SUCCESS")
+        self.assertEqual(self.check.failure_kw, "ERROR")
         self.assertEqual(self.check.methods, "POST")
         self.assertTrue(self.check.manual_resume)
 
@@ -33,7 +37,7 @@ class FilteringRulesTestCase(BaseTestCase):
         self.check.method = "POST"
         self.check.save()
 
-        payload = {"subject": "SUCCESS", "methods": "", "filter_by_subject": "yes"}
+        payload = {"methods": "", "filter_by_subject": "yes"}
 
         self.client.login(username="alice@example.org", password="password")
         r = self.client.post(self.url, data=payload)
@@ -42,25 +46,22 @@ class FilteringRulesTestCase(BaseTestCase):
         self.check.refresh_from_db()
         self.assertEqual(self.check.methods, "")
 
-    def test_it_clears_subject(self):
-        self.check.subject = "SUCCESS"
-        self.check.subject_fail = "ERROR"
+    def test_it_clears_filtering_fields(self):
+        self.check.filter_subject = True
+        self.check.filter_body = True
+        self.check.success_kw = "SUCCESS"
+        self.check.failure_kw = "ERROR"
         self.check.save()
 
-        payload = {
-            "methods": "",
-            "filter_by_subject": "no",
-            "subject": "foo",
-            "subject_fail": "bar",
-        }
-
         self.client.login(username="alice@example.org", password="password")
-        r = self.client.post(self.url, data=payload)
+        r = self.client.post(self.url, data={"methods": ""})
         self.assertRedirects(r, self.redirect_url)
 
         self.check.refresh_from_db()
-        self.assertEqual(self.check.subject, "")
-        self.assertEqual(self.check.subject_fail, "")
+        self.assertFalse(self.check.filter_subject)
+        self.assertFalse(self.check.filter_body)
+        self.assertEqual(self.check.success_kw, "")
+        self.assertEqual(self.check.failure_kw, "")
 
     def test_it_clears_manual_resume_flag(self):
         self.check.manual_resume = True

hc/front/views.py

@@ -415,8 +415,11 @@ def filtering_rules(request, code):
 
     form = forms.FilteringRulesForm(request.POST)
     if form.is_valid():
-        check.subject = form.cleaned_data["subject"]
-        check.subject_fail = form.cleaned_data["subject_fail"]
+        check.filter_subject = form.cleaned_data["filter_subject"]
+        check.filter_body = form.cleaned_data["filter_body"]
+        check.success_kw = form.cleaned_data["success_kw"]
+        check.failure_kw = form.cleaned_data["failure_kw"]
+
         check.methods = form.cleaned_data["methods"]
         check.manual_resume = form.cleaned_data["manual_resume"]
         check.save()

hc/lib/html.py

@@ -0,0 +1,30 @@
+from html.parser import HTMLParser
+
+
+class TextOnlyParser(HTMLParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.active = True
+        self.buf = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ("script", "style"):
+            self.active = False
+
+    def handle_endtag(self, tag):
+        if tag in ("script", "style"):
+            self.active = True
+
+    def handle_data(self, data):
+        if self.active and data:
+            self.buf.append(data)
+
+    def get_text(self):
+        messy = "".join(self.buf)
+        return " ".join(messy.split())
+
+
+def html2text(html):
+    parser = TextOnlyParser()
+    parser.feed(html)
+    return parser.get_text()

hc/lib/tests/test_html.py

@@ -0,0 +1,19 @@
+from django.test import TestCase
+
+from hc.lib.html import html2text
+
+
+class HtmlTestCase(TestCase):
+    def test_it_works(self):
+        sample = """
+            <style>css goes here</style>
+            <h1 class="foo">Hello</h1>
+            World
+            <script>js goes here</script>
+            """
+
+        self.assertEqual(html2text(sample), "Hello World")
+
+    def test_it_does_not_inject_whitespace(self):
+        sample = """<b>S</b>UCCESS"""
+        self.assertEqual(html2text(sample), "SUCCESS")

static/js/details.js

@@ -182,9 +182,9 @@ $(function () {
 
 
     // Enable/disable fields in the "Filtering Rules" modal
-    $("input[type=radio][name=filter_by_subject]").on("change", function() {
-        var enableInputs = this.value == "yes";
-        $(".filter-by-subject").prop("disabled", !enableInputs);
+    $("input.filter-toggle").on("change", function() {
+        var enableInputs = $("input.filter-toggle:checked").length > 0;
+        $(".filter-kw").prop("disabled", !enableInputs);
     });
 
 });

templates/front/filtering_rules_modal.html

@@ -39,59 +39,61 @@
 
                 <div class="modal-body">
                     <h2>Inbound Emails</h2>
-                    <label class="radio-container">
+                    <label class="checkbox-container">
                         <input
-                            type="radio"
-                            name="filter_by_subject"
-                            value="no"
-                            {% if not check.subject and not check.subject_fail %}checked{% endif %} />
-                        <span class="radiomark"></span>
-                        No filtering. Treat all emails as "success"
-                    </label>
-                    <label class="radio-container">
-                        <input
-                            type="radio"
-                            name="filter_by_subject"
-                            value="yes"
-                            {% if check.subject or check.subject_fail %}checked{% endif %} />
-                        <span class="radiomark"></span>
-                        Filter by keywords in the Subject line:
+                            type="checkbox"
+                            class="filter-toggle"
+                            name="filter_subject"
+                            {% if check.filter_subject %}checked{% endif %} />
+                        <span class="checkmark"></span>
+                        Filter by keywords in the Subject line
                     </label>
 
+                    <label class="checkbox-container">
+                        <input
+                            type="checkbox"
+                            class="filter-toggle"
+                            name="filter_body"
+                            {% if check.filter_body %}checked{% endif %} />
+                        <span class="checkmark"></span>
+                        Filter by keywords in the message body
+                    </label>
 
                     <div class="form-group">
-                        <label for="update-name-input" class="col-sm-4 control-label">
+                        <label for="success_kw" class="col-sm-4 control-label">
                             Success Keywords
                         </label>
                         <div class="col-sm-7">
                             <input
-                                name="subject"
+                                id="success_kw"
+                                name="success_kw"
                                 type="text"
                                 maxlength="200"
-                                value="{{ check.subject }}"
-                                {% if not check.subject and not check.subject_fail %}disabled{% endif %}
-                                class="form-control filter-by-subject" />
+                                value="{{ check.success_kw }}"
+                                {% if not check.filter_subject and not check.filter_body %}disabled{% endif %}
+                                class="form-control filter-kw" />
                             <span class="help-block">
-                                Comma-separated list of keywords. If Subject contains
-                                any of the keywords, treat it as "success".
+                                Comma-separated list of keywords. If subject or body
+                                contains any of the keywords, classify the email as "success".
                             </span>
                         </div>
                     </div>
                     <div class="form-group">
-                        <label for="update-name-input" class="col-sm-4 control-label">
+                        <label for="failure_kw" class="col-sm-4 control-label">
                             Failure Keywords
                         </label>
                         <div class="col-sm-7">
                             <input
-                                name="subject_fail"
+                                id="failure_kw"
+                                name="failure_kw"
                                 type="text"
                                 maxlength="200"
-                                value="{{ check.subject_fail }}"
-                                {% if not check.subject and not check.subject_fail %}disabled{% endif %}
-                                class="form-control filter-by-subject" />
+                                value="{{ check.failure_kw }}"
+                                {% if not check.filter_subject and not check.filter_body %}disabled{% endif %}
+                                class="form-control filter-kw" />
                             <span class="help-block">
-                                Comma-separated list of keywords. If Subject contains
-                                any of the keywords, treat it as "failure".
+                                Comma-separated list of keywords. If subject or body
+                                contains any of the keywords, classify the email as "failure".
                             </span>
                         </div>
                     </div>