Implement documentation search

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 ### Improvements
 - Add support for EMAIL_USE_SSL environment variable (#685)
 - Switch from requests to pycurl
+- Implement documentation search
 
 ### Bug Fixes
 - Fix the handling of TooManyRedirects exceptions

hc/front/forms.py

@@ -333,3 +333,7 @@ class AddGotifyForm(forms.Form):
 
     def get_value(self):
         return json.dumps(dict(self.cleaned_data), sort_keys=True)
+
+
+class SearchForm(forms.Form):
+    q = forms.RegexField(regex=r"^[0-9a-zA-Z\s]{3,100}$")

hc/front/management/commands/populate_searchdb.py

@@ -0,0 +1,38 @@
+import os
+import sqlite3
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from hc.front.views import _replace_placeholders
+from hc.lib.html import html2text
+
+
+class Command(BaseCommand):
+    help = "Renders Markdown to HTML"
+
+    def handle(self, *args, **options):
+        con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
+        cur = con.cursor()
+        cur.execute("DROP TABLE IF EXISTS docs")
+        cur.execute(
+            """CREATE VIRTUAL TABLE docs USING FTS5(slug, title, body, tokenize="trigram")"""
+        )
+
+        docs_path = os.path.join(settings.BASE_DIR, "templates/docs")
+        for filename in os.listdir(docs_path):
+            if not filename.endswith(".html"):
+                continue
+
+            slug = filename[:-5]  # cut ".html"
+            print("Processing %s" % slug)
+
+            html = open(os.path.join(docs_path, filename), "r").read()
+            html = _replace_placeholders(slug, html)
+
+            lines = html.split("\n")
+            title = html2text(lines[0])
+            text = html2text("\n".join(lines[1:]), skip_pre=True)
+
+            cur.execute("INSERT INTO docs VALUES (?, ?, ?)", (slug, title, text))
+
+        con.commit()

hc/front/tests/test_search.py

@@ -0,0 +1,17 @@
+from hc.test import BaseTestCase
+
+
+class SearchTestCase(BaseTestCase):
+    def test_it_works(self):
+        r = self.client.get("/docs/search/?q=failure")
+        self.assertContains(
+            r, "You can actively signal a <span>failure</span>", status_code=200
+        )
+
+    def test_it_handles_no_results(self):
+        r = self.client.get("/docs/search/?q=asfghjkl")
+        self.assertContains(r, "Your search query matched no results", status_code=200)
+
+    def test_it_rejects_special_characters(self):
+        r = self.client.get("/docs/search/?q=api/v1")
+        self.assertContains(r, "Your search query matched no results", status_code=200)

hc/front/urls.py

@@ -106,5 +106,6 @@ urlpatterns = [
     path("projects/<uuid:code>/", include(project_urls)),
     path("docs/", views.serve_doc, name="hc-docs"),
     path("docs/cron/", views.docs_cron, name="hc-docs-cron"),
+    path("docs/search/", views.docs_search, name="hc-docs-search"),
     path("docs/<slug:doc>/", views.serve_doc, name="hc-serve-doc"),
 ]

hc/front/views.py

@@ -5,6 +5,7 @@ import json
 import os
 import re
 from secrets import token_urlsafe
+import sqlite3
 from urllib.parse import urlencode, urlparse
 
 from cron_descriptor import ExpressionDescriptor
@@ -348,6 +349,29 @@ def dashboard(request):
     return render(request, "front/dashboard.html", {})
 
 
+def _replace_placeholders(doc, html):
+    if doc.startswith("self_hosted"):
+        return html
+
+    replaces = {
+        "{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
+        "{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
+        "SITE_NAME": settings.SITE_NAME,
+        "SITE_ROOT": settings.SITE_ROOT,
+        "SITE_HOSTNAME": site_hostname(),
+        "SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
+        "PING_ENDPOINT": settings.PING_ENDPOINT,
+        "PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
+        "PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
+        "IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
+    }
+
+    for placeholder, value in replaces.items():
+        html = html.replace(placeholder, value)
+
+    return html
+
+
 def serve_doc(request, doc="introduction"):
     # Filenames in /templates/docs/ consist of lowercase letters and underscores,
     # -- make sure we don't accept anything else
@@ -361,23 +385,7 @@ def serve_doc(request, doc="introduction"):
     with open(path, "r", encoding="utf-8") as f:
         content = f.read()
 
-    if not doc.startswith("self_hosted"):
-        replaces = {
-            "{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
-            "{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
-            "SITE_NAME": settings.SITE_NAME,
-            "SITE_ROOT": settings.SITE_ROOT,
-            "SITE_HOSTNAME": site_hostname(),
-            "SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
-            "PING_ENDPOINT": settings.PING_ENDPOINT,
-            "PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
-            "PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
-            "IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
-        }
-
-        for placeholder, value in replaces.items():
-            content = content.replace(placeholder, value)
-
+    content = _replace_placeholders(doc, content)
     ctx = {
         "page": "docs",
         "section": doc,
@@ -388,6 +396,30 @@ def serve_doc(request, doc="introduction"):
     return render(request, "front/docs_single.html", ctx)
 
 
+@csrf_exempt
+def docs_search(request):
+    form = forms.SearchForm(request.GET)
+    if not form.is_valid():
+        ctx = {"results": []}
+        return render(request, "front/docs_search.html", ctx)
+
+    query = """
+        SELECT slug, title, snippet(docs, 2, '<span>', '</span>', '&hellip;', 50)
+        FROM docs
+        WHERE docs MATCH ?
+        ORDER BY bm25(docs, 2.0, 10.0, 1.0)
+        LIMIT 8
+    """
+
+    q = form.cleaned_data["q"]
+    con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
+    cur = con.cursor()
+    res = cur.execute(query, (q,))
+
+    ctx = {"results": res.fetchall()}
+    return render(request, "front/docs_search.html", ctx)
+
+
 def docs_cron(request):
     return render(request, "front/docs_cron.html", {})
 

hc/lib/html.py

@@ -6,13 +6,14 @@ class TextOnlyParser(HTMLParser):
         super().__init__(*args, **kwargs)
         self.active = True
         self.buf = []
+        self.skiplist = set(["script", "style"])
 
     def handle_starttag(self, tag, attrs):
-        if tag in ("script", "style"):
+        if tag in self.skiplist:
             self.active = False
 
     def handle_endtag(self, tag):
-        if tag in ("script", "style"):
+        if tag in self.skiplist:
             self.active = True
 
     def handle_data(self, data):
@@ -24,7 +25,10 @@ class TextOnlyParser(HTMLParser):
         return " ".join(messy.split())
 
 
-def html2text(html):
+def html2text(html, skip_pre=False):
     parser = TextOnlyParser()
+    if skip_pre:
+        parser.skiplist.add("pre")
+
     parser.feed(html)
     return parser.get_text()

static/css/search.css

@@ -0,0 +1,23 @@
+@media (min-width: 992px) {
+  #docs-search-form {
+    margin-right: 20%;
+  }
+}
+
+#search-results {
+    display: none;
+}
+
+#search-results.on {
+    display: block;
+}
+
+.docs-nav.off {
+    display: none;
+}
+
+#search-results li span {
+    background: #ffef82;
+    border-radius: 2px;
+    color: #111;
+}

static/js/search.js

@@ -0,0 +1,31 @@
+$(function() {
+    var base = document.getElementById("base-url").getAttribute("href").slice(0, -1);
+    var input = $("#docs-search");
+
+    input.on("keyup focus", function() {
+        var q = this.value;
+        if (q.length < 3) {
+            $("#search-results").removeClass("on");
+            $("#docs-nav").removeClass("off");
+            return
+        }
+
+        $.ajax({
+            url: base + "/docs/search/",
+            type: "get",
+            data: {q: q},
+            success: function(data) {
+                if (q != input.val()) {
+                    return;  // ignore stale results
+                }
+
+                $("#search-results").html(data).addClass("on");
+                $("#docs-nav").addClass("off");
+            }
+        });
+    });
+
+    // input.on("blur", function() {
+    //     $("#search-results").removeClass("on");
+    // });
+});

templates/base.html

@@ -52,6 +52,7 @@
     <link rel="stylesheet" href="{% static 'css/profile.css' %}" type="text/css">
     <link rel="stylesheet" href="{% static 'css/projects.css' %}" type="text/css">
     <link rel="stylesheet" href="{% static 'css/radio.css' %}" type="text/css">
+    <link rel="stylesheet" href="{% static 'css/search.css' %}" type="text/css">
     <link rel="stylesheet" href="{% static 'css/settings.css' %}" type="text/css">
     <link rel="stylesheet" href="{% static 'css/snippet-copy.css' %}" type="text/css">
     <link rel="stylesheet" href="{% static 'css/syntax.css' %}" type="text/css">

templates/front/base_docs.html

@@ -1,61 +0,0 @@
-{% extends "base.html" %}
-{% load hc_extras %}
-
-
-{% block content %}
-<div class="row">
-    <div class="col-sm-3">
-
-    <ul class="docs-nav">
-        <li class="nav-header">{{ site_name }}</li>
-        <li {% if section == "introduction" %} class="active"{% endif %}>
-            <a href="{% url 'hc-docs' %}">Introduction</a>
-        </li>
-        {% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
-        {% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
-        {% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
-        {% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
-
-        <li class="nav-header">API</li>
-        {% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
-        {% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
-
-        <li class="nav-header">Pinging Examples</li>
-        {% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
-        {% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
-        {% include "front/docs_nav_item.html" with slug="python" title="Python" %}
-        {% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
-        {% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
-        {% include "front/docs_nav_item.html" with slug="go" title="Go" %}
-        {% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
-        {% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
-        {% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
-        {% include "front/docs_nav_item.html" with slug="email" title="Email" %}
-
-        <li class="nav-header">Guides</li>
-        {% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
-        {% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
-        {% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
-        {% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
-        {% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
-        {% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
-
-        <li class="nav-header">Developer Tools</li>
-        {% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
-
-        <li class="nav-header">Self-hosted</li>
-        {% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
-        {% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
-        {% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
-
-        <li class="nav-header">Reference</li>
-        <li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
-    </ul>
-
-    </div>
-    <div class="col-sm-9">
-        {% block docs_content %}
-        {% endblock %}
-    </div>
-</div>
-{% endblock %}

templates/front/docs_search.html

@@ -0,0 +1,14 @@
+<ul class="docs-nav">
+    <li class="nav-header">Search Results</li>
+    {% if results %}
+        {% for slug, title, snippet in results %}
+        <li>
+            <a href="{% url 'hc-serve-doc' slug %}">{{ title|safe }}</a>
+            <p>{{ snippet|safe }}</p>
+        </li>
+        {% endfor %}
+        </ul>
+    {% else %}
+        <li>Your search query matched no results.</li>
+    {% endif %}
+</ul>

templates/front/docs_single.html

@@ -1,12 +1,75 @@
-{% extends "front/base_docs.html" %}
-{% load compress static hc_extras %}
+{% extends "base.html" %}
+{% load compress hc_extras static %}
 
 {% block title %}{{ first_line|striptags }} - {{ site_name }}{% endblock %}
 {% block description %}{% endblock %}
 {% block keywords %}{% endblock %}
 
-{% block docs_content %}
-<div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
+{% block content %}
+<div class="row">
+    <div class="col-sm-3">
+
+    <div id="docs-search-form">
+        <input
+            id="docs-search"
+            type="text"
+            name="q"
+            class="form-control input-sm"
+            placeholder="Search docs&hellip;"
+            autocomplete="off">
+    </div>
+    <div id="search-results"></div>
+    <ul id="docs-nav" class="docs-nav">
+        <li class="nav-header">{{ site_name }}</li>
+        <li {% if section == "introduction" %} class="active"{% endif %}>
+            <a href="{% url 'hc-docs' %}">Introduction</a>
+        </li>
+        {% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
+        {% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
+        {% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
+        {% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
+
+        <li class="nav-header">API</li>
+        {% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
+        {% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
+
+        <li class="nav-header">Pinging Examples</li>
+        {% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
+        {% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
+        {% include "front/docs_nav_item.html" with slug="python" title="Python" %}
+        {% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
+        {% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
+        {% include "front/docs_nav_item.html" with slug="go" title="Go" %}
+        {% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
+        {% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
+        {% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
+        {% include "front/docs_nav_item.html" with slug="email" title="Email" %}
+
+        <li class="nav-header">Guides</li>
+        {% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
+        {% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
+        {% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
+        {% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
+        {% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
+        {% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
+
+        <li class="nav-header">Developer Tools</li>
+        {% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
+
+        <li class="nav-header">Self-hosted</li>
+        {% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
+        {% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
+        {% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
+
+        <li class="nav-header">Reference</li>
+        <li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
+    </ul>
+
+    </div>
+    <div class="col-sm-9">
+        <div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
+    </div>
+</div>
 {% endblock %}
 
 {% block scripts %}
@@ -15,5 +78,6 @@
 <script src="{% static 'js/bootstrap.min.js' %}"></script>
 <script src="{% static 'js/clipboard.min.js' %}"></script>
 <script src="{% static 'js/snippet-copy.js' %}"></script>
+<script src="{% static 'js/search.js' %}"></script>
 {% endcompress %}
 {% endblock %}