diff --git a/CHANGELOG.md b/CHANGELOG.md
index c00051bb..c56a9122 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
### Improvements
- Add support for EMAIL_USE_SSL environment variable (#685)
- Switch from requests to pycurl
+- Implement documentation search
### Bug Fixes
- Fix the handling of TooManyRedirects exceptions
diff --git a/hc/front/forms.py b/hc/front/forms.py
index def0121d..58a6bd3f 100644
--- a/hc/front/forms.py
+++ b/hc/front/forms.py
@@ -333,3 +333,7 @@ class AddGotifyForm(forms.Form):
def get_value(self):
return json.dumps(dict(self.cleaned_data), sort_keys=True)
+
+
+class SearchForm(forms.Form):
+ q = forms.RegexField(regex=r"^[0-9a-zA-Z\s]{3,100}$")
diff --git a/hc/front/management/commands/populate_searchdb.py b/hc/front/management/commands/populate_searchdb.py
new file mode 100644
index 00000000..6184cf7e
--- /dev/null
+++ b/hc/front/management/commands/populate_searchdb.py
@@ -0,0 +1,38 @@
+import os
+import sqlite3
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from hc.front.views import _replace_placeholders
+from hc.lib.html import html2text
+
+
+class Command(BaseCommand):
+ help = "Renders Markdown to HTML"
+
+ def handle(self, *args, **options):
+ con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
+ cur = con.cursor()
+ cur.execute("DROP TABLE IF EXISTS docs")
+ cur.execute(
+ """CREATE VIRTUAL TABLE docs USING FTS5(slug, title, body, tokenize="trigram")"""
+ )
+
+ docs_path = os.path.join(settings.BASE_DIR, "templates/docs")
+ for filename in os.listdir(docs_path):
+ if not filename.endswith(".html"):
+ continue
+
+ slug = filename[:-5] # cut ".html"
+ print("Processing %s" % slug)
+
+ html = open(os.path.join(docs_path, filename), "r").read()
+ html = _replace_placeholders(slug, html)
+
+ lines = html.split("\n")
+ title = html2text(lines[0])
+ text = html2text("\n".join(lines[1:]), skip_pre=True)
+
+ cur.execute("INSERT INTO docs VALUES (?, ?, ?)", (slug, title, text))
+
+ con.commit()
diff --git a/hc/front/tests/test_search.py b/hc/front/tests/test_search.py
new file mode 100644
index 00000000..ca120351
--- /dev/null
+++ b/hc/front/tests/test_search.py
@@ -0,0 +1,17 @@
+from hc.test import BaseTestCase
+
+
+class SearchTestCase(BaseTestCase):
+ def test_it_works(self):
+ r = self.client.get("/docs/search/?q=failure")
+ self.assertContains(
+ r, "You can actively signal a <span>failure</span>", status_code=200
+ )
+
+ def test_it_handles_no_results(self):
+ r = self.client.get("/docs/search/?q=asfghjkl")
+ self.assertContains(r, "Your search query matched no results", status_code=200)
+
+ def test_it_rejects_special_characters(self):
+ r = self.client.get("/docs/search/?q=api/v1")
+ self.assertContains(r, "Your search query matched no results", status_code=200)
diff --git a/hc/front/urls.py b/hc/front/urls.py
index c6e9f129..3e692bcb 100644
--- a/hc/front/urls.py
+++ b/hc/front/urls.py
@@ -106,5 +106,6 @@ urlpatterns = [
path("projects/<uuid:code>/", include(project_urls)),
path("docs/", views.serve_doc, name="hc-docs"),
path("docs/cron/", views.docs_cron, name="hc-docs-cron"),
+ path("docs/search/", views.docs_search, name="hc-docs-search"),
path("docs/<slug:doc>/", views.serve_doc, name="hc-serve-doc"),
]
diff --git a/hc/front/views.py b/hc/front/views.py
index d11b327e..3ec53abe 100644
--- a/hc/front/views.py
+++ b/hc/front/views.py
@@ -5,6 +5,7 @@ import json
import os
import re
from secrets import token_urlsafe
+import sqlite3
from urllib.parse import urlencode, urlparse
from cron_descriptor import ExpressionDescriptor
@@ -348,6 +349,29 @@ def dashboard(request):
return render(request, "front/dashboard.html", {})
+def _replace_placeholders(doc, html):
+ if doc.startswith("self_hosted"):
+ return html
+
+ replaces = {
+ "{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
+ "{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
+ "SITE_NAME": settings.SITE_NAME,
+ "SITE_ROOT": settings.SITE_ROOT,
+ "SITE_HOSTNAME": site_hostname(),
+ "SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
+ "PING_ENDPOINT": settings.PING_ENDPOINT,
+ "PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
+ "PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
+ "IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
+ }
+
+ for placeholder, value in replaces.items():
+ html = html.replace(placeholder, value)
+
+ return html
+
+
def serve_doc(request, doc="introduction"):
# Filenames in /templates/docs/ consist of lowercase letters and underscores,
# -- make sure we don't accept anything else
@@ -361,23 +385,7 @@ def serve_doc(request, doc="introduction"):
with open(path, "r", encoding="utf-8") as f:
content = f.read()
- if not doc.startswith("self_hosted"):
- replaces = {
- "{{ default_timeout }}": str(int(DEFAULT_TIMEOUT.total_seconds())),
- "{{ default_grace }}": str(int(DEFAULT_GRACE.total_seconds())),
- "SITE_NAME": settings.SITE_NAME,
- "SITE_ROOT": settings.SITE_ROOT,
- "SITE_HOSTNAME": site_hostname(),
- "SITE_SCHEME": urlparse(settings.SITE_ROOT).scheme,
- "PING_ENDPOINT": settings.PING_ENDPOINT,
- "PING_URL": settings.PING_ENDPOINT + "your-uuid-here",
- "PING_BODY_LIMIT": str(settings.PING_BODY_LIMIT or 100),
- "IMG_URL": os.path.join(settings.STATIC_URL, "img/docs"),
- }
-
- for placeholder, value in replaces.items():
- content = content.replace(placeholder, value)
-
+ content = _replace_placeholders(doc, content)
ctx = {
"page": "docs",
"section": doc,
@@ -388,6 +396,30 @@ def serve_doc(request, doc="introduction"):
return render(request, "front/docs_single.html", ctx)
+@csrf_exempt
+def docs_search(request):
+ form = forms.SearchForm(request.GET)
+ if not form.is_valid():
+ ctx = {"results": []}
+ return render(request, "front/docs_search.html", ctx)
+
+ query = """
+ SELECT slug, title, snippet(docs, 2, '<span>', '</span>', '…', 50)
+ FROM docs
+ WHERE docs MATCH ?
+ ORDER BY bm25(docs, 2.0, 10.0, 1.0)
+ LIMIT 8
+ """
+
+ q = form.cleaned_data["q"]
+ con = sqlite3.connect(os.path.join(settings.BASE_DIR, "search.db"))
+ cur = con.cursor()
+ res = cur.execute(query, (q,))
+
+ ctx = {"results": res.fetchall()}
+ return render(request, "front/docs_search.html", ctx)
+
+
def docs_cron(request):
return render(request, "front/docs_cron.html", {})
diff --git a/hc/lib/html.py b/hc/lib/html.py
index bcab70c0..2dbfe585 100644
--- a/hc/lib/html.py
+++ b/hc/lib/html.py
@@ -6,13 +6,14 @@ class TextOnlyParser(HTMLParser):
super().__init__(*args, **kwargs)
self.active = True
self.buf = []
+ self.skiplist = set(["script", "style"])
def handle_starttag(self, tag, attrs):
- if tag in ("script", "style"):
+ if tag in self.skiplist:
self.active = False
def handle_endtag(self, tag):
- if tag in ("script", "style"):
+ if tag in self.skiplist:
self.active = True
def handle_data(self, data):
@@ -24,7 +25,10 @@ class TextOnlyParser(HTMLParser):
return " ".join(messy.split())
-def html2text(html):
+def html2text(html, skip_pre=False):
parser = TextOnlyParser()
+ if skip_pre:
+ parser.skiplist.add("pre")
+
parser.feed(html)
return parser.get_text()
diff --git a/search.db b/search.db
new file mode 100644
index 00000000..b4ad4aee
Binary files /dev/null and b/search.db differ
diff --git a/static/css/search.css b/static/css/search.css
new file mode 100644
index 00000000..6f4a9b58
--- /dev/null
+++ b/static/css/search.css
@@ -0,0 +1,23 @@
+@media (min-width: 992px) {
+ #docs-search-form {
+ margin-right: 20%;
+ }
+}
+
+#search-results {
+ display: none;
+}
+
+#search-results.on {
+ display: block;
+}
+
+.docs-nav.off {
+ display: none;
+}
+
+#search-results li span {
+ background: #ffef82;
+ border-radius: 2px;
+ color: #111;
+}
diff --git a/static/js/search.js b/static/js/search.js
new file mode 100644
index 00000000..3e7eadfb
--- /dev/null
+++ b/static/js/search.js
@@ -0,0 +1,31 @@
+$(function() {
+ var base = document.getElementById("base-url").getAttribute("href").slice(0, -1);
+ var input = $("#docs-search");
+
+ input.on("keyup focus", function() {
+ var q = this.value;
+ if (q.length < 3) {
+ $("#search-results").removeClass("on");
+ $("#docs-nav").removeClass("off");
+ return
+ }
+
+ $.ajax({
+ url: base + "/docs/search/",
+ type: "get",
+ data: {q: q},
+ success: function(data) {
+ if (q != input.val()) {
+ return; // ignore stale results
+ }
+
+ $("#search-results").html(data).addClass("on");
+ $("#docs-nav").addClass("off");
+ }
+ });
+ });
+
+ // input.on("blur", function() {
+ // $("#search-results").removeClass("on");
+ // });
+});
\ No newline at end of file
diff --git a/templates/base.html b/templates/base.html
index 523fbfdd..339f7504 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -52,6 +52,7 @@
<link rel="stylesheet" href="{% static 'css/profile.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/projects.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/radio.css' %}" type="text/css">
+ <link rel="stylesheet" href="{% static 'css/search.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/settings.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/snippet-copy.css' %}" type="text/css">
<link rel="stylesheet" href="{% static 'css/syntax.css' %}" type="text/css">
diff --git a/templates/front/base_docs.html b/templates/front/base_docs.html
deleted file mode 100644
index 59b23ee5..00000000
--- a/templates/front/base_docs.html
+++ /dev/null
@@ -1,61 +0,0 @@
-{% extends "base.html" %}
-{% load hc_extras %}
-
-
-{% block content %}
-<div class="row">
- <div class="col-sm-3">
-
- <ul class="docs-nav">
- <li class="nav-header">{{ site_name }}</li>
- <li {% if section == "introduction" %} class="active"{% endif %}>
- <a href="{% url 'hc-docs' %}">Introduction</a>
- </li>
- {% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
- {% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
- {% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
- {% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
-
- <li class="nav-header">API</li>
- {% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
- {% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
-
- <li class="nav-header">Pinging Examples</li>
- {% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
- {% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
- {% include "front/docs_nav_item.html" with slug="python" title="Python" %}
- {% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
- {% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
- {% include "front/docs_nav_item.html" with slug="go" title="Go" %}
- {% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
- {% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
- {% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
- {% include "front/docs_nav_item.html" with slug="email" title="Email" %}
-
- <li class="nav-header">Guides</li>
- {% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
- {% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
- {% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
- {% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
- {% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
- {% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
-
- <li class="nav-header">Developer Tools</li>
- {% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
-
- <li class="nav-header">Self-hosted</li>
- {% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
- {% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
- {% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
-
- <li class="nav-header">Reference</li>
- <li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
- </ul>
-
- </div>
- <div class="col-sm-9">
- {% block docs_content %}
- {% endblock %}
- </div>
-</div>
-{% endblock %}
diff --git a/templates/front/docs_search.html b/templates/front/docs_search.html
new file mode 100644
index 00000000..1e39c121
--- /dev/null
+++ b/templates/front/docs_search.html
@@ -0,0 +1,14 @@
+<ul class="docs-nav">
+ <li class="nav-header">Search Results</li>
+ {% if results %}
+ {% for slug, title, snippet in results %}
+ <li>
+ <a href="{% url 'hc-serve-doc' slug %}">{{ title|safe }}</a>
+ <p>{{ snippet|safe }}</p>
+ </li>
+ {% endfor %}
+ </ul>
+ {% else %}
+ <li>Your search query matched no results.</li>
+ {% endif %}
+</ul>
diff --git a/templates/front/docs_single.html b/templates/front/docs_single.html
index 78c962b9..e26e9cdd 100644
--- a/templates/front/docs_single.html
+++ b/templates/front/docs_single.html
@@ -1,12 +1,75 @@
-{% extends "front/base_docs.html" %}
-{% load compress static hc_extras %}
+{% extends "base.html" %}
+{% load compress hc_extras static %}
{% block title %}{{ first_line|striptags }} - {{ site_name }}{% endblock %}
{% block description %}{% endblock %}
{% block keywords %}{% endblock %}
-{% block docs_content %}
-<div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
+{% block content %}
+<div class="row">
+ <div class="col-sm-3">
+
+ <div id="docs-search-form">
+ <input
+ id="docs-search"
+ type="text"
+ name="q"
+ class="form-control input-sm"
+ placeholder="Search docs…"
+ autocomplete="off">
+ </div>
+ <div id="search-results"></div>
+ <ul id="docs-nav" class="docs-nav">
+ <li class="nav-header">{{ site_name }}</li>
+ <li {% if section == "introduction" %} class="active"{% endif %}>
+ <a href="{% url 'hc-docs' %}">Introduction</a>
+ </li>
+ {% include "front/docs_nav_item.html" with slug="configuring_checks" title="Configuring checks" %}
+ {% include "front/docs_nav_item.html" with slug="configuring_notifications" title="Configuring notifications" %}
+ {% include "front/docs_nav_item.html" with slug="projects_teams" title="Projects and teams" %}
+ {% include "front/docs_nav_item.html" with slug="badges" title="Badges" %}
+
+ <li class="nav-header">API</li>
+ {% include "front/docs_nav_item.html" with slug="http_api" title="Pinging API" %}
+ {% include "front/docs_nav_item.html" with slug="api" title="Management API" %}
+
+ <li class="nav-header">Pinging Examples</li>
+ {% include "front/docs_nav_item.html" with slug="reliability_tips" title="Reliability Tips" %}
+ {% include "front/docs_nav_item.html" with slug="bash" title="Shell scripts" %}
+ {% include "front/docs_nav_item.html" with slug="python" title="Python" %}
+ {% include "front/docs_nav_item.html" with slug="ruby" title="Ruby" %}
+ {% include "front/docs_nav_item.html" with slug="php" title="PHP" %}
+ {% include "front/docs_nav_item.html" with slug="go" title="Go" %}
+ {% include "front/docs_nav_item.html" with slug="csharp" title="C#" %}
+ {% include "front/docs_nav_item.html" with slug="javascript" title="Javascript" %}
+ {% include "front/docs_nav_item.html" with slug="powershell" title="PowerShell" %}
+ {% include "front/docs_nav_item.html" with slug="email" title="Email" %}
+
+ <li class="nav-header">Guides</li>
+ {% include "front/docs_nav_item.html" with slug="monitoring_cron_jobs" title="Monitoring cron jobs" %}
+ {% include "front/docs_nav_item.html" with slug="signaling_failures" title="Signaling failures" %}
+ {% include "front/docs_nav_item.html" with slug="measuring_script_run_time" title="Measuring script run time" %}
+ {% include "front/docs_nav_item.html" with slug="attaching_logs" title="Attaching logs" %}
+ {% include "front/docs_nav_item.html" with slug="cloning_checks" title="Cloning checks" %}
+ {% include "front/docs_nav_item.html" with slug="configuring_prometheus" title="Configuring Prometheus" %}
+
+ <li class="nav-header">Developer Tools</li>
+ {% include "front/docs_nav_item.html" with slug="resources" title="Third-party resources" %}
+
+ <li class="nav-header">Self-hosted</li>
+ {% include "front/docs_nav_item.html" with slug="self_hosted" title="Overview" %}
+ {% include "front/docs_nav_item.html" with slug="self_hosted_configuration" title="Configuration" %}
+ {% include "front/docs_nav_item.html" with slug="self_hosted_docker" title="Running with Docker" %}
+
+ <li class="nav-header">Reference</li>
+ <li><a href="{% url 'hc-docs-cron' %}">Cron syntax cheatsheet</a></li>
+ </ul>
+
+ </div>
+ <div class="col-sm-9">
+ <div class="docs-content docs-{{ section }}">{{ content|safe }}</div>
+ </div>
+</div>
{% endblock %}
{% block scripts %}
@@ -15,5 +78,6 @@
<script src="{% static 'js/bootstrap.min.js' %}"></script>
<script src="{% static 'js/clipboard.min.js' %}"></script>
<script src="{% static 'js/snippet-copy.js' %}"></script>
+<script src="{% static 'js/search.js' %}"></script>
{% endcompress %}
{% endblock %}