From b743afd8d4cac035b10646021c0f9358abea310b Mon Sep 17 00:00:00 2001
From: Bram Wiepjes <bramw@protonmail.com>
Date: Mon, 24 Jul 2023 11:11:14 +0000
Subject: [PATCH] Implement posthog product analytics
---
backend/requirements/base.in | 1 +
backend/requirements/base.txt | 11 ++-
backend/requirements/dev.in | 2 +-
backend/requirements/dev.txt | 2 +-
backend/src/baserow/config/settings/base.py | 10 +++
.../contrib/database/airtable/utils.py | 4 +-
backend/src/baserow/core/apps.py | 3 +
backend/src/baserow/core/posthog.py | 71 +++++++++++++++++++
.../database/airtable/test_airtable_utils.py | 4 ++
backend/tests/baserow/core/test_posthog.py | 35 +++++++++
...nt_optional_posthog_product_analytics.json | 7 ++
docker-compose.local-build.yml | 2 +
docker-compose.no-caddy.yml | 2 +
docker-compose.yml | 2 +
docs/installation/configuration.md | 7 ++
.../airtable/ImportFromAirtable.vue | 2 +-
16 files changed, 159 insertions(+), 6 deletions(-)
create mode 100644 backend/src/baserow/core/posthog.py
create mode 100644 backend/tests/baserow/core/test_posthog.py
create mode 100644 changelog/entries/unreleased/feature/implement_optional_posthog_product_analytics.json
diff --git a/backend/requirements/base.in b/backend/requirements/base.in
index 41cf764bc..3e19123af 100644
--- a/backend/requirements/base.in
+++ b/backend/requirements/base.in
@@ -61,4 +61,5 @@ Brotli==1.0.9
loguru==0.6.0
django-cachalot==2.5.3
celery-singleton==0.3.1
+posthog==3.0.1
https://github.com/fellowapp/prosemirror-py/archive/refs/tags/v0.3.5.zip
diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt
index 790de9ed2..51f12a3e6 100644
--- a/backend/requirements/base.txt
+++ b/backend/requirements/base.txt
@@ -37,7 +37,9 @@ azure-core==1.26.4
azure-storage-blob==12.16.0
# via django-storages
backoff==2.2.1
- # via opentelemetry-exporter-otlp-proto-http
+ # via
+ # opentelemetry-exporter-otlp-proto-http
+ # posthog
billiard==3.6.4.0
# via celery
boto3==1.26.103
@@ -211,6 +213,8 @@ kombu==5.2.4
# via celery
loguru==0.6.0
# via -r base.in
+monotonic==1.6
+ # via posthog
msgpack==1.0.4
# via channels-redis
ndg-httpsclient==0.5.1
@@ -315,6 +319,8 @@ opentelemetry-util-http==0.38b0
# opentelemetry-instrumentation-wsgi
pillow==9.4.0
# via -r base.in
+posthog==3.0.1
+ # via -r base.in
prompt-toolkit==3.0.31
# via click-repl
prosemirror @ https://github.com/fellowapp/prosemirror-py/archive/refs/tags/v0.3.5.zip
@@ -360,6 +366,7 @@ python-dateutil==2.8.2
# botocore
# celery-redbeat
# faker
+ # posthog
# pysaml2
# python-crontab
python-dotenv==0.21.0
@@ -393,6 +400,7 @@ requests==2.31.0
# google-api-core
# google-cloud-storage
# opentelemetry-exporter-otlp-proto-http
+ # posthog
# pysaml2
# requests-oauthlib
requests-oauthlib==1.3.1
@@ -413,6 +421,7 @@ six==1.16.0
# click-repl
# google-auth
# isodate
+ # posthog
# python-dateutil
# service-identity
sniffio==1.3.0
diff --git a/backend/requirements/dev.in b/backend/requirements/dev.in
index a3c54f6f3..0503711d5 100644
--- a/backend/requirements/dev.in
+++ b/backend/requirements/dev.in
@@ -23,7 +23,7 @@ pytest-html==3.2.0
coverage==7.2.2
pytest-split==0.8.0
bandit==1.7.5
-pip-tools==6.12.3
+pip-tools==6.13.0
autopep8==2.0.2
pytest-unordered==0.5.2
debugpy==1.6.6
diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt
index d5465a4c5..f8ef9a7e9 100644
--- a/backend/requirements/dev.txt
+++ b/backend/requirements/dev.txt
@@ -183,7 +183,7 @@ pexpect==4.8.0
# via ipython
pickleshare==0.7.5
# via ipython
-pip-tools==6.12.3
+pip-tools==6.13.0
# via -r dev.in
platformdirs==2.5.2
# via black
diff --git a/backend/src/baserow/config/settings/base.py b/backend/src/baserow/config/settings/base.py
index 6fe417e2c..fd89eac31 100644
--- a/backend/src/baserow/config/settings/base.py
+++ b/backend/src/baserow/config/settings/base.py
@@ -12,6 +12,7 @@ from urllib.parse import urljoin, urlparse
from django.core.exceptions import ImproperlyConfigured
import dj_database_url
+import posthog
from corsheaders.defaults import default_headers
from baserow.cachalot_patch import patch_cachalot_for_baserow
@@ -1108,6 +1109,15 @@ USE_PG_FULLTEXT_SEARCH = str_to_bool(
PG_SEARCH_CONFIG = os.getenv("BASEROW_PG_SEARCH_CONFIG", "simple")
AUTO_VACUUM_AFTER_SEARCH_UPDATE = str_to_bool(os.getenv("BASEROW_AUTO_VACUUM", "true"))
+POSTHOG_PROJECT_API_KEY = os.getenv("POSTHOG_PROJECT_API_KEY", "")
+POSTHOG_HOST = os.getenv("POSTHOG_HOST", "")
+POSTHOG_ENABLED = POSTHOG_PROJECT_API_KEY and POSTHOG_HOST
+if POSTHOG_ENABLED:
+ posthog.project_api_key = POSTHOG_PROJECT_API_KEY
+ posthog.host = POSTHOG_HOST
+else:
+ posthog.disabled = True
+
# Indicates whether we are running the tests or not. Set to True in the test.py settings
# file used by pytest.ini
TESTS = False
diff --git a/backend/src/baserow/contrib/database/airtable/utils.py b/backend/src/baserow/contrib/database/airtable/utils.py
index a2d6090a1..72dc056fc 100644
--- a/backend/src/baserow/contrib/database/airtable/utils.py
+++ b/backend/src/baserow/contrib/database/airtable/utils.py
@@ -11,7 +11,7 @@ def extract_share_id_from_url(public_base_url: str) -> str:
:return: The extracted share id.
"""
- result = re.search(r"https:\/\/airtable.com\/shr(.*)$", public_base_url)
+ result = re.search(r"https:\/\/airtable.com\/(shr|app)(.*)$", public_base_url)
if not result:
raise ValueError(
@@ -19,4 +19,4 @@ def extract_share_id_from_url(public_base_url: str) -> str:
f"https://airtable.com/shrxxxxxxxxxxxxxx)"
)
- return f"shr{result.group(1)}"
+ return f"{result.group(1)}{result.group(2)}"
diff --git a/backend/src/baserow/core/apps.py b/backend/src/baserow/core/apps.py
index ec2235d0b..980429cf2 100755
--- a/backend/src/baserow/core/apps.py
+++ b/backend/src/baserow/core/apps.py
@@ -286,6 +286,9 @@ class CoreConfig(AppConfig):
WorkspaceInvitationRejectedNotificationType()
)
+ # Must import the Posthog signal, otherwise it won't work.
+ import baserow.core.posthog # noqa: F403, F401
+
self._setup_health_checks()
# Clear the key after migration so we will trigger a new template sync.
diff --git a/backend/src/baserow/core/posthog.py b/backend/src/baserow/core/posthog.py
new file mode 100644
index 000000000..22d30e18b
--- /dev/null
+++ b/backend/src/baserow/core/posthog.py
@@ -0,0 +1,71 @@
+from copy import deepcopy
+from typing import Optional
+
+from django.conf import settings
+from django.contrib.auth.models import AbstractUser
+from django.dispatch import receiver
+
+import posthog
+
+from baserow.core.action.signals import ActionCommandType, action_done
+from baserow.core.models import Workspace
+
+
+def capture_event(
+ user: AbstractUser,
+ event: str,
+ properties: dict,
+ session: Optional[str] = None,
+ workspace: Optional[Workspace] = None,
+):
+ """
+ Captures a Posthog event in a consistent property format.
+
+ :param user: The user that performed the event.
+ :param event: Unique name identifying the event.
+ :param properties: A dictionary containing the properties that must be added.
+ Note that the `user_email`, `user_session`, `workspace_id`,
+ and `workspace_name` will be overwritten.
+ :param session: A unique session id that identifies the user throughout their
+ session.
+ :param workspace: Optionally the workspace related to the event.
+ """
+
+ if not settings.POSTHOG_ENABLED:
+ return
+
+ properties = deepcopy(properties)
+ properties["user_email"] = user.email
+
+ if session is not None:
+ properties["user_session"] = session
+
+ if workspace is not None:
+ properties["workspace_id"] = workspace.id
+ properties["workspace_name"] = workspace.name
+
+ posthog.capture(
+ distinct_id=user.id,
+ event=event,
+ properties=properties,
+ )
+
+
+@receiver(action_done)
+def capture_event_action_done(
+ sender,
+ user,
+ action_type,
+ action_params,
+ action_timestamp,
+ action_command_type,
+ workspace,
+ session,
+ **kwargs,
+):
+ # Only capture do commands for now because the undo might make it more difficult
+ # to do analytics on the data.
+ if action_command_type == ActionCommandType.DO:
+ capture_event(
+ user, action_type.type, action_params, workspace=workspace, session=session
+ )
diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
index bad6177dd..cde32ddff 100644
--- a/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
+++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
@@ -11,6 +11,10 @@ def test_extract_share_id_from_url():
extract_share_id_from_url("https://airtable.com/shrxxxxxxxxxxxxxx")
== "shrxxxxxxxxxxxxxx"
)
+ assert (
+ extract_share_id_from_url("https://airtable.com/appxxxxxxxxxxxxxx")
+ == "appxxxxxxxxxxxxxx"
+ )
assert (
extract_share_id_from_url("https://airtable.com/shrXxmp0WmqsTkFWTzv")
== "shrXxmp0WmqsTkFWTzv"
diff --git a/backend/tests/baserow/core/test_posthog.py b/backend/tests/baserow/core/test_posthog.py
new file mode 100644
index 000000000..64915996e
--- /dev/null
+++ b/backend/tests/baserow/core/test_posthog.py
@@ -0,0 +1,35 @@
+from unittest.mock import patch
+
+from django.test.utils import override_settings
+
+import pytest
+
+from baserow.core.posthog import capture_event
+
+
+@pytest.mark.django_db
+@override_settings(POSTHOG_ENABLED=False)
+@patch("baserow.core.posthog.posthog")
+def test_not_capture_event_if_not_enabled(mock_posthog, data_fixture):
+ user = data_fixture.create_user()
+ capture_event(user, "test", {})
+ mock_posthog.capture.assert_not_called()
+
+
+@pytest.mark.django_db
+@override_settings(POSTHOG_ENABLED=True)
+@patch("baserow.core.posthog.posthog")
+def test_capture_event_if_enabled(mock_posthog, data_fixture):
+ user = data_fixture.create_user()
+ workspace = data_fixture.create_workspace()
+ capture_event(user, "test", {}, session="session", workspace=workspace)
+ mock_posthog.capture.assert_called_once_with(
+ distinct_id=user.id,
+ event="test",
+ properties={
+ "user_email": user.email,
+ "user_session": "session",
+ "workspace_id": workspace.id,
+ "workspace_name": workspace.name,
+ },
+ )
diff --git a/changelog/entries/unreleased/feature/implement_optional_posthog_product_analytics.json b/changelog/entries/unreleased/feature/implement_optional_posthog_product_analytics.json
new file mode 100644
index 000000000..b53331c72
--- /dev/null
+++ b/changelog/entries/unreleased/feature/implement_optional_posthog_product_analytics.json
@@ -0,0 +1,7 @@
+{
+ "type": "feature",
+ "message": "Implemented optional Posthog product analytics.",
+ "issue_number": null,
+ "bullet_points": [],
+ "created_at": "2023-07-21"
+}
diff --git a/docker-compose.local-build.yml b/docker-compose.local-build.yml
index 85540c1c3..7029fbbc2 100644
--- a/docker-compose.local-build.yml
+++ b/docker-compose.local-build.yml
@@ -78,6 +78,8 @@ x-backend-variables: &backend-variables
BASEROW_DEPLOYMENT_ENV:
OTEL_EXPORTER_OTLP_ENDPOINT:
OTEL_RESOURCE_ATTRIBUTES:
+ POSTHOG_PROJECT_API_KEY:
+ POSTHOG_HOST:
PRIVATE_BACKEND_URL: http://backend:8000
PUBLIC_BACKEND_URL:
diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml
index f0e25a00c..b7224b59b 100644
--- a/docker-compose.no-caddy.yml
+++ b/docker-compose.no-caddy.yml
@@ -98,6 +98,8 @@ x-backend-variables: &backend-variables
BASEROW_DEPLOYMENT_ENV:
OTEL_EXPORTER_OTLP_ENDPOINT:
OTEL_RESOURCE_ATTRIBUTES:
+ POSTHOG_PROJECT_API_KEY:
+ POSTHOG_HOST:
PRIVATE_BACKEND_URL: http://backend:8000
BASEROW_PUBLIC_URL:
diff --git a/docker-compose.yml b/docker-compose.yml
index 488af6e88..83215b8b3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -94,6 +94,8 @@ x-backend-variables: &backend-variables
BASEROW_DEPLOYMENT_ENV:
OTEL_EXPORTER_OTLP_ENDPOINT:
OTEL_RESOURCE_ATTRIBUTES:
+ POSTHOG_PROJECT_API_KEY:
+ POSTHOG_HOST:
PRIVATE_BACKEND_URL: http://backend:8000
PUBLIC_BACKEND_URL:
diff --git a/docs/installation/configuration.md b/docs/installation/configuration.md
index 971d863b6..7e0d76331 100644
--- a/docs/installation/configuration.md
+++ b/docs/installation/configuration.md
@@ -262,3 +262,10 @@ domain than your Baserow, you need to make sure CORS is configured correctly.
| BASEROW\_PLUGIN\_URLS | A comma separated list of plugin urls to install on startup. | |
| BASEROW\_DISABLE\_PLUGIN\_INSTALL\_ON\_STARTUP | When set to any non-empty values no automatic startup check and/or install of plugins will be run. Disables the above two env variables. |
| BASEROW\_PLUGIN\_DIR | **INTERNAL** Sets the folder where the Baserow plugin scripts look for plugins. | In the all-in-one image `/baserow/data/plugins`, otherwise `/baserow/plugins` |
+
+### Posthog configuration
+
+| Name | Description | Defaults |
+|----------------------------|-----------------------------------------------------------------|-------------------------------------------------------------------------------|
+| POSTHOG\_PROJECT\_API\_KEY | Set this to your Posthog project API key for product analytics. | |
+| POSTHOG\_HOST | Set this to your Posthog host for product analytics. | |
diff --git a/web-frontend/modules/database/components/airtable/ImportFromAirtable.vue b/web-frontend/modules/database/components/airtable/ImportFromAirtable.vue
index 3fdb4094b..27ca781b5 100644
--- a/web-frontend/modules/database/components/airtable/ImportFromAirtable.vue
+++ b/web-frontend/modules/database/components/airtable/ImportFromAirtable.vue
@@ -149,7 +149,7 @@ export default {
validations: {
airtableUrl: {
valid(value) {
- const regex = /https:\/\/airtable.com\/shr(.*)$/g
+ const regex = /https:\/\/airtable.com\/[shr|app](.*)$/g
return !!value.match(regex)
},
},