From 3a4500ed8e037d5e13dae141345050292331a3be Mon Sep 17 00:00:00 2001
From: Bram Wiepjes <bramw@protonmail.com>
Date: Fri, 14 Mar 2025 11:46:11 +0000
Subject: [PATCH] [3/3] Final airtable import changes
---
.../contrib/database/airtable/registry.py | 11 +-
.../contrib/database/airtable/utils.py | 13 +-
.../commands/install_airtable_templates.py | 146 ++++++++++++++++++
.../database/airtable/test_airtable_utils.py | 7 +
.../airtable/test_airtable_view_types.py | 29 ++++
web-frontend/modules/database/locales/en.json | 6 +-
6 files changed, 204 insertions(+), 8 deletions(-)
create mode 100644 backend/src/baserow/contrib/database/management/commands/install_airtable_templates.py
diff --git a/backend/src/baserow/contrib/database/airtable/registry.py b/backend/src/baserow/contrib/database/airtable/registry.py
index 01e7221e5..0cb203a5f 100644
--- a/backend/src/baserow/contrib/database/airtable/registry.py
+++ b/backend/src/baserow/contrib/database/airtable/registry.py
@@ -458,6 +458,7 @@ class AirtableViewType(Instance):
filters = []
conjunction = filter_object.get("conjunction", None)
filter_set = filter_object.get("filterSet", None)
+ column_id = filter_object.get("columnId", None)
if conjunction and filter_set:
# The filter_object is a nested structure, where if the `conjunction` and
@@ -488,7 +489,7 @@ class AirtableViewType(Instance):
filters.extend(child_filters)
return filters, filter_groups
- else:
+ elif column_id:
baserow_filter = self.get_filter(
field_mapping,
row_id_mapping,
@@ -504,6 +505,8 @@ class AirtableViewType(Instance):
else:
return [baserow_filter], []
+ return [], []
+
def get_select_column_decoration(
self,
field_mapping: dict,
@@ -567,7 +570,8 @@ class AirtableViewType(Instance):
)
# Pop the first group because that shouldn't be in Baserow, and the type is
# defined on the view.
- root_group = filter_groups.pop(0)
+ if len(filter_groups) > 0:
+ root_group = filter_groups.pop(0)
color = AIRTABLE_BASEROW_COLOR_MAPPING.get(
color_definition.get("color", ""),
"blue",
@@ -709,7 +713,8 @@ class AirtableViewType(Instance):
)
# Pop the first group because that shouldn't be in Baserow, and the type is
# defined on the view.
- view.filter_type = filter_groups.pop(0).filter_type
+ if len(filter_groups) > 0:
+ view.filter_type = filter_groups.pop(0).filter_type
sorts = self.get_sorts(
field_mapping,
diff --git a/backend/src/baserow/contrib/database/airtable/utils.py b/backend/src/baserow/contrib/database/airtable/utils.py
index 716754ba8..fb7f4f610 100644
--- a/backend/src/baserow/contrib/database/airtable/utils.py
+++ b/backend/src/baserow/contrib/database/airtable/utils.py
@@ -1,6 +1,6 @@
import json
import re
-from typing import Any, Optional
+from typing import Any, Optional, Union
from requests import Response
@@ -245,7 +245,7 @@ def quill_to_markdown(ops: list) -> str:
return "".join(md_output).strip()
-def airtable_date_filter_value_to_baserow(value: Optional[dict]) -> str:
+def airtable_date_filter_value_to_baserow(value: Optional[Union[dict, str]]) -> str:
"""
Converts the provided Airtable filter date value to the Baserow compatible date
value string.
@@ -257,6 +257,15 @@ def airtable_date_filter_value_to_baserow(value: Optional[dict]) -> str:
if value is None:
return ""
+ # If the value is a string, then it contains an exact date. This is the old format
+ # of Airtable. In that case, we can conert it to the correct format.
+ if isinstance(value, str):
+ value = {
+ "mode": "exactDate",
+ "exactDate": value,
+ "timeZone": "", # it's okay to leave the timezone empty in Baserow.
+ }
+
mode = value["mode"]
if "exactDate" in value:
# By default, Airtable adds the time, but that is not needed in Baserow.
diff --git a/backend/src/baserow/contrib/database/management/commands/install_airtable_templates.py b/backend/src/baserow/contrib/database/management/commands/install_airtable_templates.py
new file mode 100644
index 000000000..fb677e6fc
--- /dev/null
+++ b/backend/src/baserow/contrib/database/management/commands/install_airtable_templates.py
@@ -0,0 +1,146 @@
+import json
+import re
+import sys
+from tempfile import NamedTemporaryFile
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+import requests
+from tqdm import tqdm
+
+from baserow.contrib.database.airtable.config import AirtableImportConfig
+from baserow.contrib.database.airtable.constants import AIRTABLE_BASE_URL
+from baserow.contrib.database.airtable.exceptions import AirtableBaseNotPublic
+from baserow.contrib.database.airtable.handler import BASE_HEADERS, AirtableHandler
+from baserow.contrib.database.airtable.utils import (
+ parse_json_and_remove_invalid_surrogate_characters,
+)
+from baserow.core.models import Workspace
+from baserow.core.utils import Progress, remove_invalid_surrogate_characters
+
+
+class Command(BaseCommand):
+ help = (
+ "This command fetches all Airtable templates, and attemps to import them into "
+ "the given workspace. It's created for testing purposes of the Airtable import."
+ )
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "workspace_id",
+ type=int,
+ help="The workspace ID where a copy of the imported Airtable base must be "
+ "added to.",
+ )
+ parser.add_argument(
+ "--start",
+ type=int,
+ help="From which index should the import start.",
+ default=0,
+ )
+ parser.add_argument(
+ "--limit",
+ type=int,
+ help="The maximum number of templates to install.",
+ default=-1,
+ )
+
+ def handle(self, *args, **options):
+ workspace_id = options["workspace_id"]
+ start_index = options["start"]
+ limit = options["limit"]
+
+ try:
+ workspace = Workspace.objects.get(pk=workspace_id)
+ except Workspace.DoesNotExist:
+ self.stdout.write(
+ self.style.ERROR(f"The workspace with id {workspace_id} was not found.")
+ )
+ sys.exit(1)
+
+ html_url = f"{AIRTABLE_BASE_URL}/templates"
+ html_response = requests.get(html_url, headers=BASE_HEADERS) # nosec
+
+ if not html_response.ok:
+ raise Exception("test")
+
+ decoded_content = remove_invalid_surrogate_characters(html_response.content)
+ raw_init_data = re.search(
+ "window.initData = (.*?)<\\/script>", decoded_content
+ ).group(1)
+ init_data = json.loads(raw_init_data)
+ client_code_version = init_data["codeVersion"]
+ page_load_id = init_data["pageLoadId"]
+
+ templates_url = (
+ f"{AIRTABLE_BASE_URL}/v0.3/exploreApplications"
+ f"?templateStatus=listed"
+ f"&shouldDisplayFull=true"
+ f"&descriptionSnippetMaxLength=300"
+ f"&categoryType=templateDesktopV2"
+ )
+
+ response = requests.get(
+ templates_url,
+ headers={
+ "x-airtable-inter-service-client": "webClient",
+ "x-airtable-inter-service-client-code-version": client_code_version,
+ "x-airtable-page-load-id": page_load_id,
+ "X-Requested-With": "XMLHttpRequest",
+ "x-time-zone": "Europe/Amsterdam",
+ "x-user-locale": "en",
+ **BASE_HEADERS,
+ },
+ timeout=3 * 60,
+ ) # nosec
+
+ json_decoded_content = parse_json_and_remove_invalid_surrogate_characters(
+ response
+ )
+
+ applications_by_id = json_decoded_content["exploreApplicationsById"].values()
+ i = 0
+ for index, application in enumerate(applications_by_id):
+ share_id = application["shareId"]
+ title = application["title"]
+
+ if limit != -1 and i >= limit:
+ print("finished!")
+ return
+
+ if index < start_index - 1:
+ print(
+ f"Skipping {title} {share_id} {index + 1}/{len(applications_by_id)}"
+ )
+ continue
+
+ i += 1
+ print(
+ f"Going to import {title} {share_id} {index + 1}/{len(applications_by_id)}"
+ )
+
+ with tqdm(total=1000) as progress_bar:
+ progress = Progress(1000)
+
+ def progress_updated(percentage, state):
+ progress_bar.set_description(state)
+ progress_bar.update(progress.progress - progress_bar.n)
+
+ progress.register_updated_event(progress_updated)
+
+ with NamedTemporaryFile() as download_files_buffer:
+ config = AirtableImportConfig(skip_files=True)
+ with transaction.atomic():
+ try:
+ AirtableHandler.import_from_airtable_to_workspace(
+ workspace,
+ share_id,
+ progress_builder=progress.create_child_builder(
+ represents_progress=progress.total
+ ),
+ download_files_buffer=download_files_buffer,
+ config=config,
+ )
+ except AirtableBaseNotPublic:
+ print(" Skipping because it's not public.")
diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
index 8b7cf3bb1..dc81a9ef3 100644
--- a/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
+++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py
@@ -289,6 +289,13 @@ def test_airtable_date_filter_value_to_baserow():
)
+def test_airtable_date_string_filter_value_to_baserow():
+ assert (
+ airtable_date_filter_value_to_baserow("2025-02-05T00:00:00.000Z")
+ == "?2025-02-05?exact_date"
+ )
+
+
def test_airtable_invalid_date_filter_value_to_baserow():
with pytest.raises(KeyError):
assert airtable_date_filter_value_to_baserow(
diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py
index cd7f42b1b..6786f9792 100644
--- a/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py
+++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py
@@ -567,6 +567,35 @@ def test_import_grid_view_filters_and_groups():
]
+@pytest.mark.django_db
+def test_import_grid_view_empty_filters():
+ view_data = deepcopy(RAW_AIRTABLE_VIEW_DATA)
+ field_mapping = deepcopy(FIELD_MAPPING)
+ for field_object in field_mapping.values():
+ field_object["baserow_field"].content_type = ContentType.objects.get_for_model(
+ field_object["baserow_field"]
+ )
+
+ view_data["filters"] = {"filterSet": [], "conjunction": "and"}
+
+ airtable_view_type = airtable_view_type_registry.get("grid")
+ import_report = AirtableImportReport()
+ serialized_view = airtable_view_type.to_serialized_baserow_view(
+ field_mapping,
+ ROW_ID_MAPPING,
+ RAW_AIRTABLE_TABLE,
+ RAW_AIRTABLE_VIEW,
+ view_data,
+ AirtableImportConfig(),
+ import_report,
+ )
+
+ assert serialized_view["filter_type"] == "AND"
+ assert serialized_view["filters_disabled"] is False
+ assert serialized_view["filters"] == []
+ assert serialized_view["filter_groups"] == []
+
+
@pytest.mark.django_db
def test_import_grid_view_color_config_select_column_not_existing_column():
view_data = deepcopy(RAW_AIRTABLE_VIEW_DATA)
diff --git a/web-frontend/modules/database/locales/en.json b/web-frontend/modules/database/locales/en.json
index c9738e4ec..8eeba08f7 100644
--- a/web-frontend/modules/database/locales/en.json
+++ b/web-frontend/modules/database/locales/en.json
@@ -757,12 +757,12 @@
"databaseForm": {
"importLabel": "Would you like to import existing data?",
"emptyLabel": "Start from scratch",
- "airtableLabel": "Import from Airtable (beta)"
+ "airtableLabel": "Import from Airtable"
},
"importFromAirtable": {
"airtableShareLinkTitle": "Share a link to your Base",
- "airtableShareLinkDescription": "To import your Airtable base, you need to have a shared link to your entire base. In Airtable, click on the share button in the top right corner after opening your base. After that you must choose the \"Access to base\" option. In the share modal you can click on the \"Create a shared link to the whole base\" button and then on “Private read-only link”. Copy the public link and paste it in the input below.",
- "airtableShareLinkBeta": "Note that this feature is in beta, your tables, fields (except formula, lookup and count) and data will be imported. Your views will not be imported.",
+ "airtableShareLinkDescription": "To import your Airtable base, you need to have a shared link to your entire base. In Airtable, click on the share button in the top right corner after opening your base. After that you must choose the \"Share via link\" option. In the share modal you can click on the \"Share publicly\" tab and then on “Enabl;e shared base link”. Copy the public link and paste it in the input below.",
+ "airtableShareLinkBeta": "This functionality will import most of the data, but there are incompatibilities. A table named \"Airtable import report\" will therefore be added containing a list of things that were not or partially imported.",
"airtableShareLinkPaste": "Paste the link here",
"importButtonLabel": "Import from Airtable",
"openButtonLabel": "Open imported database",