[3/3] Final airtable import changes

backend/src/baserow/contrib/database/airtable/registry.py

@@ -458,6 +458,7 @@ class AirtableViewType(Instance):
         filters = []
         conjunction = filter_object.get("conjunction", None)
         filter_set = filter_object.get("filterSet", None)
+        column_id = filter_object.get("columnId", None)
 
         if conjunction and filter_set:
             # The filter_object is a nested structure, where if the `conjunction` and
@@ -488,7 +489,7 @@ class AirtableViewType(Instance):
                 filters.extend(child_filters)
 
             return filters, filter_groups
-        else:
+        elif column_id:
             baserow_filter = self.get_filter(
                 field_mapping,
                 row_id_mapping,
@@ -504,6 +505,8 @@ class AirtableViewType(Instance):
             else:
                 return [baserow_filter], []
 
+        return [], []
+
     def get_select_column_decoration(
         self,
         field_mapping: dict,
@@ -567,7 +570,8 @@ class AirtableViewType(Instance):
             )
             # Pop the first group because that shouldn't be in Baserow, and the type is
             # defined on the view.
-            root_group = filter_groups.pop(0)
+            if len(filter_groups) > 0:
+                root_group = filter_groups.pop(0)
             color = AIRTABLE_BASEROW_COLOR_MAPPING.get(
                 color_definition.get("color", ""),
                 "blue",
@@ -709,7 +713,8 @@ class AirtableViewType(Instance):
             )
             # Pop the first group because that shouldn't be in Baserow, and the type is
             # defined on the view.
-            view.filter_type = filter_groups.pop(0).filter_type
+            if len(filter_groups) > 0:
+                view.filter_type = filter_groups.pop(0).filter_type
 
         sorts = self.get_sorts(
             field_mapping,

backend/src/baserow/contrib/database/airtable/utils.py

@@ -1,6 +1,6 @@
 import json
 import re
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from requests import Response
 
@@ -245,7 +245,7 @@ def quill_to_markdown(ops: list) -> str:
     return "".join(md_output).strip()
 
 
-def airtable_date_filter_value_to_baserow(value: Optional[dict]) -> str:
+def airtable_date_filter_value_to_baserow(value: Optional[Union[dict, str]]) -> str:
     """
     Converts the provided Airtable filter date value to the Baserow compatible date
     value string.
@@ -257,6 +257,15 @@ def airtable_date_filter_value_to_baserow(value: Optional[dict]) -> str:
     if value is None:
         return ""
 
+    # If the value is a string, then it contains an exact date. This is the old format
+    # of Airtable. In that case, we can conert it to the correct format.
+    if isinstance(value, str):
+        value = {
+            "mode": "exactDate",
+            "exactDate": value,
+            "timeZone": "",  # it's okay to leave the timezone empty in Baserow.
+        }
+
     mode = value["mode"]
     if "exactDate" in value:
         # By default, Airtable adds the time, but that is not needed in Baserow.

backend/src/baserow/contrib/database/management/commands/install_airtable_templates.py

@@ -0,0 +1,146 @@
+import json
+import re
+import sys
+from tempfile import NamedTemporaryFile
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+import requests
+from tqdm import tqdm
+
+from baserow.contrib.database.airtable.config import AirtableImportConfig
+from baserow.contrib.database.airtable.constants import AIRTABLE_BASE_URL
+from baserow.contrib.database.airtable.exceptions import AirtableBaseNotPublic
+from baserow.contrib.database.airtable.handler import BASE_HEADERS, AirtableHandler
+from baserow.contrib.database.airtable.utils import (
+    parse_json_and_remove_invalid_surrogate_characters,
+)
+from baserow.core.models import Workspace
+from baserow.core.utils import Progress, remove_invalid_surrogate_characters
+
+
+class Command(BaseCommand):
+    help = (
+        "This command fetches all Airtable templates, and attemps to import them into "
+        "the given workspace. It's created for testing purposes of the Airtable import."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "workspace_id",
+            type=int,
+            help="The workspace ID where a copy of the imported Airtable base must be "
+            "added to.",
+        )
+        parser.add_argument(
+            "--start",
+            type=int,
+            help="From which index should the import start.",
+            default=0,
+        )
+        parser.add_argument(
+            "--limit",
+            type=int,
+            help="The maximum number of templates to install.",
+            default=-1,
+        )
+
+    def handle(self, *args, **options):
+        workspace_id = options["workspace_id"]
+        start_index = options["start"]
+        limit = options["limit"]
+
+        try:
+            workspace = Workspace.objects.get(pk=workspace_id)
+        except Workspace.DoesNotExist:
+            self.stdout.write(
+                self.style.ERROR(f"The workspace with id {workspace_id} was not found.")
+            )
+            sys.exit(1)
+
+        html_url = f"{AIRTABLE_BASE_URL}/templates"
+        html_response = requests.get(html_url, headers=BASE_HEADERS)  # nosec
+
+        if not html_response.ok:
+            raise Exception("test")
+
+        decoded_content = remove_invalid_surrogate_characters(html_response.content)
+        raw_init_data = re.search(
+            "window.initData = (.*?)<\\/script>", decoded_content
+        ).group(1)
+        init_data = json.loads(raw_init_data)
+        client_code_version = init_data["codeVersion"]
+        page_load_id = init_data["pageLoadId"]
+
+        templates_url = (
+            f"{AIRTABLE_BASE_URL}/v0.3/exploreApplications"
+            f"?templateStatus=listed"
+            f"&shouldDisplayFull=true"
+            f"&descriptionSnippetMaxLength=300"
+            f"&categoryType=templateDesktopV2"
+        )
+
+        response = requests.get(
+            templates_url,
+            headers={
+                "x-airtable-inter-service-client": "webClient",
+                "x-airtable-inter-service-client-code-version": client_code_version,
+                "x-airtable-page-load-id": page_load_id,
+                "X-Requested-With": "XMLHttpRequest",
+                "x-time-zone": "Europe/Amsterdam",
+                "x-user-locale": "en",
+                **BASE_HEADERS,
+            },
+            timeout=3 * 60,
+        )  # nosec
+
+        json_decoded_content = parse_json_and_remove_invalid_surrogate_characters(
+            response
+        )
+
+        applications_by_id = json_decoded_content["exploreApplicationsById"].values()
+        i = 0
+        for index, application in enumerate(applications_by_id):
+            share_id = application["shareId"]
+            title = application["title"]
+
+            if limit != -1 and i >= limit:
+                print("finished!")
+                return
+
+            if index < start_index - 1:
+                print(
+                    f"Skipping {title} {share_id} {index + 1}/{len(applications_by_id)}"
+                )
+                continue
+
+            i += 1
+            print(
+                f"Going to import {title} {share_id} {index + 1}/{len(applications_by_id)}"
+            )
+
+            with tqdm(total=1000) as progress_bar:
+                progress = Progress(1000)
+
+                def progress_updated(percentage, state):
+                    progress_bar.set_description(state)
+                    progress_bar.update(progress.progress - progress_bar.n)
+
+                progress.register_updated_event(progress_updated)
+
+                with NamedTemporaryFile() as download_files_buffer:
+                    config = AirtableImportConfig(skip_files=True)
+                    with transaction.atomic():
+                        try:
+                            AirtableHandler.import_from_airtable_to_workspace(
+                                workspace,
+                                share_id,
+                                progress_builder=progress.create_child_builder(
+                                    represents_progress=progress.total
+                                ),
+                                download_files_buffer=download_files_buffer,
+                                config=config,
+                            )
+                        except AirtableBaseNotPublic:
+                            print("  Skipping because it's not public.")

backend/tests/baserow/contrib/database/airtable/test_airtable_utils.py

@@ -289,6 +289,13 @@ def test_airtable_date_filter_value_to_baserow():
     )
 
 
+def test_airtable_date_string_filter_value_to_baserow():
+    assert (
+        airtable_date_filter_value_to_baserow("2025-02-05T00:00:00.000Z")
+        == "?2025-02-05?exact_date"
+    )
+
+
 def test_airtable_invalid_date_filter_value_to_baserow():
     with pytest.raises(KeyError):
         assert airtable_date_filter_value_to_baserow(

backend/tests/baserow/contrib/database/airtable/test_airtable_view_types.py

@@ -567,6 +567,35 @@ def test_import_grid_view_filters_and_groups():
     ]
 
 
+@pytest.mark.django_db
+def test_import_grid_view_empty_filters():
+    view_data = deepcopy(RAW_AIRTABLE_VIEW_DATA)
+    field_mapping = deepcopy(FIELD_MAPPING)
+    for field_object in field_mapping.values():
+        field_object["baserow_field"].content_type = ContentType.objects.get_for_model(
+            field_object["baserow_field"]
+        )
+
+    view_data["filters"] = {"filterSet": [], "conjunction": "and"}
+
+    airtable_view_type = airtable_view_type_registry.get("grid")
+    import_report = AirtableImportReport()
+    serialized_view = airtable_view_type.to_serialized_baserow_view(
+        field_mapping,
+        ROW_ID_MAPPING,
+        RAW_AIRTABLE_TABLE,
+        RAW_AIRTABLE_VIEW,
+        view_data,
+        AirtableImportConfig(),
+        import_report,
+    )
+
+    assert serialized_view["filter_type"] == "AND"
+    assert serialized_view["filters_disabled"] is False
+    assert serialized_view["filters"] == []
+    assert serialized_view["filter_groups"] == []
+
+
 @pytest.mark.django_db
 def test_import_grid_view_color_config_select_column_not_existing_column():
     view_data = deepcopy(RAW_AIRTABLE_VIEW_DATA)

web-frontend/modules/database/locales/en.json

@@ -757,12 +757,12 @@
   "databaseForm": {
     "importLabel": "Would you like to import existing data?",
     "emptyLabel": "Start from scratch",
-    "airtableLabel": "Import from Airtable (beta)"
+    "airtableLabel": "Import from Airtable"
   },
   "importFromAirtable": {
     "airtableShareLinkTitle": "Share a link to your Base",
-    "airtableShareLinkDescription": "To import your Airtable base, you need to have a shared link to your entire base. In Airtable, click on the share button in the top right corner after opening your base. After that you must choose the \"Access to base\" option. In the share modal you can click on the \"Create a shared link to the whole base\" button and then on “Private read-only link”. Copy the public link and paste it in the input below.",
-    "airtableShareLinkBeta": "Note that this feature is in beta, your tables, fields (except formula, lookup and count) and data will be imported. Your views will not be imported.",
+    "airtableShareLinkDescription": "To import your Airtable base, you need to have a shared link to your entire base. In Airtable, click on the share button in the top right corner after opening your base. After that you must choose the \"Share via link\" option. In the share modal you can click on the \"Share publicly\" tab and then on “Enabl;e shared base link”. Copy the public link and paste it in the input below.",
+    "airtableShareLinkBeta": "This functionality will import most of the data, but there are incompatibilities. A table named \"Airtable import report\" will therefore be added containing a list of things that were not or partially imported.",
     "airtableShareLinkPaste": "Paste the link here",
     "importButtonLabel": "Import from Airtable",
     "openButtonLabel": "Open imported database",