Allow choosing view in local Baserow table data sync

changelog/entries/unreleased/feature/3266_optionally_choose_view_in_baserow_table_data_sync.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "message": "Optionally allow choosing view in the local Baserow table data sync.",
+  "issue_number": 3266,
+  "bullet_points": [],
+  "created_at": "2024-12-26"
+}

enterprise/backend/src/baserow_enterprise/data_sync/baserow_table_data_sync.py

@@ -45,6 +45,9 @@ from baserow.contrib.database.fields.utils import get_field_id_from_field_key
 from baserow.contrib.database.rows.operations import ReadDatabaseRowOperationType
 from baserow.contrib.database.table.exceptions import TableDoesNotExist
 from baserow.contrib.database.table.handler import TableHandler
+from baserow.contrib.database.views.exceptions import ViewDoesNotExist
+from baserow.contrib.database.views.handler import ViewHandler
+from baserow.contrib.database.views.registries import view_type_registry
 from baserow.core.db import specific_iterator, specific_queryset
 from baserow.core.handler import CoreHandler
 from baserow.core.utils import ChildProgressBuilder
@@ -175,14 +178,20 @@ class BaserowFieldDataSyncProperty(DataSyncProperty):
 class LocalBaserowTableDataSyncType(DataSyncType):
     type = "local_baserow_table"
     model_class = LocalBaserowTableDataSync
-    allowed_fields = ["source_table_id", "authorized_user_id"]
+    allowed_fields = ["source_table_id", "source_table_view_id", "authorized_user_id"]
-    serializer_field_names = ["source_table_id"]
+    serializer_field_names = ["source_table_id", "source_table_view_id"]
     serializer_field_overrides = {
         "source_table_id": serializers.IntegerField(
             help_text="The ID of the source table that must be synced.",
             required=True,
             allow_null=False,
         ),
+        "source_table_view_id": serializers.IntegerField(
+            help_text="If provided, then only the visible fields and rows matching the "
+            "filters will be synced.",
+            required=False,
+            allow_null=True,
+        ),
     }
 
     def prepare_values(self, user, values):
@@ -207,7 +216,7 @@ class LocalBaserowTableDataSyncType(DataSyncType):
             instance.authorized_user = user
             instance.save()
 
-    def _get_table(self, instance):
+    def _get_table_and_view(self, instance):
         try:
             table = TableHandler().get_table(instance.source_table_id)
         except TableDoesNotExist:
@@ -222,10 +231,26 @@ class LocalBaserowTableDataSyncType(DataSyncType):
         ):
             raise SyncError("The authorized user doesn't have access to the table.")
 
-        return table
+        view = None
+        view_id = instance.source_table_view_id
+        if view_id is not None:
+            try:
+                view = (
+                    ViewHandler()
+                    .get_view_as_user(
+                        instance.authorized_user,
+                        instance.source_table_view_id,
+                        table_id=table.id,
+                    )
+                    .specific
+                )
+            except ViewDoesNotExist:
+                raise SyncError(f"The view with id {view_id} does not exist.")
+
+        return table, view
 
     def get_properties(self, instance) -> List[DataSyncProperty]:
-        table = self._get_table(instance)
+        table, view = self._get_table_and_view(instance)
         # The `table_id` is not set if when just listing the properties using the
         # `DataSyncTypePropertiesView` endpoint, but it will be set when creating the
         # view.
@@ -233,9 +258,18 @@ class LocalBaserowTableDataSyncType(DataSyncType):
             LicenseHandler.raise_if_workspace_doesnt_have_feature(
                 DATA_SYNC, instance.table.database.workspace
             )
-        fields = specific_iterator(
+
-            table.field_set.all().prefetch_related("select_options")
+        field_queryset = table.field_set.all().prefetch_related("select_options")
-        )
+
+        # If a view is provided, then we don't want to expose hidden fields,
+        # so we filter on the visible options to prevent that.
+        if view:
+            view_type = view_type_registry.get_by_model(view)
+            visible_field_options = view_type.get_visible_field_options_in_order(view)
+            visible_field_ids = {o.field_id for o in visible_field_options}
+            field_queryset = field_queryset.filter(id__in=visible_field_ids)
+
+        fields = specific_iterator(field_queryset)
         properties = [RowIDDataSyncProperty("id", "Row ID")]
 
         return properties + [
@@ -259,7 +293,7 @@ class LocalBaserowTableDataSyncType(DataSyncType):
         # that must completed. We're therefore using working with a total of 10 where
         # most of it is related to fetching the row values.
         progress = ChildProgressBuilder.build(progress_builder, child_total=10)
-        table = self._get_table(instance)
+        table, view = self._get_table_and_view(instance)
         enabled_properties = DataSyncSyncedProperty.objects.filter(
             data_sync=instance
         ).prefetch_related(
@@ -267,8 +301,16 @@ class LocalBaserowTableDataSyncType(DataSyncType):
         )
         enabled_property_field_ids = [p.key for p in enabled_properties]
         model = table.get_model()
+        queryset = model.objects.all()
+
+        # If a view is provided then we must not expose rows that don't match the
+        # filters.
+        if view:
+            queryset = ViewHandler().apply_filters(view, queryset)
+            queryset = ViewHandler().apply_sorting(view, queryset)
+
         progress.increment(by=1)  # makes the total `1`
-        rows_queryset = model.objects.all().values(*["id"] + enabled_property_field_ids)
+        rows_queryset = queryset.values(*["id"] + enabled_property_field_ids)
         progress.increment(by=7)  # makes the total `8`
 
         # Loop over all properties and rows to prepare the value if needed .This is

enterprise/backend/src/baserow_enterprise/data_sync/models.py

@@ -3,6 +3,7 @@ from django.db import models
 
 from baserow.contrib.database.data_sync.models import DataSync
 from baserow.contrib.database.table.models import Table
+from baserow.contrib.database.views.models import View
 
 User = get_user_model()
 
@@ -15,6 +16,15 @@ class LocalBaserowTableDataSync(DataSync):
         help_text="The source table containing the data you would like to get the data "
         "from.",
     )
+    # Deliberately don't make a ForeignKey because if the view is deleted the data sync
+    # must fail in that case. If the view fields are filters are ignored, it could
+    # accidentally expose data.
+    source_table_view_id = models.PositiveIntegerField(
+        View,
+        null=True,
+        help_text="If provided, then only the visible fields and rows matching the "
+        "filters will be synced.",
+    )
     authorized_user = models.ForeignKey(
         User,
         null=True,

enterprise/backend/src/baserow_enterprise/migrations/0036_localbaserowtabledatasync_source_table_view_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.9 on 2024-12-26 12:08
+
+from django.db import migrations, models
+
+import baserow.contrib.database.views.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("baserow_enterprise", "0035_hubspotcontactsdatasync"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="localbaserowtabledatasync",
+            name="source_table_view_id",
+            field=models.PositiveIntegerField(
+                help_text="If provided, then only the visible fields and rows matching the filters will be synced.",
+                null=True,
+                verbose_name=baserow.contrib.database.views.models.View,
+            ),
+        ),
+    ]

enterprise/backend/tests/baserow_enterprise_tests/data_sync/test_local_baserow_table_data_sync_type.py

@@ -1268,3 +1268,319 @@ def test_change_source_table_with_changing_synced_fields(
     # Expect the other field to be removed.
     assert len(response_json["synced_properties"]) == 1
     assert response_json["synced_properties"][0]["key"] == "id"
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_create_data_sync_view_does_not_exist(enterprise_data_fixture):
+    enterprise_data_fixture.enable_enterprise()
+
+    user = enterprise_data_fixture.create_user()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+
+    database = enterprise_data_fixture.create_database_application(user=user)
+    handler = DataSyncHandler()
+
+    with pytest.raises(SyncError) as e:
+        handler.create_data_sync_table(
+            user=user,
+            database=database,
+            table_name="Test",
+            type_name="local_baserow_table",
+            synced_properties=["id"],
+            source_table_id=source_table.id,
+            source_table_view_id=0,
+        )
+
+    assert "does not exist" in str(e)
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_create_data_sync_view_does_not_belong_to_table(enterprise_data_fixture):
+    enterprise_data_fixture.enable_enterprise()
+
+    user = enterprise_data_fixture.create_user()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+    view = enterprise_data_fixture.create_grid_view()
+
+    database = enterprise_data_fixture.create_database_application(user=user)
+    handler = DataSyncHandler()
+
+    with pytest.raises(SyncError) as e:
+        handler.create_data_sync_table(
+            user=user,
+            database=database,
+            table_name="Test",
+            type_name="local_baserow_table",
+            synced_properties=["id"],
+            source_table_id=source_table.id,
+            source_table_view_id=view.id,
+        )
+
+    assert "does not exist" in str(e)
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_create_data_sync_with_view_provided(enterprise_data_fixture):
+    enterprise_data_fixture.enable_enterprise()
+
+    user = enterprise_data_fixture.create_user()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+    public_field = enterprise_data_fixture.create_text_field(
+        table=source_table, name="Text", primary=True
+    )
+
+    grid = enterprise_data_fixture.create_grid_view(
+        table=source_table, user=user, public=True, create_options=False
+    )
+    enterprise_data_fixture.create_grid_view_field_option(
+        grid, public_field, hidden=False
+    )
+
+    database = enterprise_data_fixture.create_database_application(user=user)
+    handler = DataSyncHandler()
+
+    data_sync = handler.create_data_sync_table(
+        user=user,
+        database=database,
+        table_name="Test",
+        type_name="local_baserow_table",
+        synced_properties=["id", f"field_{public_field.id}"],
+        source_table_id=source_table.id,
+        source_table_view_id=grid.id,
+    )
+
+    assert isinstance(data_sync, LocalBaserowTableDataSync)
+    assert data_sync.source_table_id == source_table.id
+    assert data_sync.authorized_user_id == user.id
+
+    fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
+    assert len(fields) == 2
+    assert fields[0].name == "Row ID"
+    assert isinstance(fields[0], NumberField)
+    assert fields[0].primary is True
+    assert fields[0].read_only is True
+    assert fields[0].immutable_type is True
+    assert fields[0].immutable_properties is True
+    assert fields[1].name == "Text"
+    assert fields[1].primary is False
+    assert fields[1].read_only is True
+    assert fields[1].immutable_type is True
+    assert fields[1].immutable_properties is True
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_get_properties_with_view_provided_only_public_fields(
+    enterprise_data_fixture, api_client
+):
+    enterprise_data_fixture.enable_enterprise()
+
+    user, token = enterprise_data_fixture.create_user_and_token()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+    public_field = enterprise_data_fixture.create_text_field(
+        table=source_table, name="Text", primary=True
+    )
+    hidden_field = enterprise_data_fixture.create_text_field(
+        table=source_table,
+        name="Number",
+        primary=False,
+    )
+
+    grid = enterprise_data_fixture.create_grid_view(
+        table=source_table, user=user, public=True, create_options=False
+    )
+    enterprise_data_fixture.create_grid_view_field_option(
+        grid, public_field, hidden=False
+    )
+    enterprise_data_fixture.create_grid_view_field_option(
+        grid, hidden_field, hidden=True
+    )
+
+    url = reverse("api:database:data_sync:properties")
+    response = api_client.post(
+        url,
+        {
+            "type": "local_baserow_table",
+            "source_table_id": source_table.id,
+            "source_table_view_id": grid.id,
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"JWT {token}",
+    )
+    assert response.status_code == HTTP_200_OK
+    assert response.json() == [
+        {
+            "unique_primary": True,
+            "key": "id",
+            "name": "Row ID",
+            "field_type": "number",
+            "initially_selected": True,
+        },
+        {
+            "unique_primary": False,
+            "key": f"field_{public_field.id}",
+            "name": "Text",
+            "field_type": "text",
+            "initially_selected": True,
+        },
+    ]
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_get_properties_with_table_view_id_none(enterprise_data_fixture, api_client):
+    enterprise_data_fixture.enable_enterprise()
+
+    user, token = enterprise_data_fixture.create_user_and_token()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+
+    url = reverse("api:database:data_sync:properties")
+    response = api_client.post(
+        url,
+        {
+            "type": "local_baserow_table",
+            "source_table_id": source_table.id,
+            "source_table_view_id": None,
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"JWT {token}",
+    )
+    assert response.status_code == HTTP_200_OK
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_sync_data_sync_table_with_view_provided_having_filter_and_sort(
+    enterprise_data_fixture,
+):
+    enterprise_data_fixture.enable_enterprise()
+
+    user = enterprise_data_fixture.create_user()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+    public_field = enterprise_data_fixture.create_text_field(
+        table=source_table, name="Text", primary=True
+    )
+
+    grid = enterprise_data_fixture.create_grid_view(
+        table=source_table, user=user, public=True, create_options=False
+    )
+
+    enterprise_data_fixture.create_grid_view_field_option(
+        grid, public_field, hidden=False
+    )
+    enterprise_data_fixture.create_view_filter(
+        view=grid, field=public_field, type="not_equal", value="B"
+    )
+    enterprise_data_fixture.create_view_sort(view=grid, field=public_field, order="ASC")
+
+    source_model = source_table.get_model()
+    source_row_1 = source_model.objects.create(
+        **{
+            f"field_{public_field.id}": "C",
+        }
+    )
+    source_model.objects.create(
+        **{
+            f"field_{public_field.id}": "B",
+        }
+    )
+    source_model.objects.create(
+        **{
+            f"field_{public_field.id}": "A",
+        }
+    )
+
+    database = enterprise_data_fixture.create_database_application(user=user)
+    handler = DataSyncHandler()
+
+    data_sync = handler.create_data_sync_table(
+        user=user,
+        database=database,
+        table_name="Test",
+        type_name="local_baserow_table",
+        synced_properties=["id", f"field_{public_field.id}"],
+        source_table_id=source_table.id,
+        source_table_view_id=grid.id,
+    )
+
+    handler.sync_data_sync_table(user=user, data_sync=data_sync)
+
+    fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
+    field_1_field = fields[1]
+
+    model = data_sync.table.get_model()
+    assert model.objects.all().count() == 2
+    row = list(model.objects.all())
+    assert getattr(row[0], f"field_{field_1_field.id}") == "A"
+    assert getattr(row[1], f"field_{field_1_field.id}") == "C"
+
+
+@pytest.mark.django_db
+@override_settings(DEBUG=True)
+def test_source_table_view_deleted(enterprise_data_fixture):
+    enterprise_data_fixture.enable_enterprise()
+
+    user = enterprise_data_fixture.create_user()
+
+    source_table = enterprise_data_fixture.create_database_table(
+        user=user, name="Source"
+    )
+
+    grid = enterprise_data_fixture.create_grid_view(
+        table=source_table, user=user, public=True, create_options=False
+    )
+
+    database = enterprise_data_fixture.create_database_application(user=user)
+    handler = DataSyncHandler()
+
+    data_sync = handler.create_data_sync_table(
+        user=user,
+        database=database,
+        table_name="Test",
+        type_name="local_baserow_table",
+        synced_properties=["id"],
+        source_table_id=source_table.id,
+        source_table_view_id=grid.id,
+    )
+
+    grid_id = grid.id
+    grid.delete()
+
+    with pytest.raises(SyncError) as e:
+        handler.create_data_sync_table(
+            user=user,
+            database=database,
+            table_name="Test",
+            type_name="local_baserow_table",
+            synced_properties=["id"],
+            source_table_id=source_table.id,
+            source_table_view_id=grid.id,
+        )
+
+    assert "does not exist" in str(e)
+
+    data_sync.refresh_from_db()
+    # We expect the view to still exist so that it fails because if it's set to
+    # `null`, it might expose all table data.
+    assert data_sync.source_table_view_id == grid_id

enterprise/web-frontend/modules/baserow_enterprise/components/dataSync/LocalBaserowTableDataSync.vue

@@ -9,7 +9,7 @@
       />
     </div>
     <div class="row margin-bottom-3">
-      <div class="col col-4">
+      <div class="col col-6 margin-bottom-2">
         <FormGroup
           small-label
           :label="$t('localBaserowTableDataSync.workspace')"
@@ -29,7 +29,7 @@
           </Dropdown>
         </FormGroup>
       </div>
-      <div class="col col-4">
+      <div class="col col-6 margin-bottom-2">
         <FormGroup
           small-label
           :label="$t('localBaserowTableDataSync.database')"
@@ -49,7 +49,7 @@
           </Dropdown>
         </FormGroup>
       </div>
-      <div class="col col-4">
+      <div class="col col-6">
         <FormGroup
           :error="fieldHasErrors('source_table_id')"
           small-label
@@ -57,10 +57,10 @@
           required
         >
           <Dropdown
-            v-model="values.source_table_id"
+            :value="values.source_table_id"
             :error="fieldHasErrors('source_table_id')"
             :disabled="disabled"
-            @input="$v.values.source_table_id.$touch()"
+            @input="tableChanged"
           >
             <DropdownItem
               v-for="table in tables"
@@ -81,6 +81,32 @@
           </template>
         </FormGroup>
       </div>
+      <div class="col col-6">
+        <FormGroup
+          :error="fieldHasErrors('source_table_view_id')"
+          small-label
+          :label="$t('localBaserowTableDataSync.view')"
+          :help-icon-tooltip="$t('localBaserowTableDataSync.viewHelper')"
+          required
+        >
+          <div v-if="viewsLoading" class="loading"></div>
+          <Dropdown
+            v-else
+            v-model="values.source_table_view_id"
+            :error="fieldHasErrors('source_table_view_id')"
+            :disabled="disabled"
+            @input="$v.values.source_table_view_id.$touch()"
+          >
+            <DropdownItem
+              v-for="view in views"
+              :key="view.id"
+              :name="view.name"
+              :value="view.id"
+              :icon="view._.type.iconClass"
+            ></DropdownItem>
+          </Dropdown>
+        </FormGroup>
+      </div>
     </div>
   </form>
 </template>
@@ -91,6 +117,7 @@ import { required, numeric } from 'vuelidate/lib/validators'
 
 import form from '@baserow/modules/core/mixins/form'
 import { DatabaseApplicationType } from '@baserow/modules/database/applicationTypes'
+import ViewService from '@baserow/modules/database/services/view'
 
 export default {
   name: 'LocalBaserowTableDataSync',
@@ -109,13 +136,16 @@ export default {
   },
   data() {
     return {
-      allowedValues: ['source_table_id'],
+      allowedValues: ['source_table_id', 'source_table_view_id'],
       values: {
         source_table_id: '',
+        source_table_view_id: null,
       },
       selectedWorkspaceId:
         this.$store.getters['workspace/getSelected'].id || null,
       selectedDatabaseId: null,
+      views: [],
+      viewsLoading: false,
     }
   },
   computed: {
@@ -150,6 +180,13 @@ export default {
       userName: 'auth/getName',
     }),
   },
+  watch: {
+    'values.source_table_id'(newValueType, oldValue) {
+      if (newValueType !== oldValue) {
+        this.loadViewsIfNeeded()
+      }
+    },
+  },
   mounted() {
     // If the source table id is set, the database and workspace ID must be selected
     // in the dropdown.
@@ -175,6 +212,7 @@ export default {
   validations: {
     values: {
       source_table_id: { required, numeric },
+      source_table_view_id: { numeric },
     },
   },
   methods: {
@@ -185,6 +223,7 @@ export default {
       this.selectedWorkspaceId = value
       this.selectedDatabaseId = null
       this.values.source_table_id = null
+      this.values.source_table_view_id = null
     },
     databaseChanged(value) {
       if (this.selectedDatabaseId === value) {
@@ -192,6 +231,49 @@ export default {
       }
       this.selectedDatabaseId = value
       this.values.source_table_id = null
+      this.values.source_table_view_id = null
+    },
+    tableChanged(value) {
+      this.$v.values.source_table_id.$touch()
+      if (this.values.source_table_id === value) {
+        return
+      }
+      this.values.source_table_id = value
+      this.values.source_table_view_id = null
+    },
+    async loadViewsIfNeeded() {
+      if (this.values.source_table_id === null) {
+        return
+      }
+
+      this.viewsLoading = true
+
+      try {
+        // Because the authorized user changes when a view is created or updated, it's
+        // fine to just fetch all the views that the user has access to.
+        const { data } = await ViewService(this.$client).fetchAll(
+          this.values.source_table_id,
+          false,
+          false,
+          false,
+          false
+        )
+        this.views = data
+          .filter((view) => {
+            const viewType = this.$registry.get('view', view.type)
+            return viewType.canFilter
+          })
+          .map((view) => {
+            const viewType = this.$registry.get('view', view.type)
+            view._ = { type: viewType.serialize() }
+            return view
+          })
+          .sort((a, b) => {
+            return a.order - b.order
+          })
+      } finally {
+        this.viewsLoading = false
+      }
     },
   },
 }

enterprise/web-frontend/modules/baserow_enterprise/locales/en.json

@@ -370,7 +370,9 @@
     "authorizing": "You're authorizing your account to select the data from the source table.",
     "workspace": "Workspace",
     "database": "Database",
-    "table": "Table"
+    "table": "Table",
+    "view": "View",
+    "viewHelper": "Only visibile fields and rows matching the filters are synced."
   },
   "jiraIssuesDataSync": {
     "jiraUrl": "Jira instance URL",

web-frontend/modules/database/components/dataSync/SyncTableModal.vue

@@ -57,6 +57,7 @@ export default {
   methods: {
     show() {
       this.job = null
+      this.hideError()
       modal.methods.show.bind(this)()
     },
     hidden() {