1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-07 22:35:36 +00:00

Allow choosing view in local Baserow table data sync

This commit is contained in:
Bram Wiepjes 2025-01-09 10:56:14 +00:00
parent 0900e8654a
commit 6fb9b87e63
8 changed files with 500 additions and 17 deletions
changelog/entries/unreleased/feature
enterprise
backend
web-frontend/modules/baserow_enterprise
components/dataSync
locales
web-frontend/modules/database/components/dataSync

View file

@ -0,0 +1,7 @@
{
"type": "feature",
"message": "Optionally allow choosing view in the local Baserow table data sync.",
"issue_number": 3266,
"bullet_points": [],
"created_at": "2024-12-26"
}

View file

@ -45,6 +45,9 @@ from baserow.contrib.database.fields.utils import get_field_id_from_field_key
from baserow.contrib.database.rows.operations import ReadDatabaseRowOperationType
from baserow.contrib.database.table.exceptions import TableDoesNotExist
from baserow.contrib.database.table.handler import TableHandler
from baserow.contrib.database.views.exceptions import ViewDoesNotExist
from baserow.contrib.database.views.handler import ViewHandler
from baserow.contrib.database.views.registries import view_type_registry
from baserow.core.db import specific_iterator, specific_queryset
from baserow.core.handler import CoreHandler
from baserow.core.utils import ChildProgressBuilder
@ -175,14 +178,20 @@ class BaserowFieldDataSyncProperty(DataSyncProperty):
class LocalBaserowTableDataSyncType(DataSyncType):
type = "local_baserow_table"
model_class = LocalBaserowTableDataSync
allowed_fields = ["source_table_id", "authorized_user_id"]
serializer_field_names = ["source_table_id"]
allowed_fields = ["source_table_id", "source_table_view_id", "authorized_user_id"]
serializer_field_names = ["source_table_id", "source_table_view_id"]
serializer_field_overrides = {
"source_table_id": serializers.IntegerField(
help_text="The ID of the source table that must be synced.",
required=True,
allow_null=False,
),
"source_table_view_id": serializers.IntegerField(
help_text="If provided, then only the visible fields and rows matching the "
"filters will be synced.",
required=False,
allow_null=True,
),
}
def prepare_values(self, user, values):
@ -207,7 +216,7 @@ class LocalBaserowTableDataSyncType(DataSyncType):
instance.authorized_user = user
instance.save()
def _get_table(self, instance):
def _get_table_and_view(self, instance):
try:
table = TableHandler().get_table(instance.source_table_id)
except TableDoesNotExist:
@ -222,10 +231,26 @@ class LocalBaserowTableDataSyncType(DataSyncType):
):
raise SyncError("The authorized user doesn't have access to the table.")
return table
view = None
view_id = instance.source_table_view_id
if view_id is not None:
try:
view = (
ViewHandler()
.get_view_as_user(
instance.authorized_user,
instance.source_table_view_id,
table_id=table.id,
)
.specific
)
except ViewDoesNotExist:
raise SyncError(f"The view with id {view_id} does not exist.")
return table, view
def get_properties(self, instance) -> List[DataSyncProperty]:
table = self._get_table(instance)
table, view = self._get_table_and_view(instance)
# The `table_id` is not set if when just listing the properties using the
# `DataSyncTypePropertiesView` endpoint, but it will be set when creating the
# view.
@ -233,9 +258,18 @@ class LocalBaserowTableDataSyncType(DataSyncType):
LicenseHandler.raise_if_workspace_doesnt_have_feature(
DATA_SYNC, instance.table.database.workspace
)
fields = specific_iterator(
table.field_set.all().prefetch_related("select_options")
)
field_queryset = table.field_set.all().prefetch_related("select_options")
# If a view is provided, then we don't want to expose hidden fields,
# so we filter on the visible options to prevent that.
if view:
view_type = view_type_registry.get_by_model(view)
visible_field_options = view_type.get_visible_field_options_in_order(view)
visible_field_ids = {o.field_id for o in visible_field_options}
field_queryset = field_queryset.filter(id__in=visible_field_ids)
fields = specific_iterator(field_queryset)
properties = [RowIDDataSyncProperty("id", "Row ID")]
return properties + [
@ -259,7 +293,7 @@ class LocalBaserowTableDataSyncType(DataSyncType):
# that must completed. We're therefore using working with a total of 10 where
# most of it is related to fetching the row values.
progress = ChildProgressBuilder.build(progress_builder, child_total=10)
table = self._get_table(instance)
table, view = self._get_table_and_view(instance)
enabled_properties = DataSyncSyncedProperty.objects.filter(
data_sync=instance
).prefetch_related(
@ -267,8 +301,16 @@ class LocalBaserowTableDataSyncType(DataSyncType):
)
enabled_property_field_ids = [p.key for p in enabled_properties]
model = table.get_model()
queryset = model.objects.all()
# If a view is provided then we must not expose rows that don't match the
# filters.
if view:
queryset = ViewHandler().apply_filters(view, queryset)
queryset = ViewHandler().apply_sorting(view, queryset)
progress.increment(by=1) # makes the total `1`
rows_queryset = model.objects.all().values(*["id"] + enabled_property_field_ids)
rows_queryset = queryset.values(*["id"] + enabled_property_field_ids)
progress.increment(by=7) # makes the total `8`
# Loop over all properties and rows to prepare the value if needed .This is

View file

@ -3,6 +3,7 @@ from django.db import models
from baserow.contrib.database.data_sync.models import DataSync
from baserow.contrib.database.table.models import Table
from baserow.contrib.database.views.models import View
User = get_user_model()
@ -15,6 +16,15 @@ class LocalBaserowTableDataSync(DataSync):
help_text="The source table containing the data you would like to get the data "
"from.",
)
# Deliberately don't make a ForeignKey because if the view is deleted the data sync
# must fail in that case. If the view fields are filters are ignored, it could
# accidentally expose data.
source_table_view_id = models.PositiveIntegerField(
View,
null=True,
help_text="If provided, then only the visible fields and rows matching the "
"filters will be synced.",
)
authorized_user = models.ForeignKey(
User,
null=True,

View file

@ -0,0 +1,23 @@
# Generated by Django 5.0.9 on 2024-12-26 12:08
from django.db import migrations, models
import baserow.contrib.database.views.models
class Migration(migrations.Migration):
dependencies = [
("baserow_enterprise", "0035_hubspotcontactsdatasync"),
]
operations = [
migrations.AddField(
model_name="localbaserowtabledatasync",
name="source_table_view_id",
field=models.PositiveIntegerField(
help_text="If provided, then only the visible fields and rows matching the filters will be synced.",
null=True,
verbose_name=baserow.contrib.database.views.models.View,
),
),
]

View file

@ -1268,3 +1268,319 @@ def test_change_source_table_with_changing_synced_fields(
# Expect the other field to be removed.
assert len(response_json["synced_properties"]) == 1
assert response_json["synced_properties"][0]["key"] == "id"
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_create_data_sync_view_does_not_exist(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id"],
source_table_id=source_table.id,
source_table_view_id=0,
)
assert "does not exist" in str(e)
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_create_data_sync_view_does_not_belong_to_table(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
view = enterprise_data_fixture.create_grid_view()
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
with pytest.raises(SyncError) as e:
handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id"],
source_table_id=source_table.id,
source_table_view_id=view.id,
)
assert "does not exist" in str(e)
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_create_data_sync_with_view_provided(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
public_field = enterprise_data_fixture.create_text_field(
table=source_table, name="Text", primary=True
)
grid = enterprise_data_fixture.create_grid_view(
table=source_table, user=user, public=True, create_options=False
)
enterprise_data_fixture.create_grid_view_field_option(
grid, public_field, hidden=False
)
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id", f"field_{public_field.id}"],
source_table_id=source_table.id,
source_table_view_id=grid.id,
)
assert isinstance(data_sync, LocalBaserowTableDataSync)
assert data_sync.source_table_id == source_table.id
assert data_sync.authorized_user_id == user.id
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
assert len(fields) == 2
assert fields[0].name == "Row ID"
assert isinstance(fields[0], NumberField)
assert fields[0].primary is True
assert fields[0].read_only is True
assert fields[0].immutable_type is True
assert fields[0].immutable_properties is True
assert fields[1].name == "Text"
assert fields[1].primary is False
assert fields[1].read_only is True
assert fields[1].immutable_type is True
assert fields[1].immutable_properties is True
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_get_properties_with_view_provided_only_public_fields(
enterprise_data_fixture, api_client
):
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
public_field = enterprise_data_fixture.create_text_field(
table=source_table, name="Text", primary=True
)
hidden_field = enterprise_data_fixture.create_text_field(
table=source_table,
name="Number",
primary=False,
)
grid = enterprise_data_fixture.create_grid_view(
table=source_table, user=user, public=True, create_options=False
)
enterprise_data_fixture.create_grid_view_field_option(
grid, public_field, hidden=False
)
enterprise_data_fixture.create_grid_view_field_option(
grid, hidden_field, hidden=True
)
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "local_baserow_table",
"source_table_id": source_table.id,
"source_table_view_id": grid.id,
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
assert response.json() == [
{
"unique_primary": True,
"key": "id",
"name": "Row ID",
"field_type": "number",
"initially_selected": True,
},
{
"unique_primary": False,
"key": f"field_{public_field.id}",
"name": "Text",
"field_type": "text",
"initially_selected": True,
},
]
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_get_properties_with_table_view_id_none(enterprise_data_fixture, api_client):
enterprise_data_fixture.enable_enterprise()
user, token = enterprise_data_fixture.create_user_and_token()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
url = reverse("api:database:data_sync:properties")
response = api_client.post(
url,
{
"type": "local_baserow_table",
"source_table_id": source_table.id,
"source_table_view_id": None,
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_200_OK
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_sync_data_sync_table_with_view_provided_having_filter_and_sort(
enterprise_data_fixture,
):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
public_field = enterprise_data_fixture.create_text_field(
table=source_table, name="Text", primary=True
)
grid = enterprise_data_fixture.create_grid_view(
table=source_table, user=user, public=True, create_options=False
)
enterprise_data_fixture.create_grid_view_field_option(
grid, public_field, hidden=False
)
enterprise_data_fixture.create_view_filter(
view=grid, field=public_field, type="not_equal", value="B"
)
enterprise_data_fixture.create_view_sort(view=grid, field=public_field, order="ASC")
source_model = source_table.get_model()
source_row_1 = source_model.objects.create(
**{
f"field_{public_field.id}": "C",
}
)
source_model.objects.create(
**{
f"field_{public_field.id}": "B",
}
)
source_model.objects.create(
**{
f"field_{public_field.id}": "A",
}
)
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id", f"field_{public_field.id}"],
source_table_id=source_table.id,
source_table_view_id=grid.id,
)
handler.sync_data_sync_table(user=user, data_sync=data_sync)
fields = specific_iterator(data_sync.table.field_set.all().order_by("id"))
field_1_field = fields[1]
model = data_sync.table.get_model()
assert model.objects.all().count() == 2
row = list(model.objects.all())
assert getattr(row[0], f"field_{field_1_field.id}") == "A"
assert getattr(row[1], f"field_{field_1_field.id}") == "C"
@pytest.mark.django_db
@override_settings(DEBUG=True)
def test_source_table_view_deleted(enterprise_data_fixture):
enterprise_data_fixture.enable_enterprise()
user = enterprise_data_fixture.create_user()
source_table = enterprise_data_fixture.create_database_table(
user=user, name="Source"
)
grid = enterprise_data_fixture.create_grid_view(
table=source_table, user=user, public=True, create_options=False
)
database = enterprise_data_fixture.create_database_application(user=user)
handler = DataSyncHandler()
data_sync = handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id"],
source_table_id=source_table.id,
source_table_view_id=grid.id,
)
grid_id = grid.id
grid.delete()
with pytest.raises(SyncError) as e:
handler.create_data_sync_table(
user=user,
database=database,
table_name="Test",
type_name="local_baserow_table",
synced_properties=["id"],
source_table_id=source_table.id,
source_table_view_id=grid.id,
)
assert "does not exist" in str(e)
data_sync.refresh_from_db()
# We expect the view to still exist so that it fails because if it's set to
# `null`, it might expose all table data.
assert data_sync.source_table_view_id == grid_id

View file

@ -9,7 +9,7 @@
/>
</div>
<div class="row margin-bottom-3">
<div class="col col-4">
<div class="col col-6 margin-bottom-2">
<FormGroup
small-label
:label="$t('localBaserowTableDataSync.workspace')"
@ -29,7 +29,7 @@
</Dropdown>
</FormGroup>
</div>
<div class="col col-4">
<div class="col col-6 margin-bottom-2">
<FormGroup
small-label
:label="$t('localBaserowTableDataSync.database')"
@ -49,7 +49,7 @@
</Dropdown>
</FormGroup>
</div>
<div class="col col-4">
<div class="col col-6">
<FormGroup
:error="fieldHasErrors('source_table_id')"
small-label
@ -57,10 +57,10 @@
required
>
<Dropdown
v-model="values.source_table_id"
:value="values.source_table_id"
:error="fieldHasErrors('source_table_id')"
:disabled="disabled"
@input="$v.values.source_table_id.$touch()"
@input="tableChanged"
>
<DropdownItem
v-for="table in tables"
@ -81,6 +81,32 @@
</template>
</FormGroup>
</div>
<div class="col col-6">
<FormGroup
:error="fieldHasErrors('source_table_view_id')"
small-label
:label="$t('localBaserowTableDataSync.view')"
:help-icon-tooltip="$t('localBaserowTableDataSync.viewHelper')"
required
>
<div v-if="viewsLoading" class="loading"></div>
<Dropdown
v-else
v-model="values.source_table_view_id"
:error="fieldHasErrors('source_table_view_id')"
:disabled="disabled"
@input="$v.values.source_table_view_id.$touch()"
>
<DropdownItem
v-for="view in views"
:key="view.id"
:name="view.name"
:value="view.id"
:icon="view._.type.iconClass"
></DropdownItem>
</Dropdown>
</FormGroup>
</div>
</div>
</form>
</template>
@ -91,6 +117,7 @@ import { required, numeric } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
import { DatabaseApplicationType } from '@baserow/modules/database/applicationTypes'
import ViewService from '@baserow/modules/database/services/view'
export default {
name: 'LocalBaserowTableDataSync',
@ -109,13 +136,16 @@ export default {
},
data() {
return {
allowedValues: ['source_table_id'],
allowedValues: ['source_table_id', 'source_table_view_id'],
values: {
source_table_id: '',
source_table_view_id: null,
},
selectedWorkspaceId:
this.$store.getters['workspace/getSelected'].id || null,
selectedDatabaseId: null,
views: [],
viewsLoading: false,
}
},
computed: {
@ -150,6 +180,13 @@ export default {
userName: 'auth/getName',
}),
},
watch: {
'values.source_table_id'(newValueType, oldValue) {
if (newValueType !== oldValue) {
this.loadViewsIfNeeded()
}
},
},
mounted() {
// If the source table id is set, the database and workspace ID must be selected
// in the dropdown.
@ -175,6 +212,7 @@ export default {
validations: {
values: {
source_table_id: { required, numeric },
source_table_view_id: { numeric },
},
},
methods: {
@ -185,6 +223,7 @@ export default {
this.selectedWorkspaceId = value
this.selectedDatabaseId = null
this.values.source_table_id = null
this.values.source_table_view_id = null
},
databaseChanged(value) {
if (this.selectedDatabaseId === value) {
@ -192,6 +231,49 @@ export default {
}
this.selectedDatabaseId = value
this.values.source_table_id = null
this.values.source_table_view_id = null
},
tableChanged(value) {
this.$v.values.source_table_id.$touch()
if (this.values.source_table_id === value) {
return
}
this.values.source_table_id = value
this.values.source_table_view_id = null
},
async loadViewsIfNeeded() {
if (this.values.source_table_id === null) {
return
}
this.viewsLoading = true
try {
// Because the authorized user changes when a view is created or updated, it's
// fine to just fetch all the views that the user has access to.
const { data } = await ViewService(this.$client).fetchAll(
this.values.source_table_id,
false,
false,
false,
false
)
this.views = data
.filter((view) => {
const viewType = this.$registry.get('view', view.type)
return viewType.canFilter
})
.map((view) => {
const viewType = this.$registry.get('view', view.type)
view._ = { type: viewType.serialize() }
return view
})
.sort((a, b) => {
return a.order - b.order
})
} finally {
this.viewsLoading = false
}
},
},
}

View file

@ -370,7 +370,9 @@
"authorizing": "You're authorizing your account to select the data from the source table.",
"workspace": "Workspace",
"database": "Database",
"table": "Table"
"table": "Table",
"view": "View",
"viewHelper": "Only visibile fields and rows matching the filters are synced."
},
"jiraIssuesDataSync": {
"jiraUrl": "Jira instance URL",

View file

@ -57,6 +57,7 @@ export default {
methods: {
show() {
this.job = null
this.hideError()
modal.methods.show.bind(this)()
},
hidden() {