archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-04 21:25:24 +00:00
Code Issues Projects Releases Wiki Activity

Resolve "Add support for public batch row operations (inserts, updates, deletes)"

Browse source
This commit is contained in:
Petr Stribny 2022-05-09 14:42:36 +00:00
parent 8b60db5382
commit aa23479400
8 changed files with 1564 additions and 35 deletions
backend
src/baserow/contrib/database
rows
handler.pysignals.py
views
handler.py
ws
public/rows
signals.py
rows
signals.py
tests/baserow/contrib/database
view
test_view_handler.py
ws/public
test_public_ws_rows_signals.py
docs/apis
web-socket-api.md

10
backend/src/baserow/contrib/database/rows/handler.py
View file

@ -16,6 +16,8 @@ from .exceptions import RowDoesNotExist, RowIdsNotUnique
from .signals import (
before_row_update,
before_row_delete,
before_rows_update,
before_rows_delete,
row_created,
rows_created,
row_updated,
@ -845,9 +847,9 @@ class RowHandler:
if field_id in row_values or field["name"] in row_values:
updated_field_ids.add(field_id)
before_return = before_row_update.send(
before_return = before_rows_update.send(
self,
row=list(rows_to_update),
rows=list(rows_to_update),
user=user,
table=table,
model=model,
@ -1212,8 +1214,8 @@ class RowHandler:
db_rows_ids = [db_row.id for db_row in rows]
raise RowDoesNotExist(sorted(list(set(row_ids) - set(db_rows_ids))))
before_return = before_row_delete.send(
self, row=rows, user=user, table=table, model=model
before_return = before_rows_delete.send(
self, rows=rows, user=user, table=table, model=model
)
trashed_rows = TrashedRows()

2
backend/src/baserow/contrib/database/rows/signals.py
View file

@ -5,6 +5,8 @@ from django.dispatch import Signal
# fails because of a validation error.
before_row_update = Signal()
before_row_delete = Signal()
before_rows_update = Signal()
before_rows_delete = Signal()
row_created = Signal()
rows_created = Signal()

97
backend/src/baserow/contrib/database/views/handler.py
View file

@ -1,4 +1,5 @@
from collections import defaultdict
from dataclasses import dataclass
from copy import deepcopy
from typing import (
Dict,
@ -6,6 +7,7 @@ from typing import (
List,
Optional,
Iterable,
Set,
Tuple,
Type,
Union,
@ -1673,30 +1675,38 @@ class ViewHandler:
serialized using user_field_names=True.
:return: A copy of the serialized_row with all hidden fields removed.
"""
return self.restrict_rows_for_view(view, [serialized_row])[0]
def restrict_rows_for_view(
self, view: View, serialized_rows: List[Dict[str, Any]]
self,
view: View,
serialized_rows: List[Dict[str, Any]],
allowed_row_ids: Optional[List[int]] = None,
) -> List[Dict[str, Any]]:
"""
Removes any fields which are hidden in the view from the provided serialized
row ensuring no data is leaked according to the views field options.
Removes any fields which are hidden in the view and any rows that don't match
the allowed list of ids from the provided serializes rows ensuring no data is
leaked.
:param view: The view to restrict the row by.
:param serialized_rows: A list of python dictionaries which are the result of
serializing the rows containing `field_XXX` keys per field value. They
must not be serialized using user_field_names=True.
:return: A copy of the serialized_row with all hidden fields removed.
:param allowed_row_ids: A list of ids of rows that can be returned. If set to
None, all passed rows can be returned.
:return: A copy of the allowed serialized_rows with all hidden fields removed.
"""
view_type = view_type_registry.get_by_model(view.specific_class)
hidden_field_options = view_type.get_hidden_field_options(view)
restricted_rows = []
for serialized_row in serialized_rows:
row_copy = deepcopy(serialized_row)
for hidden_field_option in hidden_field_options:
row_copy.pop(f"field_{hidden_field_option.field_id}", None)
restricted_rows.append(row_copy)
if allowed_row_ids is None or serialized_row["id"] in allowed_row_ids:
row_copy = deepcopy(serialized_row)
for hidden_field_option in hidden_field_options:
row_copy.pop(f"field_{hidden_field_option.field_id}", None)
restricted_rows.append(row_copy)
return restricted_rows
def _get_public_view_jwt_secret(self, view: View) -> str:
@ -1754,6 +1764,27 @@ class ViewHandler:
return False
@dataclass
class PublicViewRows:
"""
Keeps track of which rows are allowed to be sent as a public signal
for a particular view.
When no row ids are set it is assumed that any row id is allowed.
"""
ALL_ROWS_ALLOWED = None
view: View
allowed_row_ids: Optional[Set[int]]
def all_allowed(self):
return self.allowed_row_ids is PublicViewRows.ALL_ROWS_ALLOWED
def __iter__(self):
return iter((self.view, self.allowed_row_ids))
class CachingPublicViewRowChecker:
"""
A helper class to check which public views a row is visible in. Will pre-calculate
@ -1825,10 +1856,60 @@ class CachingPublicViewRowChecker:
return views + self._always_visible_views
def get_public_views_where_rows_are_visible(self, rows) -> List[PublicViewRows]:
"""
WARNING: If you are reusing the same checker and calling this method with the
same rows multiple times you must have correctly set which fields in the rows
might be updated in the checkers initials `updated_field_ids` attribute. This
is because for a given view, if we know none of the fields it filters on
will be updated we can cache the first check of if that rows exist as any
further changes to the rows wont be affecting filtered fields. Hence
`updated_field_ids` needs to be set if you are ever changing the rows and
reusing the same CachingPublicViewRowChecker instance.
:param rows: Rows in the checkers table.
:return: A list of PublicViewRows with view and a list of row ids where the rows
are visible for this checkers table.
"""
visible_views_rows = []
row_ids = {row.id for row in rows}
for view, filter_qs, can_use_cache in self._views_with_filters:
if can_use_cache:
for id in row_ids:
if id not in self._view_row_check_cache[view.id]:
visible_ids = set(self._check_rows_visible(filter_qs, rows))
for visible_id in visible_ids:
self._view_row_check_cache[view.id][visible_id] = True
break
else:
visible_ids = row_ids
if len(visible_ids) > 0:
visible_views_rows.append(PublicViewRows(view, visible_ids))
else:
visible_ids = set(self._check_rows_visible(filter_qs, rows))
if len(visible_ids) > 0:
visible_views_rows.append(PublicViewRows(view, visible_ids))
for visible_view in self._always_visible_views:
visible_views_rows.append(
PublicViewRows(visible_view, PublicViewRows.ALL_ROWS_ALLOWED)
)
return visible_views_rows
# noinspection PyMethodMayBeStatic
def _check_row_visible(self, filter_qs, row):
return filter_qs.filter(id=row.id).exists()
# noinspection PyMethodMayBeStatic
def _check_rows_visible(self, filter_qs, rows):
return filter_qs.filter(id__in=[row.id for row in rows]).values_list(
"id", flat=True
)
def _view_row_checks_can_be_cached(self, view):
if self._updated_field_ids is None:
return True

243
backend/src/baserow/contrib/database/ws/public/rows/signals.py
View file

@ -1,4 +1,4 @@
from typing import Optional, Any, Dict, Iterable
from typing import Optional, Any, Dict, Iterable, List
from django.db import transaction
from django.dispatch import receiver
@ -10,18 +10,21 @@ from baserow.contrib.database.api.rows.serializers import (
)
from baserow.contrib.database.rows import signals as row_signals
from baserow.contrib.database.table.models import GeneratedTableModel
from baserow.contrib.database.views.handler import ViewHandler
from baserow.contrib.database.views.handler import PublicViewRows, ViewHandler
from baserow.contrib.database.views.models import View
from baserow.contrib.database.views.registries import view_type_registry
from baserow.contrib.database.ws.rows.signals import (
before_row_update,
before_rows_update,
RealtimeRowMessages,
)
from baserow.ws.registries import page_registry
def _serialize_row(model, row):
return get_row_serializer_class(model, RowSerializer, is_response=True)(row).data
def _serialize_row(model, row, many=False):
return get_row_serializer_class(model, RowSerializer, is_response=True)(
row, many=many
).data
def _send_row_created_event_to_views(
@ -50,6 +53,33 @@ def _send_row_created_event_to_views(
)
def _send_rows_created_event_to_views(
serialized_rows: List[Dict[Any, Any]],
before: Optional[GeneratedTableModel],
public_views: List[PublicViewRows],
):
view_page_type = page_registry.get("view")
handler = ViewHandler()
for (public_view, visible_row_ids) in public_views:
view_type = view_type_registry.get_by_model(public_view.specific_class)
if not view_type.when_shared_publicly_requires_realtime_events:
continue
restricted_serialized_rows = handler.restrict_rows_for_view(
public_view, serialized_rows, visible_row_ids
)
view_page_type.broadcast(
RealtimeRowMessages.rows_created(
table_id=PUBLIC_PLACEHOLDER_ENTITY_ID,
serialized_rows=restricted_serialized_rows,
metadata={},
before=before,
),
slug=public_view.slug,
)
def _send_row_deleted_event_to_views(
serialized_deleted_row: Dict[Any, Any], public_views: Iterable[View]
):
@ -72,6 +102,29 @@ def _send_row_deleted_event_to_views(
)
def _send_rows_deleted_event_to_views(
serialized_deleted_rows: List[Dict[Any, Any]],
public_views: List[PublicViewRows],
):
view_page_type = page_registry.get("view")
handler = ViewHandler()
for (public_view, deleted_row_ids) in public_views:
view_type = view_type_registry.get_by_model(public_view.specific_class)
if not view_type.when_shared_publicly_requires_realtime_events:
continue
restricted_serialized_deleted_rows = handler.restrict_rows_for_view(
public_view, serialized_deleted_rows, deleted_row_ids
)
view_page_type.broadcast(
RealtimeRowMessages.rows_deleted(
table_id=PUBLIC_PLACEHOLDER_ENTITY_ID,
serialized_rows=restricted_serialized_deleted_rows,
),
slug=public_view.slug,
)
@receiver(row_signals.row_created)
def public_row_created(sender, row, before, user, table, model, **kwargs):
row_checker = ViewHandler().get_public_views_row_checker(
@ -86,12 +139,22 @@ def public_row_created(sender, row, before, user, table, model, **kwargs):
)
@receiver(row_signals.rows_created)
def public_rows_created(sender, rows, before, user, table, model, **kwargs):
row_checker = ViewHandler().get_public_views_row_checker(
table, model, only_include_views_which_want_realtime_events=True
)
transaction.on_commit(
lambda: _send_rows_created_event_to_views(
_serialize_row(model, rows, many=True),
before,
row_checker.get_public_views_where_rows_are_visible(rows),
),
)
@receiver(row_signals.before_row_delete)
def public_before_row_delete(sender, row, user, table, model, **kwargs):
# TODO: Batch row deletes are not yet supported for public grid.
# For now, this signal call will be ignored.
if isinstance(row, list):
return
row_checker = ViewHandler().get_public_views_row_checker(
table, model, only_include_views_which_want_realtime_events=True
)
@ -103,6 +166,19 @@ def public_before_row_delete(sender, row, user, table, model, **kwargs):
}
@receiver(row_signals.before_rows_delete)
def public_before_rows_delete(sender, rows, user, table, model, **kwargs):
row_checker = ViewHandler().get_public_views_row_checker(
table, model, only_include_views_which_want_realtime_events=True
)
return {
"deleted_rows_public_views": (
row_checker.get_public_views_where_rows_are_visible(rows)
),
"deleted_rows": _serialize_row(model, rows, many=True),
}
@receiver(row_signals.row_deleted)
def public_row_deleted(
sender, row_id, row, user, table, model, before_return, **kwargs
@ -118,14 +194,23 @@ def public_row_deleted(
)
@receiver(row_signals.rows_deleted)
def public_rows_deleted(sender, rows, user, table, model, before_return, **kwargs):
public_views = dict(before_return)[public_before_rows_delete][
"deleted_rows_public_views"
]
serialized_deleted_rows = dict(before_return)[public_before_rows_delete][
"deleted_rows"
]
transaction.on_commit(
lambda: _send_rows_deleted_event_to_views(serialized_deleted_rows, public_views)
)
@receiver(row_signals.before_row_update)
def public_before_row_update(
sender, row, user, table, model, updated_field_ids, **kwargs
):
# TODO: Batch row updates are not yet supported for public grid.
# For now, this signal call will be ignored.
if isinstance(row, list):
return
# Generate a serialized version of the row before it is updated. The
# `row_updated` receiver needs this serialized version because it can't serialize
# the old row after it has been updated.
@ -141,6 +226,24 @@ def public_before_row_update(
}
@receiver(row_signals.before_rows_update)
def public_before_rows_update(
sender, rows, user, table, model, updated_field_ids, **kwargs
):
row_checker = ViewHandler().get_public_views_row_checker(
table,
model,
only_include_views_which_want_realtime_events=True,
updated_field_ids=updated_field_ids,
)
return {
"old_rows_public_views": row_checker.get_public_views_where_rows_are_visible(
rows
),
"caching_row_checker": row_checker,
}
@receiver(row_signals.row_updated)
def public_row_updated(
sender, row, user, table, model, before_return, updated_field_ids, **kwargs
@ -206,3 +309,119 @@ def public_row_updated(
)
transaction.on_commit(_send_created_updated_deleted_row_signals_to_views)
@receiver(row_signals.rows_updated)
def public_rows_updated(
sender, rows, user, table, model, before_return, updated_field_ids, **kwargs
):
before_return_dict = dict(before_return)[public_before_rows_update]
serialized_old_rows = dict(before_return)[before_rows_update]
serialized_updated_rows = _serialize_row(model, rows, many=True)
old_row_public_views: List[PublicViewRows] = before_return_dict[
"old_rows_public_views"
]
existing_checker = before_return_dict["caching_row_checker"]
public_view_rows: List[
PublicViewRows
] = existing_checker.get_public_views_where_rows_are_visible(rows)
view_slug_to_updated_public_view_rows = {
view.view.slug: view for view in public_view_rows
}
# When a row is updated from the point of view of a public view it might not always
# result in a `row_updated` event. For example if the row was previously not visible
# in the public view due to its filters, but the row update makes it now match
# the filters we want to send a `row_created` event to that views page as the
# clients won't know anything about the row and hence a `row_updated` event makes
# no sense for them.
public_views_where_rows_were_created: List[PublicViewRows] = []
public_views_where_rows_were_updated: List[PublicViewRows] = []
public_views_where_rows_were_deleted: List[PublicViewRows] = []
for old_public_view_rows in old_row_public_views:
(old_row_view, old_visible_ids) = old_public_view_rows
updated_public_view_rows = view_slug_to_updated_public_view_rows.pop(
old_row_view.slug, None
)
if updated_public_view_rows is None:
public_views_where_rows_were_deleted.append(
PublicViewRows(old_row_view, None)
)
else:
new_visible_ids = updated_public_view_rows.allowed_row_ids
if (
old_visible_ids == PublicViewRows.ALL_ROWS_ALLOWED
and new_visible_ids == PublicViewRows.ALL_ROWS_ALLOWED
):
public_views_where_rows_were_updated.append(
PublicViewRows(old_row_view, PublicViewRows.ALL_ROWS_ALLOWED)
)
continue
if old_visible_ids == PublicViewRows.ALL_ROWS_ALLOWED:
old_visible_ids = new_visible_ids
if new_visible_ids == PublicViewRows.ALL_ROWS_ALLOWED:
new_visible_ids = old_visible_ids
deleted_ids = old_visible_ids - new_visible_ids
if len(deleted_ids) > 0:
public_views_where_rows_were_deleted.append(
PublicViewRows(old_row_view, deleted_ids)
)
created_ids = new_visible_ids - old_visible_ids
if len(created_ids) > 0:
public_views_where_rows_were_created.append(
PublicViewRows(old_row_view, created_ids)
)
updated_ids = new_visible_ids - created_ids - deleted_ids
if len(updated_ids) > 0:
public_views_where_rows_were_updated.append(
PublicViewRows(old_row_view, updated_ids)
)
# Any remaining views in the updated_rows_public_views dict are views which
# previously didn't show the old row, but now show the new row, so we want created.
public_views_where_rows_were_created = public_views_where_rows_were_created + list(
view_slug_to_updated_public_view_rows.values()
)
def _send_created_updated_deleted_row_signals_to_views():
_send_rows_deleted_event_to_views(
serialized_old_rows, public_views_where_rows_were_deleted
)
_send_rows_created_event_to_views(
serialized_updated_rows,
before=None,
public_views=public_views_where_rows_were_created,
)
view_page_type = page_registry.get("view")
handler = ViewHandler()
for (public_view, visible_row_ids) in public_views_where_rows_were_updated:
visible_fields_only_updated_rows = handler.restrict_rows_for_view(
public_view, serialized_updated_rows, visible_row_ids
)
visible_fields_only_old_rows = handler.restrict_rows_for_view(
public_view, serialized_old_rows, visible_row_ids
)
view_page_type.broadcast(
RealtimeRowMessages.rows_updated(
table_id=PUBLIC_PLACEHOLDER_ENTITY_ID,
serialized_rows_before_update=visible_fields_only_old_rows,
serialized_rows=visible_fields_only_updated_rows,
metadata={},
),
slug=public_view.slug,
)
transaction.on_commit(_send_created_updated_deleted_row_signals_to_views)

22
backend/src/baserow/contrib/database/ws/rows/signals.py
View file

@ -60,8 +60,13 @@ def before_row_update(sender, row, user, table, model, updated_field_ids, **kwar
# Generate a serialized version of the row before it is updated. The
# `row_updated` receiver needs this serialized version because it can't serialize
# the old row after it has been updated.
return get_row_serializer_class(model, RowSerializer, is_response=True)(row).data
@receiver(row_signals.before_rows_update)
def before_rows_update(sender, rows, user, table, model, updated_field_ids, **kwargs):
return get_row_serializer_class(model, RowSerializer, is_response=True)(
row, many=isinstance(row, list)
rows, many=True
).data
@ -97,7 +102,7 @@ def rows_updated(
lambda: table_page_type.broadcast(
RealtimeRowMessages.rows_updated(
table_id=table.id,
serialized_rows_before_update=dict(before_return)[before_row_update],
serialized_rows_before_update=dict(before_return)[before_rows_update],
serialized_rows=get_row_serializer_class(
model, RowSerializer, is_response=True
)(rows, many=True).data,
@ -116,13 +121,16 @@ def before_row_delete(sender, row, user, table, model, **kwargs):
# Generate a serialized version of the row before it is deleted. The
# `row_deleted` receiver needs this serialized version because it can't serialize
# the row after is has been deleted.
if isinstance(row, list):
return get_row_serializer_class(model, RowSerializer, is_response=True)(
row, many=True
).data
return get_row_serializer_class(model, RowSerializer, is_response=True)(row).data
@receiver(row_signals.before_rows_delete)
def before_rows_delete(sender, rows, user, table, model, **kwargs):
return get_row_serializer_class(model, RowSerializer, is_response=True)(
rows, many=True
).data
@receiver(row_signals.row_deleted)
def row_deleted(sender, row_id, row, user, table, model, before_return, **kwargs):
table_page_type = page_registry.get("table")
@ -144,7 +152,7 @@ def rows_deleted(sender, rows, user, table, model, before_return, **kwargs):
lambda: table_page_type.broadcast(
RealtimeRowMessages.rows_deleted(
table_id=table.id,
serialized_rows=dict(before_return)[before_row_delete],
serialized_rows=dict(before_return)[before_rows_delete],
),
getattr(user, "web_socket_id", None),
table_id=table.id,

84
backend/tests/baserow/contrib/database/view/test_view_handler.py
View file

@ -7,7 +7,7 @@ from django.core.exceptions import ValidationError
from baserow.contrib.database.rows.handler import RowHandler
from baserow.contrib.database.views.view_types import GridViewType
from baserow.core.exceptions import UserNotInGroup
from baserow.contrib.database.views.handler import ViewHandler
from baserow.contrib.database.views.handler import ViewHandler, PublicViewRows
from baserow.contrib.database.views.models import (
View,
GridView,
@ -1497,6 +1497,88 @@ def test_get_public_views_which_include_row(data_fixture, django_assert_num_quer
]
@pytest.mark.django_db
def test_get_public_views_which_include_rows(data_fixture):
user = data_fixture.create_user()
table = data_fixture.create_database_table(user=user)
visible_field = data_fixture.create_text_field(table=table)
hidden_field = data_fixture.create_text_field(table=table)
public_view1 = data_fixture.create_grid_view(
user,
create_options=False,
table=table,
public=True,
order=0,
)
public_view2 = data_fixture.create_grid_view(
user, table=table, public=True, order=1
)
public_view3 = data_fixture.create_grid_view(
user, table=table, public=True, order=2
)
# Should not appear in any results
data_fixture.create_form_view(user, table=table, public=True)
data_fixture.create_grid_view(user, table=table)
data_fixture.create_grid_view_field_option(public_view1, hidden_field, hidden=True)
data_fixture.create_grid_view_field_option(public_view2, hidden_field, hidden=True)
# Public View 1 has filters which match row 1
data_fixture.create_view_filter(
view=public_view1, field=visible_field, type="equal", value="Visible"
)
data_fixture.create_view_filter(
view=public_view1, field=hidden_field, type="equal", value="Hidden"
)
# Public View 2 has filters which match row 2
data_fixture.create_view_filter(
view=public_view2, field=visible_field, type="equal", value="Visible"
)
data_fixture.create_view_filter(
view=public_view2, field=hidden_field, type="equal", value="Not Match"
)
# Public View 3 has filters which match both rows
data_fixture.create_view_filter(
view=public_view2, field=visible_field, type="equal", value="Visible"
)
row = RowHandler().create_row(
user=user,
table=table,
values={
f"field_{visible_field.id}": "Visible",
f"field_{hidden_field.id}": "Hidden",
},
)
row2 = RowHandler().create_row(
user=user,
table=table,
values={
f"field_{visible_field.id}": "Visible",
f"field_{hidden_field.id}": "Not Match",
},
)
model = table.get_model()
checker = ViewHandler().get_public_views_row_checker(
table, model, only_include_views_which_want_realtime_events=True
)
assert checker.get_public_views_where_rows_are_visible([row, row2]) == [
PublicViewRows(
view=ViewHandler().get_view(public_view1.id), allowed_row_ids={1}
),
PublicViewRows(
view=ViewHandler().get_view(public_view2.id), allowed_row_ids={2}
),
PublicViewRows(
view=ViewHandler().get_view(public_view3.id),
allowed_row_ids=PublicViewRows.ALL_ROWS_ALLOWED,
),
]
@pytest.mark.django_db
def test_public_view_row_checker_caches_when_only_unfiltered_fields_updated(
data_fixture, django_assert_num_queries

1139
backend/tests/baserow/contrib/database/ws/public/test_public_ws_rows_signals.py
View file

File diff suppressed because it is too large Load diff

2
docs/apis/web-socket-api.md
View file

@ -150,6 +150,8 @@ are subscribed to the page.
* `row_deleted`
* `before_row_update`
* `before_row_delete`
* `before_rows_update`
* `before_rows_delete`
* `view_created`
* `view_updated`
* `view_deleted`