1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-15 01:28:30 +00:00

Merge branch '3364-login-is-slow' into 'develop'

Resolve "Login is slow"

Closes 

See merge request 
This commit is contained in:
Davide Silvestri 2025-02-24 11:02:09 +00:00
commit 966ecdb801
24 changed files with 233 additions and 77 deletions

View file

@ -85,28 +85,33 @@ class AllApplicationsView(APIView):
returned.
"""
workspaces = Workspace.objects.filter(users=request.user)
workspaces = Workspace.objects.filter(users=request.user).prefetch_related(
"workspaceuser_set", "template_set"
)
# Compute list of readable application ids
applications_ids = []
all_applications_qs = None
for workspace in workspaces:
applications = Application.objects.filter(
workspace=workspace, workspace__trashed=False
)
applications = CoreHandler().filter_queryset(
).select_related("content_type")
applications_qs = CoreHandler().filter_queryset(
request.user,
ListApplicationsWorkspaceOperationType.type,
applications,
workspace=workspace,
)
applications_ids += applications.values_list("id", flat=True)
if all_applications_qs is None:
all_applications_qs = applications_qs
else:
all_applications_qs = all_applications_qs.union(applications_qs)
# Then filter with these ids
applications = specific_iterator(
Application.objects.select_related("content_type", "workspace")
.prefetch_related("workspace__template_set")
.filter(id__in=applications_ids)
.order_by("workspace_id", "order"),
.filter(id__in=all_applications_qs.values("id"))
.order_by("workspace_id", "order", "id"),
per_content_type_queryset_hook=(
lambda model, queryset: application_type_registry.get_by_model(
model

View file

@ -48,8 +48,7 @@ class BuilderSerializer(serializers.ModelSerializer):
:return: A list of serialized pages that belong to this instance.
"""
pages = PageHandler().get_pages(instance)
pages = instance.page_set.all()
user = self.context.get("user")
request = self.context.get("request")

View file

@ -7,6 +7,7 @@ from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.core.files.storage import Storage
from django.db import transaction
from django.db.models import Prefetch
from django.db.transaction import Atomic
from django.urls import include, path
@ -495,8 +496,10 @@ class BuilderApplicationType(ApplicationType):
return None
def enhance_queryset(self, queryset):
queryset = queryset.prefetch_related("page_set")
queryset = queryset.prefetch_related("user_sources")
queryset = queryset.prefetch_related("integrations")
queryset = queryset.select_related("favicon_file").prefetch_related(
"user_sources",
"integrations",
Prefetch("page_set", queryset=Page.objects_with_shared.all()),
)
queryset = theme_config_block_registry.enhance_list_builder_queryset(queryset)
return queryset

View file

@ -3,6 +3,7 @@ from typing import Type, TypeVar
from django.db.models import QuerySet
from baserow.contrib.builder.models import Builder
from baserow.core.registry import (
CustomFieldsInstanceMixin,
CustomFieldsRegistryMixin,
@ -89,6 +90,21 @@ class ThemeConfigBlockType(
return instance
def enhance_queryset(self, queryset: QuerySet[Builder]) -> QuerySet[Builder]:
"""
Enhance the queryset to select the related theme config block model. This method
is used by enhance_list_builder_queryset to select all related theme config
block models in a single query. By default, this method selects the related
theme config but it can be customized by subclasses to add additional
select_related or prefetch_related calls.
:param queryset: The queryset that lists the builder applications.
:return: The same queryset with proper select_related and/or prefetch_related to
reduce the number of queries necessary to fetch the data.
"""
return queryset.select_related(self.related_name_in_builder_model)
ThemeConfigBlockTypeSubClass = TypeVar(
"ThemeConfigBlockTypeSubClass", bound=ThemeConfigBlockType
@ -115,9 +131,8 @@ class ThemeConfigBlockRegistry(
:return: The enhanced queryset.
"""
for theme_config_block in self.get_all():
related_name = theme_config_block.related_name_in_builder_model
queryset = queryset.select_related(related_name)
for theme_config_block_type in self.get_all():
queryset = theme_config_block_type.enhance_queryset(queryset)
return queryset

View file

@ -2,9 +2,11 @@ from typing import Any, Dict, Optional
from zipfile import ZipFile
from django.core.files.storage import Storage
from django.db.models import QuerySet
from rest_framework import serializers
from baserow.contrib.builder.models import Builder
from baserow.core.user_files.handler import UserFileHandler
from .models import (
@ -166,6 +168,11 @@ class PageThemeConfigBlockType(ThemeConfigBlockType):
return value
def enhance_queryset(self, queryset: QuerySet[Builder]) -> QuerySet[Builder]:
return queryset.select_related(
f"{self.related_name_in_builder_model}__page_background_file"
)
class InputThemeConfigBlockType(ThemeConfigBlockType):
type = "input"

View file

@ -191,7 +191,7 @@ def fill_workspace_with_data(
with transaction.atomic():
database = (
CoreHandler()
.create_application(user, workspace, "database", faker.name())
.create_application(user, workspace, "database", name=faker.name())
.specific
)
created_databases_and_tables[database] = []

View file

@ -44,6 +44,25 @@ class LocalCache:
return cache[key]
def delete(self, key: str):
"""
Delete a value from the cache. If the key does not exist, no action is taken.
If the key ends with "*", all keys starting with the prefix are deleted.
:param key: The key to delete from the cache.
"""
if not hasattr(self._local, "cache"):
return
if key.endswith("*"):
for k in list(
filter(lambda k: k.startswith(key[:-1]), self._local.cache.keys())
):
del self._local.cache[k]
else:
del self._local.cache[key]
def clear(self):
"""
Clear all data from the cache.

View file

@ -24,6 +24,7 @@ from loguru import logger
from opentelemetry import trace
from tqdm import tqdm
from baserow.core.db import specific_iterator
from baserow.core.registries import plugin_registry
from baserow.core.user.utils import normalize_email_address
@ -1339,10 +1340,17 @@ class CoreHandler(metaclass=baserow_trace_methods(tracer)):
base_queryset = Application.objects
try:
application = base_queryset.select_related("workspace", "content_type").get(
id=application_id
)
except Application.DoesNotExist as e:
application = specific_iterator(
base_queryset.select_related("workspace", "content_type").filter(
id=application_id
),
per_content_type_queryset_hook=(
lambda model, queryset: application_type_registry.get_by_model(
model
).enhance_queryset(queryset)
),
)[0]
except IndexError as e:
raise ApplicationDoesNotExist(
f"The application with id {application_id} does not exist."
) from e

View file

@ -279,7 +279,7 @@ class Workspace(HierarchicalModelMixin, TrashableModelMixin, CreatedAndUpdatedOn
@lru_cache
def has_template(self):
return self.template_set.all().exists()
return len(self.template_set.all()) > 0
def get_workspace_user(
self, user: User, include_trash: bool = False

View file

@ -186,9 +186,18 @@ class WorkspaceMemberOnlyPermissionManagerType(PermissionManagerType):
if callback is not None:
in_workspace = callback()
else:
in_workspace = WorkspaceUser.objects.filter(
user_id=actor.id, workspace_id=workspace.id
).exists()
def _in_workspace():
user_iterator = (
wu.user_id
for wu in workspace.workspaceuser_set.all()
if wu.user_id == actor.id
)
return next(user_iterator, None) is not None
in_workspace = local_cache.get(
f"user_{actor.id}_in_workspace_{workspace.id}", _in_workspace
)
getattr(actor, self.actor_cache_key)[workspace.id] = in_workspace

View file

@ -0,0 +1,7 @@
{
"type": "refactor",
"message": "Reduce the number of queries when logging in or load Baserow.",
"issue_number": 3364,
"bullet_points": [],
"created_at": "2025-02-20"
}

View file

@ -8,6 +8,7 @@ from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
from baserow_premium.license.handler import LicenseHandler
from baserow.core.cache import local_cache
from baserow.core.exceptions import PermissionDenied
from baserow.core.handler import CoreHandler
from baserow.core.mixins import TrashableModelMixin
@ -45,6 +46,34 @@ from .constants import (
from .types import NewRoleAssignment
User = get_user_model()
ROLE_ASSIGNMENT_CACHE_KEY_PREFIX = "role_assignments"
def _clear_role_assignments_from_local_cache():
"""
Simple helper to clear the local cache for role assignments when needed.
"""
local_cache.delete(f"{ROLE_ASSIGNMENT_CACHE_KEY_PREFIX}_*")
def clear_roles_from_local_cache():
"""
Decorator to use for methods that need to clear the role assignment cache at the end
of their implementation.
"""
def decorator(method):
def wrapper(*args, **kwargs):
try:
result = method(*args, **kwargs)
finally:
_clear_role_assignments_from_local_cache()
return result
return wrapper
return decorator
class RoleAssignmentHandler:
@ -322,7 +351,18 @@ class RoleAssignmentHandler:
subject_id__in=actor_by_id.keys(),
)
teams_subjects = users_teams_qs.values_list("team_id", "subject_id")
model_class_meta = actor_subject_type.model_class._meta
actor_type = f"{model_class_meta.app_label}.{model_class_meta.model_name}"
actor_ids = "_".join([str(a.id) for a in actors])
actors_cache_key = f"{actor_type}_{actor_ids}"
def _get_teams_subjects():
return users_teams_qs.values_list("team_id", "subject_id")
teams_subjects = local_cache.get(
f"{ROLE_ASSIGNMENT_CACHE_KEY_PREFIX}_{actors_cache_key}",
_get_teams_subjects,
)
# Populate double map for later use
subjects_per_team = defaultdict(list)
@ -364,27 +404,33 @@ class RoleAssignmentHandler:
]
# Final query
role_assignments = (
RoleAssignment.objects.filter(
workspace=workspace,
def _get_role_assignments():
return (
RoleAssignment.objects.filter(
workspace=workspace,
)
.filter(subjects_q, ~Q(role__uid=NO_ROLE_LOW_PRIORITY_ROLE_UID))
.annotate(
scope_type_order=Case(
*scope_cases,
default=Value(0),
output_field=IntegerField(),
),
role_priority=Case(
*role_priority_cases,
default=Value(0),
output_field=IntegerField(),
),
)
.order_by(
"scope_type_order", "scope_id", "role_priority", "subject_id", "id"
)
.select_related("subject_type")
)
.filter(subjects_q, ~Q(role__uid=NO_ROLE_LOW_PRIORITY_ROLE_UID))
.annotate(
scope_type_order=Case(
*scope_cases,
default=Value(0),
output_field=IntegerField(),
),
role_priority=Case(
*role_priority_cases,
default=Value(0),
output_field=IntegerField(),
),
)
.order_by(
"scope_type_order", "scope_id", "role_priority", "subject_id", "id"
)
.select_related("subject_type")
role_assignments = local_cache.get(
f"{ROLE_ASSIGNMENT_CACHE_KEY_PREFIX}_{workspace.id}_{actors_cache_key}",
_get_role_assignments,
)
workspace_scope_param = (content_types[Workspace].id, workspace.id)
@ -442,12 +488,24 @@ class RoleAssignmentHandler:
# WorkspaceUser permissions property
if actor_subject_type.type == UserSubjectType.type:
# Get all workspace users at once
user_permissions_by_id = dict(
CoreHandler()
.get_workspace_users(
workspace, actor_by_id.values(), include_trash=include_trash
)
.values_list("user_id", "permissions")
actor_ids_set = {a.id for a in actors}
def _get_user_permissions_by_id():
if include_trash:
wp_users = WorkspaceUser.objects_and_trash.filter(
workspace_id=workspace.id, user_id__in=actor_ids_set
)
else:
wp_users = workspace.workspaceuser_set.all()
return {
wu.user_id: wu.permissions
for wu in wp_users
if wu.user_id in actor_ids_set
}
user_permissions_by_id = local_cache.get(
f"{ROLE_ASSIGNMENT_CACHE_KEY_PREFIX}_{workspace.id}_{actors_cache_key}_{include_trash}",
_get_user_permissions_by_id,
)
for actor in actors:
@ -534,6 +592,7 @@ class RoleAssignmentHandler:
return most_precise_roles
@clear_roles_from_local_cache()
def assign_role(
self, subject, workspace, role=None, scope=None, send_signals: bool = True
) -> Optional[RoleAssignment]:
@ -608,6 +667,7 @@ class RoleAssignmentHandler:
return role_assignment
@clear_roles_from_local_cache()
def remove_role(
self, subject: Union[AbstractUser, Team], workspace: Workspace, scope=None
):

View file

@ -9,7 +9,6 @@ from django.utils import timezone as django_timezone
import pytest
import responses
from baserow_premium.license.exceptions import FeaturesNotAvailableError
from baserow_premium.license.models import License
from freezegun.api import freeze_time
from baserow.contrib.database.data_sync.handler import DataSyncHandler
@ -377,7 +376,7 @@ def test_skip_automatically_deactivated_periodic_data_syncs(enterprise_data_fixt
when=time(hour=12, minute=10, second=1, microsecond=1),
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with freeze_time("2024-10-10T12:15:00.00Z"):
with transaction.atomic():

View file

@ -744,7 +744,7 @@ def test_sync_data_sync_table_without_license(enterprise_data_fixture):
github_issues_api_token="test",
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)

View file

@ -818,7 +818,7 @@ def test_sync_data_sync_table_without_license(enterprise_data_fixture):
gitlab_access_token="test",
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)

View file

@ -610,7 +610,7 @@ def test_sync_data_sync_table_without_license(enterprise_data_fixture):
hubspot_access_token="test",
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)

View file

@ -1061,7 +1061,7 @@ def test_sync_data_sync_table_without_license(enterprise_data_fixture):
jira_api_token="test_token",
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)

View file

@ -277,10 +277,7 @@ def test_sync_data_sync_table_authorized_user_is_set(enterprise_data_fixture):
user = enterprise_data_fixture.create_user()
user_2 = enterprise_data_fixture.create_user()
workspace = enterprise_data_fixture.create_workspace(user=user)
enterprise_data_fixture.create_user_workspace(
workspace=workspace, user=user_2, order=0
)
workspace = enterprise_data_fixture.create_workspace(users=[user_2, user])
database = enterprise_data_fixture.create_database_application(workspace=workspace)
source_table = enterprise_data_fixture.create_database_table(
@ -669,7 +666,7 @@ def test_sync_data_sync_table_without_license(enterprise_data_fixture):
source_table_id=source_table.id,
)
License.objects.all().delete()
enterprise_data_fixture.delete_all_licenses()
with pytest.raises(FeaturesNotAvailableError):
handler.sync_data_sync_table(user=user, data_sync=data_sync)

View file

@ -1,6 +1,7 @@
import faker
from baserow_premium.license.models import License
from baserow.core.cache import local_cache
from baserow.core.models import Settings
from baserow_enterprise.models import Role, RoleAssignment, Team, TeamSubject
@ -21,12 +22,20 @@ class EnterpriseFixtures:
def enable_enterprise(self):
Settings.objects.update_or_create(defaults={"instance_id": "1"})
if not License.objects.filter(cached_untrusted_instance_wide=True).exists():
return License.objects.create(
license = License.objects.create(
license=VALID_ONE_SEAT_ENTERPRISE_LICENSE.decode(),
cached_untrusted_instance_wide=True,
)
else:
return License.objects.filter(cached_untrusted_instance_wide=True).get()
license = License.objects.filter(cached_untrusted_instance_wide=True).get()
local_cache.clear()
return license
def delete_all_licenses(self):
License.objects.all().delete()
local_cache.clear()
def create_team(self, **kwargs):
if "name" not in kwargs:

View file

@ -8,6 +8,7 @@ import pytest
from tqdm import tqdm
from baserow.contrib.database.object_scopes import DatabaseObjectScopeType
from baserow.core.cache import local_cache
from baserow.core.handler import CoreHandler
from baserow.core.models import WorkspaceUser
from baserow.core.registries import subject_type_registry
@ -940,6 +941,7 @@ def test_get_roles_per_scope_trashed_teams(data_fixture, enterprise_data_fixture
team.trashed = True
team.save()
local_cache.clear()
assert RoleAssignmentHandler().get_roles_per_scope(workspace, user) == [
(workspace, [admin_role]),

View file

@ -27,6 +27,7 @@ from baserow.contrib.database.table.operations import (
ReadDatabaseTableOperationType,
UpdateDatabaseTableOperationType,
)
from baserow.core.cache import local_cache
from baserow.core.exceptions import PermissionException
from baserow.core.handler import CoreHandler
from baserow.core.models import Application
@ -1808,7 +1809,7 @@ def test_fetching_permissions_does_not_extra_queries_per_snapshot(
# The first time it also fetches the settings and the content types
CoreHandler().get_permissions(viewer, workspace=workspace)
with CaptureQueriesContext(connection) as captured_1:
with CaptureQueriesContext(connection) as captured_1, local_cache.context():
CoreHandler().get_permissions(viewer, workspace=workspace)
# Let's create a snapshot of the database
@ -1816,7 +1817,7 @@ def test_fetching_permissions_does_not_extra_queries_per_snapshot(
snapshot = handler.create(database.id, admin, "Test snapshot")
handler.perform_create(snapshot, Progress(100))
with CaptureQueriesContext(connection) as captured_2:
with CaptureQueriesContext(connection) as captured_2, local_cache.context():
CoreHandler().get_permissions(viewer, workspace=workspace)
assert len(captured_2.captured_queries) == len(captured_1.captured_queries)
@ -1825,7 +1826,7 @@ def test_fetching_permissions_does_not_extra_queries_per_snapshot(
snapshot = handler.create(database.id, admin, "Test snapshot 2")
handler.perform_create(snapshot, Progress(100))
with CaptureQueriesContext(connection) as captured_3:
with CaptureQueriesContext(connection) as captured_3, local_cache.context():
CoreHandler().get_permissions(viewer, workspace=workspace)
assert len(captured_3.captured_queries) == len(captured_2.captured_queries)
@ -1835,7 +1836,7 @@ def test_fetching_permissions_does_not_extra_queries_per_snapshot(
builder = data_fixture.create_builder_application(user=admin, workspace=workspace)
page = data_fixture.create_builder_page(builder=builder)
with CaptureQueriesContext(connection) as captured_1:
with CaptureQueriesContext(connection) as captured_1, local_cache.context():
CoreHandler().get_permissions(viewer, workspace=workspace)
# Let's create a snapshot of the builder app
@ -1843,7 +1844,7 @@ def test_fetching_permissions_does_not_extra_queries_per_snapshot(
snapshot = handler.create(builder.id, admin, "Test snapshot")
handler.perform_create(snapshot, Progress(100))
with CaptureQueriesContext(connection) as captured_2:
with CaptureQueriesContext(connection) as captured_2, local_cache.context():
CoreHandler().get_permissions(viewer, workspace=workspace)
assert len(captured_1.captured_queries) == len(captured_2.captured_queries)

View file

@ -8,9 +8,11 @@ from baserow_premium.license.exceptions import InvalidLicenseError
from baserow_premium.license.models import License
from baserow_premium.license.registries import LicenseType, SeatUsageSummary
from baserow.core.cache import local_cache
from baserow.core.models import Workspace
User = get_user_model()
LICENSE_CACHE_KEY_PREFIX = "license"
class LicensePlugin:
@ -111,12 +113,17 @@ class LicensePlugin:
:param workspace: The workspace to check to see if the user has the feature for.
"""
return any(
feature in license_type.features
for license_type in self.get_active_specific_licenses_only_for_workspace(
def _available_features():
active_licenses = self.get_active_specific_licenses_only_for_workspace(
user, workspace
)
return set().union(*[license.features for license in active_licenses])
available_features = local_cache.get(
f"{LICENSE_CACHE_KEY_PREFIX}_features_{workspace.id}_{user.id}",
_available_features,
)
return feature in available_features
def get_active_instance_wide_license_types(
self, user: Optional[AbstractUser]
@ -149,7 +156,13 @@ class LicensePlugin:
if user_id is not None:
available_license_q |= Q(users__user_id__in=[user_id])
available_licenses = License.objects.filter(available_license_q).distinct()
def _get_available_licenses():
return License.objects.filter(available_license_q).distinct()
available_licenses = local_cache.get(
f"{LICENSE_CACHE_KEY_PREFIX}_{user_id}_instance_wide_licenses",
_get_available_licenses,
)
for available_license in available_licenses:
try:

View file

@ -12,6 +12,7 @@ from baserow_premium.license.registries import LicenseType, license_type_registr
from baserow_premium.plugins import PremiumPlugin
from fakeredis import FakeRedis, FakeServer
from baserow.core.cache import local_cache
from baserow.test_utils.pytest_conftest import * # noqa: F403, F401
@ -80,6 +81,7 @@ class PerWorkspaceLicensePlugin(LicensePlugin):
self.per_workspace_licenses[user.id][workspace_id].add(
license_type_registry.get(license_type)
)
local_cache.clear()
class PremiumPluginWithPerWorkspaceLicensePlugin(PremiumPlugin):

View file

@ -27,6 +27,7 @@ from cryptography.hazmat.primitives.asymmetric import padding
from freezegun import freeze_time
from rest_framework.status import HTTP_200_OK
from baserow.core.cache import local_cache
from baserow.core.exceptions import IsNotAdminError
VALID_ONE_SEAT_LICENSE = (
@ -144,23 +145,23 @@ def test_has_active_premium_license(data_fixture):
license=license, user=second_user_in_license
)
with freeze_time("2021-08-01 12:00"):
with freeze_time("2021-08-01 12:00"), local_cache.context():
assert not has_active_premium_license_features(user_in_license)
assert not has_active_premium_license_features(second_user_in_license)
assert not has_active_premium_license_features(user_not_in_license)
with freeze_time("2021-09-01 12:00"):
with freeze_time("2021-09-01 12:00"), local_cache.context():
assert has_active_premium_license_features(user_in_license)
assert has_active_premium_license_features(second_user_in_license)
assert not has_active_premium_license_features(user_not_in_license)
with freeze_time("2021-10-01 12:00"):
with freeze_time("2021-10-01 12:00"), local_cache.context():
assert not has_active_premium_license_features(user_in_license)
assert not has_active_premium_license_features(second_user_in_license)
assert not has_active_premium_license_features(user_not_in_license)
license_user_2.delete()
with freeze_time("2021-09-01 12:00"):
with freeze_time("2021-09-01 12:00"), local_cache.context():
assert has_active_premium_license_features(user_in_license)
assert not has_active_premium_license_features(second_user_in_license)
assert not has_active_premium_license_features(user_not_in_license)