Various fixes to speed up develop CI build time

.gitlab-ci.yml

@@ -411,7 +411,7 @@ backend-test-group-1:
     - PYTEST_EXTRA_ARGS=$([[ "$RUN_MORNING_TESTS" = "true" ]] && echo '--run-once-per-day-in-ci' || echo "")
     - |
       docker run \
-        -e PYTEST_SPLITS=4 \
+        -e PYTEST_SPLITS=5 \
         -e PYTEST_SPLIT_GROUP=$PYTEST_SPLIT_GROUP \
         --name=baserow_backend_test_container \
         --network=host \
@@ -445,6 +445,11 @@ backend-test-group-4:
   variables:
     PYTEST_SPLIT_GROUP: 4
 
+backend-test-group-5:
+  extends: backend-test-group-1
+  variables:
+    PYTEST_SPLIT_GROUP: 5
+
 # Collects together all the separate backend coverage databases from previous jobs and
 # combines them to generate a single report for gitlab to use. GitLab itself does not
 # correctly merge these if you just add them all separately into artifacts->reports->
@@ -470,6 +475,8 @@ collect-backend-coverage:
       artifacts: true
     - job: backend-test-group-4
       artifacts: true
+    - job: backend-test-group-5
+      artifacts: true
   script:
     - . /baserow/venv/bin/activate
     # The reports artifacts will be extracted before the script runs into reports by
@@ -496,6 +503,7 @@ build-final-backend-image:
     - job: backend-test-group-2
     - job: backend-test-group-3
     - job: backend-test-group-4
+    - job: backend-test-group-5
     - job: backend-lint
   variables:
     IMAGE_NAME: $BACKEND_IMAGE_NAME

.gitlab/ci_includes/jobs.yml

@@ -111,7 +111,7 @@
       docker context create multi-context
       docker buildx create --use --name multi --driver docker-container --platform linux/amd64 multi-context 
       
-      if [[ -n "$BUILD_ARM" ]]; then
+      if [[ -n "$BUILD_ARM" || "$CI_COMMIT_BRANCH" == "$BUILD_ARM_ON_BRANCH" ]]; then
         which ssh-agent
         eval `ssh-agent -s`
         echo "${ARM_BUILDER_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null
@@ -304,7 +304,7 @@
       docker context create multi-context
       docker buildx create --use --name multi --driver docker-container --platform linux/amd64 multi-context 
       
-      if [[ -n "$BUILD_ARM" ]]; then
+      if [[ -n "$BUILD_ARM" || "$CI_COMMIT_BRANCH" == "$BUILD_ARM_ON_BRANCH" ]]; then
         which ssh-agent
         eval `ssh-agent -s`
         echo "${ARM_BUILDER_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null

backend/src/baserow/test_utils/pytest_conftest.py

@@ -6,6 +6,7 @@ from typing import Dict, Optional
 from unittest.mock import patch
 
 from django.conf import settings as django_settings
+from django.core.files.storage import Storage
 from django.core.management import call_command
 from django.db import DEFAULT_DB_ALIAS, OperationalError, connection
 from django.db.migrations.executor import MigrationExecutor
@@ -541,3 +542,17 @@ def enable_singleton_testing(settings):
         lambda *a, **kw: FakeRedis(server=fake_redis_server),
     ):
         yield
+
+
+@pytest.fixture
+def stubbed_storage(monkeypatch):
+    class StubbedStorage(Storage):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def save(self, name, content, **kwargs):
+            return name
+
+    storage_instance = StubbedStorage()
+    monkeypatch.setattr("django.core.files.storage.default_storage", storage_instance)
+    return storage_instance

backend/tests/baserow/contrib/database/export/test_export_handler.py

@@ -599,6 +599,7 @@ def test_can_export_csv_without_header(storage_mock, data_fixture):
 
 
 @pytest.mark.django_db
+@pytest.mark.once_per_day_in_ci
 @patch("baserow.contrib.database.export.handler.default_storage")
 def test_can_export_csv_with_different_charsets(storage_mock, data_fixture):
     for _, charset in SUPPORTED_EXPORT_CHARSETS:

backend/tests/baserow/contrib/database/migrations/test_remove_field_by_id_migration.py

@@ -4,6 +4,7 @@ import pytest
 
 
 # noinspection PyPep8Naming
+@pytest.mark.once_per_day_in_ci
 def test_forwards_migration(data_fixture, migrator, teardown_table_metadata):
     migrate_from = [
         ("database", "0039_formulafield"),
@@ -76,6 +77,7 @@ def test_forwards_migration(data_fixture, migrator, teardown_table_metadata):
 
 # noinspection PyPep8Naming
 @pytest.mark.run(order=1)
+@pytest.mark.once_per_day_in_ci
 def test_backwards_migration(data_fixture, migrator, teardown_table_metadata):
     migrate_from = [
         ("database", "0040_formulafield_remove_field_by_id"),

backend/tests/baserow/contrib/database/ws/public/conftest.py

@@ -125,7 +125,10 @@ def public_realtime_view_tester(request, data_fixture):
     if request.param == "grid":
         return GridViewPublicWebsocketTester(data_fixture)
     elif request.param == "gallery":
-        return GalleryViewPublicWebsocketTester(data_fixture)
+        if request.node.get_closest_marker("once_per_day_in_ci"):
+            return GalleryViewPublicWebsocketTester(data_fixture)
+        else:
+            pytest.skip("Skipping tests without the once_per_day_in_ci marker.")
     else:
         raise Exception(
             "You must implement ViewTester and add a clause in this text "

backend/tests/baserow/core/management/test_backup_runner.py

@@ -15,6 +15,7 @@ from baserow.core.trash.handler import TrashHandler
 
 
 @pytest.mark.django_db(transaction=True)
+@pytest.mark.once_per_day_in_ci
 def test_can_backup_and_restore_baserow_reverting_changes(data_fixture, environ):
     runner = BaserowBackupRunner(
         host=connection.settings_dict["HOST"],

enterprise/backend/tests/baserow_enterprise_tests/api/audit_log/test_audit_log_workspace_views.py

@@ -202,6 +202,7 @@ def test_workspace_audit_log_can_export_to_csv_filtered_entries(
     enterprise_data_fixture,
     synced_roles,
     django_capture_on_commit_callbacks,
+    stubbed_storage,
 ):
     enterprise_data_fixture.enable_enterprise()
 

enterprise/backend/tests/baserow_enterprise_tests/enterprise/test_enterprise_migrations.py

@@ -1,235 +0,0 @@
-from django.db import connection
-from django.db.migrations.executor import MigrationExecutor
-
-import pytest
-
-from baserow_enterprise.role.constants import NO_ACCESS_ROLE_UID
-
-
-@pytest.fixture(autouse=True)
-def enable_enterprise_and_roles_for_all_tests_here(enable_enterprise, synced_roles):
-    pass
-
-
-def migrate(target):
-    executor = MigrationExecutor(connection)
-    executor.loader.build_graph()  # reload.
-    executor.migrate(target)
-    new_state = executor.loader.project_state(target)
-    return new_state
-
-
-@pytest.mark.django_db
-def test_0010_with_no_role_without_no_access(synced_roles):
-    # Database contains `NO_ROLE`, but not `NO_ACCESS`.
-    migrate_from = [
-        ("baserow_enterprise", "0009_roleassignment_subject_and_scope_uniqueness"),
-    ]
-    migrate_to = [
-        ("baserow_enterprise", "0010_rename_no_role_to_no_access"),
-    ]
-
-    old_state = migrate(migrate_from)
-
-    Role = old_state.apps.get_model("baserow_enterprise", "Role")
-    Role.objects.create(uid="NO_ROLE")
-
-    new_state = migrate(migrate_to)
-
-    Role = new_state.apps.get_model("baserow_enterprise", "Role")
-
-    assert not Role.objects.filter(uid="NO_ROLE").exists()
-    assert Role.objects.filter(uid=NO_ACCESS_ROLE_UID).exists()
-
-
-@pytest.mark.django_db
-def test_0010_with_no_role_and_no_access_only_no_access_assignments(synced_roles):
-    # Database contains `NO_ROLE` and `NO_ACCESS`, but
-    # only `RoleAssignment` pointing to `NO_ACCESS`.
-    migrate_from = [
-        ("core", "0038_group_storage_usage_updated_at"),
-        ("baserow_enterprise", "0009_roleassignment_subject_and_scope_uniqueness"),
-    ]
-    migrate_to = [
-        ("baserow_enterprise", "0010_rename_no_role_to_no_access"),
-    ]
-
-    old_state = migrate(migrate_from)
-
-    # Prior to it being renamed to Workspace.
-    Group = old_state.apps.get_model("core", "Group")
-    Role = old_state.apps.get_model("baserow_enterprise", "Role")
-    Team = old_state.apps.get_model("baserow_enterprise", "Team")
-    ContentType = old_state.apps.get_model("contenttypes", "ContentType")
-    RoleAssignment = old_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    Role.objects.create(uid="NO_ROLE")
-    no_access = Role.objects.get(uid=NO_ACCESS_ROLE_UID)
-
-    group = Group.objects.create(name="Group")
-    team = Team.objects.create(group=group)
-
-    RoleAssignment.objects.create(
-        subject_id=team.id,
-        subject_type=ContentType.objects.get_for_model(team),
-        role=no_access,
-        group=group,
-        scope_id=group.id,
-        scope_type=ContentType.objects.get_for_model(group),
-    )
-
-    new_state = migrate(migrate_to)
-
-    Role = new_state.apps.get_model("baserow_enterprise", "Role")
-    RoleAssignment = new_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    no_access = Role.objects.get(uid=NO_ACCESS_ROLE_UID)
-
-    assert not Role.objects.filter(uid="NO_ROLE").exists()
-    assert Role.objects.filter(uid=NO_ACCESS_ROLE_UID).exists()
-    assert RoleAssignment.objects.count() == 1
-    assert RoleAssignment.objects.filter(role=no_access).count() == 1
-
-
-@pytest.mark.django_db
-def test_0010_with_no_role_and_no_access_only_no_role_assignments(synced_roles):
-    # Database contains `NO_ROLE` and `NO_ACCESS`, but
-    # only `RoleAssignment` pointing to `NO_ROLE`.
-    migrate_from = [
-        ("core", "0038_group_storage_usage_updated_at"),
-        ("baserow_enterprise", "0009_roleassignment_subject_and_scope_uniqueness"),
-    ]
-    migrate_to = [
-        ("baserow_enterprise", "0010_rename_no_role_to_no_access"),
-    ]
-
-    old_state = migrate(migrate_from)
-
-    # Prior to it being renamed to Workspace.
-    Group = old_state.apps.get_model("core", "Group")
-    Role = old_state.apps.get_model("baserow_enterprise", "Role")
-    Team = old_state.apps.get_model("baserow_enterprise", "Team")
-    ContentType = old_state.apps.get_model("contenttypes", "ContentType")
-    RoleAssignment = old_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    no_role = Role.objects.create(uid="NO_ROLE")
-    assert Role.objects.filter(uid=NO_ACCESS_ROLE_UID).exists()
-
-    group = Group.objects.create(name="Group")
-    team = Team.objects.create(group=group)
-
-    RoleAssignment.objects.create(
-        subject_id=team.id,
-        subject_type=ContentType.objects.get_for_model(team),
-        role=no_role,
-        group=group,
-        scope_id=group.id,
-        scope_type=ContentType.objects.get_for_model(group),
-    )
-
-    new_state = migrate(migrate_to)
-
-    Role = new_state.apps.get_model("baserow_enterprise", "Role")
-    RoleAssignment = new_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    no_access = Role.objects.get(uid=NO_ACCESS_ROLE_UID)
-
-    assert not Role.objects.filter(uid="NO_ROLE").exists()
-    assert Role.objects.filter(uid=NO_ACCESS_ROLE_UID).exists()
-    assert RoleAssignment.objects.count() == 1
-    assert RoleAssignment.objects.filter(role=no_access).count() == 1
-
-
-@pytest.mark.django_db
-def test_0010_with_no_role_and_no_access_mixed_role_assignments(synced_roles):
-    # Database contains `NO_ROLE` and `NO_ACCESS`, but
-    # with `RoleAssignment` pointing to `NO_ROLE` and `NO_ACCESS`.
-    migrate_from = [
-        ("core", "0038_group_storage_usage_updated_at"),
-        ("baserow_enterprise", "0009_roleassignment_subject_and_scope_uniqueness"),
-    ]
-    migrate_to = [
-        ("baserow_enterprise", "0010_rename_no_role_to_no_access"),
-    ]
-
-    old_state = migrate(migrate_from)
-
-    # Prior to it being renamed to Workspace.
-    Group = old_state.apps.get_model("core", "Group")
-    Role = old_state.apps.get_model("baserow_enterprise", "Role")
-    Team = old_state.apps.get_model("baserow_enterprise", "Team")
-    ContentType = old_state.apps.get_model("contenttypes", "ContentType")
-    RoleAssignment = old_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    no_role = Role.objects.create(uid="NO_ROLE")
-    no_access = Role.objects.get(uid=NO_ACCESS_ROLE_UID)
-
-    group = Group.objects.create(name="Group")
-    sales = Team.objects.create(group=group, name="Sales")
-    engineering = Team.objects.create(group=group, name="Engineering")
-
-    team_ct = ContentType.objects.get_for_model(Team)
-    group_ct = ContentType.objects.get_for_model(Group)
-
-    RoleAssignment.objects.bulk_create(
-        [
-            RoleAssignment(
-                subject_id=sales.id,
-                subject_type=team_ct,
-                role=no_role,
-                group=group,
-                scope_id=group.id,
-                scope_type=group_ct,
-            ),
-            RoleAssignment(
-                subject_id=engineering.id,
-                subject_type=team_ct,
-                role=no_access,
-                group=group,
-                scope_id=group.id,
-                scope_type=group_ct,
-            ),
-        ]
-    )
-
-    new_state = migrate(migrate_to)
-
-    Role = new_state.apps.get_model("baserow_enterprise", "Role")
-    RoleAssignment = new_state.apps.get_model("baserow_enterprise", "RoleAssignment")
-
-    no_access = Role.objects.get(uid=NO_ACCESS_ROLE_UID)
-
-    assert not Role.objects.filter(uid="NO_ROLE").exists()
-    assert Role.objects.filter(uid=NO_ACCESS_ROLE_UID).exists()
-    assert RoleAssignment.objects.count() == 2
-    assert RoleAssignment.objects.filter(role=no_access).count() == 2
-
-
-@pytest.mark.django_db
-def test_0010_migrates_groupusers(synced_roles):
-    # `GroupUser` pointing to `NO_ROLE` are migrated to `NO_ACCESS`.
-    migrate_from = [
-        ("core", "0042_add_ip_address_to_jobs"),
-        ("baserow_enterprise", "0009_roleassignment_subject_and_scope_uniqueness"),
-    ]
-    migrate_to = [
-        ("core", "0042_add_ip_address_to_jobs"),
-        ("baserow_enterprise", "0010_rename_no_role_to_no_access"),
-    ]
-
-    old_state = migrate(migrate_from)
-
-    User = old_state.apps.get_model("auth", "User")
-    # Prior to it being renamed to Workspace.
-    Group = old_state.apps.get_model("core", "Group")
-    GroupUser = old_state.apps.get_model("core", "GroupUser")
-
-    user = User.objects.create()
-    group = Group.objects.create(name="Group")
-    GroupUser.objects.create(user=user, group=group, permissions="NO_ROLE", order=0)
-
-    new_state = migrate(migrate_to)
-
-    GroupUser = new_state.apps.get_model("core", "GroupUser")
-    assert GroupUser.objects.filter(permissions="NO_ROLE").count() == 0
-    assert GroupUser.objects.filter(permissions=NO_ACCESS_ROLE_UID).count() == 1