mirror of
https://gitlab.com/bramw/baserow.git
synced 2025-04-10 23:50:12 +00:00
Resolve "Backend fails hard when importing table data with very long column name"
This commit is contained in:
Sascha Jullmann
parent
56e6e5d211
commit
22c2f80f12
15 changed files with 366 additions and 17 deletions
backend
src/baserow/contrib/database
tests/baserow/contrib/database
api
field
table
web-frontend
modules/database
test/unit/database/mixins
|
|
@ -42,6 +42,11 @@ ERROR_INCOMPATIBLE_PRIMARY_FIELD_TYPE = (
|
|||
"The field type {e.field_type} is not compatible with the primary field.",
|
||||
)
|
||||
ERROR_MAX_FIELD_COUNT_EXCEEDED = "ERROR_MAX_FIELD_COUNT_EXCEEDED"
|
||||
ERROR_MAX_FIELD_NAME_LENGTH_EXCEEDED = (
|
||||
"ERROR_MAX_FIELD_NAME_LENGTH_EXCEEDED",
|
||||
HTTP_400_BAD_REQUEST,
|
||||
"You cannot set a field name longer than {e.max_field_name_length} characters.",
|
||||
)
|
||||
ERROR_FIELD_WITH_SAME_NAME_ALREADY_EXISTS = (
|
||||
"ERROR_FIELD_WITH_SAME_NAME_ALREADY_EXISTS",
|
||||
HTTP_400_BAD_REQUEST,
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ from baserow.api.errors import ERROR_USER_NOT_IN_GROUP
|
|||
from baserow.api.schemas import get_error_schema
|
||||
from baserow.contrib.database.api.fields.errors import (
|
||||
ERROR_MAX_FIELD_COUNT_EXCEEDED,
|
||||
ERROR_MAX_FIELD_NAME_LENGTH_EXCEEDED,
|
||||
ERROR_RESERVED_BASEROW_FIELD_NAME,
|
||||
ERROR_INVALID_BASEROW_FIELD_NAME,
|
||||
)
|
||||
from baserow.contrib.database.fields.exceptions import (
|
||||
MaxFieldLimitExceeded,
|
||||
MaxFieldNameLengthExceeded,
|
||||
ReservedBaserowFieldNameException,
|
||||
InvalidBaserowFieldName,
|
||||
)
|
||||
|
|
@ -134,6 +136,7 @@ class TablesView(APIView):
|
|||
InvalidInitialTableData: ERROR_INVALID_INITIAL_TABLE_DATA,
|
||||
InitialTableDataLimitExceeded: ERROR_INITIAL_TABLE_DATA_LIMIT_EXCEEDED,
|
||||
MaxFieldLimitExceeded: ERROR_MAX_FIELD_COUNT_EXCEEDED,
|
||||
MaxFieldNameLengthExceeded: ERROR_MAX_FIELD_NAME_LENGTH_EXCEEDED,
|
||||
InitialTableDataDuplicateName: ERROR_INITIAL_TABLE_DATA_HAS_DUPLICATE_NAMES,
|
||||
ReservedBaserowFieldNameException: ERROR_RESERVED_BASEROW_FIELD_NAME,
|
||||
InvalidBaserowFieldName: ERROR_INVALID_BASEROW_FIELD_NAME,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from baserow.contrib.database.fields.models import Field
|
||||
from baserow.core.exceptions import (
|
||||
InstanceTypeDoesNotExist,
|
||||
InstanceTypeAlreadyRegistered,
|
||||
|
|
@ -49,6 +50,14 @@ class MaxFieldLimitExceeded(Exception):
|
|||
""" Raised when the field count exceeds the limit"""
|
||||
|
||||
|
||||
class MaxFieldNameLengthExceeded(Exception):
|
||||
""" Raised when the field name exceeds the max length."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.max_field_name_length = Field.get_max_name_length()
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class OrderByFieldNotFound(Exception):
|
||||
"""Raised when the field was not found in the table."""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import logging
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
|
|
@ -21,6 +20,7 @@ from .exceptions import (
|
|||
FieldWithSameNameAlreadyExists,
|
||||
ReservedBaserowFieldNameException,
|
||||
InvalidBaserowFieldName,
|
||||
MaxFieldNameLengthExceeded,
|
||||
)
|
||||
from .models import Field, SelectOption
|
||||
from .registries import field_type_registry, field_converter_registry
|
||||
|
|
@ -51,6 +51,7 @@ def _validate_field_name(
|
|||
name key is not in field_values. When False does not return and immediately
|
||||
returns if the key is missing.
|
||||
:raises InvalidBaserowFieldName: If "name" is
|
||||
:raises MaxFieldNameLengthExceeded: When a provided field name is too long.
|
||||
:return:
|
||||
"""
|
||||
if "name" not in field_values:
|
||||
|
|
@ -63,6 +64,10 @@ def _validate_field_name(
|
|||
if existing_field is not None and existing_field.name == name:
|
||||
return
|
||||
|
||||
max_field_name_length = Field.get_max_name_length()
|
||||
if len(name) > max_field_name_length:
|
||||
raise MaxFieldNameLengthExceeded()
|
||||
|
||||
if name.strip() == "":
|
||||
raise InvalidBaserowFieldName()
|
||||
|
||||
|
|
@ -472,6 +477,8 @@ class FieldHandler:
|
|||
Finds a unused field name in the provided table. If no names in the provided
|
||||
field_names_to_try list are available then the last field name in that list will
|
||||
have a number appended which ensures it is an available unique field name.
|
||||
Respects the maximally allowed field name length. In case the field_names_to_try
|
||||
are longer than that, they will get truncated to the maximally allowed length.
|
||||
|
||||
:param table: The table whose fields to search.
|
||||
:param field_names_to_try: The field_names to try in order before starting to
|
||||
|
|
@ -484,6 +491,13 @@ class FieldHandler:
|
|||
if field_ids_to_ignore is None:
|
||||
field_ids_to_ignore = []
|
||||
|
||||
max_field_name_length = Field.get_max_name_length()
|
||||
|
||||
# If the field_name_to_try is longer than the maximally allowed
|
||||
# field name length the name needs to be truncated.
|
||||
field_names_to_try = [
|
||||
item[0:max_field_name_length] for item in field_names_to_try
|
||||
]
|
||||
# Check if any of the names to try are available by finding any existing field
|
||||
# names with the same name.
|
||||
taken_field_names = set(
|
||||
|
|
@ -506,19 +520,34 @@ class FieldHandler:
|
|||
# None of the names in the param list are available, now using the last one lets
|
||||
# append a number to the name until we find a free one.
|
||||
original_field_name = field_names_to_try[-1]
|
||||
# Lookup any existing fields which could potentially collide with our new
|
||||
# field name. This way we can skip these and ensure our new field has a
|
||||
# unique name.
|
||||
|
||||
# Lookup any existing field names. This way we can skip these and ensure our
|
||||
# new field has a unique name.
|
||||
existing_field_name_collisions = set(
|
||||
Field.objects.exclude(id__in=field_ids_to_ignore)
|
||||
.filter(table=table, name__regex=fr"^{re.escape(original_field_name)} \d+$")
|
||||
.filter(table=table)
|
||||
.order_by("name")
|
||||
.distinct()
|
||||
.values_list("name", flat=True)
|
||||
)
|
||||
i = 2
|
||||
while True:
|
||||
field_name = f"{original_field_name} {i}"
|
||||
suffix_to_append = f" {i}"
|
||||
suffix_length = len(suffix_to_append)
|
||||
length_of_original_field_name_plus_suffix = (
|
||||
len(original_field_name) + suffix_length
|
||||
)
|
||||
|
||||
# At this point we know, that the original_field_name can only
|
||||
# be maximally the length of max_field_name_length. Therefore
|
||||
# if the length_of_original_field_name_plus_suffix is longer
|
||||
# we can further truncate the field_name by the length of the
|
||||
# suffix.
|
||||
if length_of_original_field_name_plus_suffix > max_field_name_length:
|
||||
field_name = f"{original_field_name[:-suffix_length]}{suffix_to_append}"
|
||||
else:
|
||||
field_name = f"{original_field_name}{suffix_to_append}"
|
||||
|
||||
i += 1
|
||||
if field_name not in existing_field_name_collisions:
|
||||
return field_name
|
||||
|
|
|
|||
|
|
@ -70,6 +70,10 @@ class Field(
|
|||
queryset = Field.objects.filter(table=table)
|
||||
return cls.get_highest_order_of_queryset(queryset) + 1
|
||||
|
||||
@classmethod
|
||||
def get_max_name_length(cls):
|
||||
return cls._meta.get_field("name").max_length
|
||||
|
||||
@property
|
||||
def db_column(self):
|
||||
return f"field_{self.id}"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from django.db import connection
|
|||
|
||||
from baserow.contrib.database.fields.exceptions import (
|
||||
MaxFieldLimitExceeded,
|
||||
MaxFieldNameLengthExceeded,
|
||||
ReservedBaserowFieldNameException,
|
||||
InvalidBaserowFieldName,
|
||||
)
|
||||
|
|
@ -14,7 +15,7 @@ from baserow.contrib.database.fields.handler import (
|
|||
FieldHandler,
|
||||
RESERVED_BASEROW_FIELD_NAMES,
|
||||
)
|
||||
from baserow.contrib.database.fields.models import TextField
|
||||
from baserow.contrib.database.fields.models import TextField, Field
|
||||
from baserow.contrib.database.views.handler import ViewHandler
|
||||
from baserow.contrib.database.views.view_types import GridViewType
|
||||
from baserow.core.trash.handler import TrashHandler
|
||||
|
|
@ -148,6 +149,7 @@ class TableHandler:
|
|||
:return: A list containing the field names and a list containing all the rows.
|
||||
:rtype: list, list
|
||||
:raises InvalidInitialTableData: When the data doesn't contain a column or row.
|
||||
:raises MaxFieldNameLengthExceeded: When the provided name is too long.
|
||||
"""
|
||||
|
||||
if len(data) == 0:
|
||||
|
|
@ -178,6 +180,12 @@ class TableHandler:
|
|||
if len(field_name_set) != len(fields):
|
||||
raise InitialTableDataDuplicateName()
|
||||
|
||||
max_field_name_length = Field.get_max_name_length()
|
||||
long_field_names = [x for x in field_name_set if len(x) > max_field_name_length]
|
||||
|
||||
if len(long_field_names) > 0:
|
||||
raise MaxFieldNameLengthExceeded()
|
||||
|
||||
if len(field_name_set.intersection(RESERVED_BASEROW_FIELD_NAMES)) > 0:
|
||||
raise ReservedBaserowFieldNameException()
|
||||
|
||||
|
|
|
|||
|
|
@ -244,6 +244,17 @@ def test_create_field(api_client, data_fixture):
|
|||
assert response.status_code == HTTP_400_BAD_REQUEST
|
||||
assert response.json()["error"] == "ERROR_RESERVED_BASEROW_FIELD_NAME"
|
||||
|
||||
# Test creating field with too long name
|
||||
too_long_field_name = "x" * 256
|
||||
response = api_client.post(
|
||||
reverse("api:database:fields:list", kwargs={"table_id": table.id}),
|
||||
{"name": too_long_field_name, "type": "text"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"JWT {jwt_token}",
|
||||
)
|
||||
assert response.status_code == HTTP_400_BAD_REQUEST
|
||||
assert response.json()["error"] == "ERROR_REQUEST_BODY_VALIDATION"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_get_field(api_client, data_fixture):
|
||||
|
|
@ -434,6 +445,17 @@ def test_update_field(api_client, data_fixture):
|
|||
assert response.status_code == HTTP_400_BAD_REQUEST
|
||||
assert response.json()["error"] == "ERROR_FIELD_WITH_SAME_NAME_ALREADY_EXISTS"
|
||||
|
||||
too_long_field_name = "x" * 256
|
||||
url = reverse("api:database:fields:item", kwargs={"field_id": text.id})
|
||||
response = api_client.patch(
|
||||
url,
|
||||
{"name": too_long_field_name},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"JWT" f" {token}",
|
||||
)
|
||||
assert response.status_code == HTTP_400_BAD_REQUEST
|
||||
assert response.json()["error"] == "ERROR_REQUEST_BODY_VALIDATION"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_field(api_client, data_fixture):
|
||||
|
|
|
|||
|
|
@ -152,6 +152,57 @@ def test_create_table_with_data(api_client, data_fixture):
|
|||
assert response_json["error"] == "ERROR_MAX_FIELD_COUNT_EXCEEDED"
|
||||
settings.MAX_FIELD_LIMIT = field_limit
|
||||
|
||||
too_long_field_name = "x" * 256
|
||||
field_name_with_ok_length = "x" * 255
|
||||
|
||||
url = reverse("api:database:tables:list", kwargs={"database_id": database.id})
|
||||
response = api_client.post(
|
||||
url,
|
||||
{
|
||||
"name": "Test 1",
|
||||
"data": [
|
||||
[too_long_field_name, "B", "C", "D"],
|
||||
["1-1", "1-2", "1-3", "1-4", "1-5"],
|
||||
["2-1", "2-2", "2-3"],
|
||||
["3-1", "3-2"],
|
||||
],
|
||||
"first_row_header": True,
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"JWT {token}",
|
||||
)
|
||||
response_json = response.json()
|
||||
assert response.status_code == HTTP_400_BAD_REQUEST
|
||||
assert response_json["error"] == "ERROR_MAX_FIELD_NAME_LENGTH_EXCEEDED"
|
||||
|
||||
url = reverse("api:database:tables:list", kwargs={"database_id": database.id})
|
||||
response = api_client.post(
|
||||
url,
|
||||
{
|
||||
"name": "Test 1",
|
||||
"data": [
|
||||
[field_name_with_ok_length, "B", "C", "D"],
|
||||
["1-1", "1-2", "1-3", "1-4", "1-5"],
|
||||
["2-1", "2-2", "2-3"],
|
||||
["3-1", "3-2"],
|
||||
],
|
||||
"first_row_header": True,
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"JWT {token}",
|
||||
)
|
||||
response_json = response.json()
|
||||
assert response.status_code == HTTP_200_OK
|
||||
|
||||
table = Table.objects.get(id=response_json["id"])
|
||||
|
||||
text_fields = TextField.objects.filter(table=table)
|
||||
assert text_fields[0].name == field_name_with_ok_length
|
||||
assert text_fields[1].name == "B"
|
||||
assert text_fields[2].name == "C"
|
||||
assert text_fields[3].name == "D"
|
||||
assert text_fields[4].name == "Field 5"
|
||||
|
||||
url = reverse("api:database:tables:list", kwargs={"database_id": database.id})
|
||||
response = api_client.post(
|
||||
url,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from faker import Faker
|
|||
|
||||
from baserow.contrib.database.fields.exceptions import (
|
||||
FieldTypeDoesNotExist,
|
||||
MaxFieldNameLengthExceeded,
|
||||
PrimaryFieldAlreadyExists,
|
||||
CannotDeletePrimaryField,
|
||||
FieldDoesNotExist,
|
||||
|
|
@ -279,6 +280,27 @@ def test_create_field(send_mock, data_fixture):
|
|||
with pytest.raises(FieldTypeDoesNotExist):
|
||||
handler.create_field(user=user, table=table, type_name="UNKNOWN")
|
||||
|
||||
too_long_field_name = "x" * 256
|
||||
field_name_with_ok_length = "x" * 255
|
||||
|
||||
with pytest.raises(MaxFieldNameLengthExceeded):
|
||||
handler.create_field(
|
||||
user=user,
|
||||
table=table,
|
||||
type_name="text",
|
||||
name=too_long_field_name,
|
||||
text_default="Some default",
|
||||
)
|
||||
|
||||
field_with_max_length_name = handler.create_field(
|
||||
user=user,
|
||||
table=table,
|
||||
type_name="text",
|
||||
name=field_name_with_ok_length,
|
||||
text_default="Some default",
|
||||
)
|
||||
assert getattr(field_with_max_length_name, "name") == field_name_with_ok_length
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_create_primary_field(data_fixture):
|
||||
|
|
@ -429,6 +451,24 @@ def test_update_field(send_mock, data_fixture):
|
|||
with pytest.raises(FieldWithSameNameAlreadyExists):
|
||||
handler.update_field(user=user, field=field_2, name=field.name)
|
||||
|
||||
too_long_field_name = "x" * 256
|
||||
field_name_with_ok_length = "x" * 255
|
||||
|
||||
field_3 = data_fixture.create_text_field(table=table)
|
||||
with pytest.raises(MaxFieldNameLengthExceeded):
|
||||
handler.update_field(
|
||||
user=user,
|
||||
field=field_3,
|
||||
name=too_long_field_name,
|
||||
)
|
||||
|
||||
field_with_max_length_name = handler.update_field(
|
||||
user=user,
|
||||
field=field_3,
|
||||
name=field_name_with_ok_length,
|
||||
)
|
||||
assert getattr(field_with_max_length_name, "name") == field_name_with_ok_length
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_update_field_failing(data_fixture):
|
||||
|
|
@ -933,3 +973,61 @@ def test_find_next_free_field_name(data_fixture):
|
|||
handler.find_next_unused_field_name(table, ["regex like field [0-9]"])
|
||||
== "regex like field [0-9] 3"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_find_next_free_field_name_returns_strings_with_max_length(data_fixture):
|
||||
user = data_fixture.create_user()
|
||||
table = data_fixture.create_database_table(user=user)
|
||||
max_field_name_length = Field.get_max_name_length()
|
||||
exactly_length_field_name = "x" * max_field_name_length
|
||||
too_long_field_name = "x" * (max_field_name_length + 1)
|
||||
|
||||
data_fixture.create_text_field(name=exactly_length_field_name, table=table, order=1)
|
||||
handler = FieldHandler()
|
||||
|
||||
# Make sure that the returned string does not exceed the max_field_name_length
|
||||
assert (
|
||||
len(handler.find_next_unused_field_name(table, [exactly_length_field_name]))
|
||||
<= max_field_name_length
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
handler.find_next_unused_field_name(
|
||||
table, [f"{exactly_length_field_name} - test"]
|
||||
)
|
||||
)
|
||||
<= max_field_name_length
|
||||
)
|
||||
assert (
|
||||
len(handler.find_next_unused_field_name(table, [too_long_field_name]))
|
||||
<= max_field_name_length
|
||||
)
|
||||
|
||||
initial_name = (
|
||||
"xIyV4w3J4J0Zzd5ZIz4eNPucQOa9tS25ULHw2SCr4RDZ9h2AvxYr5nlGRNQR2ir517B3SkZB"
|
||||
"nw2eGnBJQAdX8A6QcSCmcbBAnG3BczFytJkHJK7cE6VsAS6tROTg7GOwSQsdImURRwEarrXo"
|
||||
"lv9H4bylyJM0bDPkgB4H6apiugZ19X0C9Fw2ed125MJHoFgTZLbJRc6joNyJSOkGkmGhBuIq"
|
||||
"RKipRYGzB4oiFKYPx5Xoc8KHTsLqVDQTWwwzhaR"
|
||||
)
|
||||
expected_name_1 = (
|
||||
"xIyV4w3J4J0Zzd5ZIz4eNPucQOa9tS25ULHw2SCr4RDZ9h2AvxYr5nlGRNQR2ir517B3SkZB"
|
||||
"nw2eGnBJQAdX8A6QcSCmcbBAnG3BczFytJkHJK7cE6VsAS6tROTg7GOwSQsdImURRwEarrXo"
|
||||
"lv9H4bylyJM0bDPkgB4H6apiugZ19X0C9Fw2ed125MJHoFgTZLbJRc6joNyJSOkGkmGhBuIq"
|
||||
"RKipRYGzB4oiFKYPx5Xoc8KHTsLqVDQTWwwzh 2"
|
||||
)
|
||||
|
||||
expected_name_2 = (
|
||||
"xIyV4w3J4J0Zzd5ZIz4eNPucQOa9tS25ULHw2SCr4RDZ9h2AvxYr5nlGRNQR2ir517B3SkZB"
|
||||
"nw2eGnBJQAdX8A6QcSCmcbBAnG3BczFytJkHJK7cE6VsAS6tROTg7GOwSQsdImURRwEarrXo"
|
||||
"lv9H4bylyJM0bDPkgB4H6apiugZ19X0C9Fw2ed125MJHoFgTZLbJRc6joNyJSOkGkmGhBuIq"
|
||||
"RKipRYGzB4oiFKYPx5Xoc8KHTsLqVDQTWwwzh 3"
|
||||
)
|
||||
|
||||
data_fixture.create_text_field(name=initial_name, table=table, order=1)
|
||||
|
||||
assert handler.find_next_unused_field_name(table, [initial_name]) == expected_name_1
|
||||
|
||||
data_fixture.create_text_field(name=expected_name_1, table=table, order=1)
|
||||
|
||||
assert handler.find_next_unused_field_name(table, [initial_name]) == expected_name_2
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ from django.conf import settings
|
|||
from decimal import Decimal
|
||||
|
||||
from baserow.core.exceptions import UserNotInGroup
|
||||
from baserow.contrib.database.fields.exceptions import MaxFieldLimitExceeded
|
||||
from baserow.contrib.database.fields.exceptions import (
|
||||
MaxFieldLimitExceeded,
|
||||
MaxFieldNameLengthExceeded,
|
||||
)
|
||||
from baserow.contrib.database.table.models import Table
|
||||
from baserow.contrib.database.table.handler import TableHandler
|
||||
from baserow.contrib.database.table.exceptions import (
|
||||
|
|
@ -248,6 +251,30 @@ def test_fill_table_with_initial_data(data_fixture):
|
|||
|
||||
settings.MAX_FIELD_LIMIT = field_limit
|
||||
|
||||
too_long_field_name = "x" * 256
|
||||
field_name_with_ok_length = "x" * 255
|
||||
|
||||
data = [
|
||||
[too_long_field_name, "B", "C", "D", "E"],
|
||||
["1-1", "1-2", "1-3", "1-4", "1-5"],
|
||||
]
|
||||
|
||||
with pytest.raises(MaxFieldNameLengthExceeded):
|
||||
table_handler.create_table(
|
||||
user, database, name="Table 3", data=data, first_row_header=True
|
||||
)
|
||||
|
||||
data = [
|
||||
[field_name_with_ok_length, "B", "C", "D", "E"],
|
||||
["1-1", "1-2", "1-3", "1-4", "1-5"],
|
||||
]
|
||||
table = table_handler.create_table(
|
||||
user, database, name="Table 3", data=data, first_row_header=True
|
||||
)
|
||||
num_fields = table.field_set.count()
|
||||
|
||||
assert num_fields == 5
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@patch("baserow.contrib.database.table.signals.table_updated.send")
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
* Fixed error when pasting into a single select field.
|
||||
* Pasting the value of a single select option into a single select field now selects the
|
||||
first option with that value.
|
||||
* The API now returns appropriate errors when trying to create a field with a name which is too long.
|
||||
* Importing table data with a column name that is too long will now truncate that name.
|
||||
|
||||
## Released (2021-08-11)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,12 @@
|
|||
>
|
||||
This field name is not allowed.
|
||||
</div>
|
||||
<div
|
||||
v-else-if="$v.values.name.$dirty && !$v.values.name.maxLength"
|
||||
class="error"
|
||||
>
|
||||
This field name is too long.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="control">
|
||||
|
|
@ -70,11 +76,14 @@
|
|||
</template>
|
||||
|
||||
<script>
|
||||
import { required } from 'vuelidate/lib/validators'
|
||||
import { required, maxLength } from 'vuelidate/lib/validators'
|
||||
|
||||
import form from '@baserow/modules/core/mixins/form'
|
||||
import { mapGetters } from 'vuex'
|
||||
import { RESERVED_BASEROW_FIELD_NAMES } from '@baserow/modules/database/utils/constants'
|
||||
import {
|
||||
RESERVED_BASEROW_FIELD_NAMES,
|
||||
MAX_FIELD_NAME_LENGTH,
|
||||
} from '@baserow/modules/database/utils/constants'
|
||||
|
||||
// @TODO focus form on open
|
||||
export default {
|
||||
|
|
@ -119,6 +128,7 @@ export default {
|
|||
values: {
|
||||
name: {
|
||||
required,
|
||||
maxLength: maxLength(MAX_FIELD_NAME_LENGTH),
|
||||
mustHaveUniqueFieldName: this.mustHaveUniqueFieldName,
|
||||
mustNotClashWithReservedName: this.mustNotClashWithReservedName,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
/**
|
||||
* Mixin that introduces helper methods for the importer form component.
|
||||
*/
|
||||
import { RESERVED_BASEROW_FIELD_NAMES } from '@baserow/modules/database/utils/constants'
|
||||
import {
|
||||
RESERVED_BASEROW_FIELD_NAMES,
|
||||
MAX_FIELD_NAME_LENGTH,
|
||||
} from '@baserow/modules/database/utils/constants'
|
||||
|
||||
export default {
|
||||
methods: {
|
||||
|
|
@ -65,7 +68,8 @@ export default {
|
|||
/**
|
||||
* Find the next un-unused column not present or used yet in the nextFreeIndexMap.
|
||||
* Will append a number to the returned columnName if it is taken, where that
|
||||
* number ensures the returned name is unique. Finally this function will update
|
||||
* number ensures the returned name is unique. Will respect the maximum allowed
|
||||
* field name length. Finally this function will update
|
||||
* the nextFreeIndexMap so future calls will not use any columns returned by
|
||||
* this function.
|
||||
* @param originalColumnName The column name to find the next free unique value for.
|
||||
|
|
@ -78,7 +82,24 @@ export default {
|
|||
findNextFreeName(originalColumnName, nextFreeIndexMap, startingIndex) {
|
||||
let i = nextFreeIndexMap.get(originalColumnName) || startingIndex
|
||||
while (true) {
|
||||
const nextColumnNameToCheck = `${originalColumnName} ${i}`
|
||||
const suffixToAppend = ` ${i}`
|
||||
let nextColumnNameToCheck
|
||||
|
||||
// If appending a number to the columnName in order to make it
|
||||
// unique will return a string that is longer than the maximum
|
||||
// allowed field name length, we need to further slice the
|
||||
// columnName as to not go above the maximum allowed length.
|
||||
if (
|
||||
originalColumnName.length + suffixToAppend.length >
|
||||
MAX_FIELD_NAME_LENGTH
|
||||
) {
|
||||
nextColumnNameToCheck = `${originalColumnName.slice(
|
||||
0,
|
||||
-suffixToAppend.length
|
||||
)}${suffixToAppend}`
|
||||
} else {
|
||||
nextColumnNameToCheck = `${originalColumnName}${suffixToAppend}`
|
||||
}
|
||||
if (!nextFreeIndexMap.has(nextColumnNameToCheck)) {
|
||||
nextFreeIndexMap.set(originalColumnName, i + 1)
|
||||
return nextColumnNameToCheck
|
||||
|
|
@ -112,22 +133,24 @@ export default {
|
|||
}
|
||||
},
|
||||
/**
|
||||
* Ensures that the uploaded field names are unique, non blank and don't use any
|
||||
* reserved Baserow field names.
|
||||
* Ensures that the uploaded field names are unique, non blank, don't exceed
|
||||
* the maximum field name length and don't use any reserved Baserow field names.
|
||||
* @param {*[]} head An array of field names to be checked.
|
||||
* @return A new array of field names which are guaranteed to be unique and valid.
|
||||
*/
|
||||
makeHeaderUniqueAndValid(head) {
|
||||
const nextFreeIndexMap = new Map()
|
||||
for (let i = 0; i < head.length; i++) {
|
||||
nextFreeIndexMap.set(head[i], 0)
|
||||
const truncatedColumn = head[i].trim().slice(0, MAX_FIELD_NAME_LENGTH)
|
||||
nextFreeIndexMap.set(truncatedColumn, 0)
|
||||
}
|
||||
const uniqueAndValidHeader = []
|
||||
for (let i = 0; i < head.length; i++) {
|
||||
const column = head[i]
|
||||
const trimmedColumn = column.trim()
|
||||
const truncatedColumn = trimmedColumn.slice(0, MAX_FIELD_NAME_LENGTH)
|
||||
const uniqueValidName = this.makeColumnNameUniqueAndValidIfNotAlready(
|
||||
trimmedColumn,
|
||||
truncatedColumn,
|
||||
nextFreeIndexMap
|
||||
)
|
||||
uniqueAndValidHeader.push(uniqueValidName)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
export const trueString = ['y', 't', 'o', 'yes', 'true', 'on', '1']
|
||||
// Please keep in sync with src/baserow/contrib/database/fields/handler.py:30
|
||||
export const RESERVED_BASEROW_FIELD_NAMES = ['id', 'order']
|
||||
export const MAX_FIELD_NAME_LENGTH = 255
|
||||
|
|
|
|||
|
|
@ -40,4 +40,61 @@ describe('test file importer', () => {
|
|||
importer.methods.makeHeaderUniqueAndValid(['', 'Field 1', '', ''])
|
||||
).toEqual(['Field 2', 'Field 1', 'Field 3', 'Field 4'])
|
||||
})
|
||||
test('too long names get truncated to max length', () => {
|
||||
const header = ['x'.repeat(255), 'y'.repeat(256), 'z'.repeat(300)]
|
||||
const expectedHeader = ['x'.repeat(255), 'y'.repeat(255), 'z'.repeat(255)]
|
||||
expect(importer.methods.makeHeaderUniqueAndValid(header)).toEqual(
|
||||
expectedHeader
|
||||
)
|
||||
})
|
||||
test('too long names with duplicates get truncated to max length', () => {
|
||||
// header with field names of 260char length
|
||||
const header = [
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxyYZ14T',
|
||||
]
|
||||
|
||||
const expectedHeader = [
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxy',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 2',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 3',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 4',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 5',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 6',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 7',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 8',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 9',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bH 10',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bH 11',
|
||||
]
|
||||
expect(importer.methods.makeHeaderUniqueAndValid(header)).toEqual(
|
||||
expectedHeader
|
||||
)
|
||||
})
|
||||
test('duplicate column names with exactly the allowed maximum name length must be correctly truncated with duplicate values', () => {
|
||||
// header with field names of exactly 255 char length
|
||||
const header = [
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxy',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxy',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxy',
|
||||
]
|
||||
|
||||
const expectedHeader = [
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHgxy',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 2',
|
||||
'bvXP1mSFkVIGfxlJZZ5ERYojgjeOVxV8K7wHuGUrusrpaRJL2gHFyad6GicYmFgFNJlibN8CcxLd1j2kireT6VxIgeN63Rr1G7vPQr9DfmUqDjDTGs8ka8gSpKsoYaUcd1FGEcmNx1B2r3w9SG0K56MmoBZklx2LmDcSJ4PL7y8gSdvYCWNuhDxcjQT3mUIVFyNrIMZ4mTCH98JH9CouOkb0KEgnZ34K8U42HWEZFLQFZ8v6ec9GixED27bHg 3',
|
||||
]
|
||||
expect(importer.methods.makeHeaderUniqueAndValid(header)).toEqual(
|
||||
expectedHeader
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue