mirror of
https://gitlab.com/bramw/baserow.git
synced 2025-04-25 05:21:30 +00:00
Merge branch 'add_columns_fill_command' into 'develop'
Add new optional fill_table parameter --add-column which creates a column of... Closes #420 See merge request bramw/baserow!215
This commit is contained in:
commit
0b75d87d29
5 changed files with 190 additions and 38 deletions
118
backend/src/baserow/contrib/database/fields/field_helpers.py
Normal file
118
backend/src/baserow/contrib/database/fields/field_helpers.py
Normal file
|
@ -0,0 +1,118 @@
|
||||||
|
import itertools
|
||||||
|
from typing import Dict, Any, List, Union, Tuple
|
||||||
|
|
||||||
|
from baserow.contrib.database.fields.models import NUMBER_TYPE_CHOICES
|
||||||
|
from baserow.contrib.database.fields.registries import field_type_registry
|
||||||
|
|
||||||
|
FieldKeywordArgPossibilities = Dict[str, Union[List[str], str]]
|
||||||
|
|
||||||
|
|
||||||
|
def map_dict_key_into_values(
|
||||||
|
field_type_kwargs: FieldKeywordArgPossibilities,
|
||||||
|
) -> Dict[str, List[Tuple[str, str]]]:
|
||||||
|
"""
|
||||||
|
Given a dictionary in the form of::
|
||||||
|
|
||||||
|
{
|
||||||
|
'key': 'value',
|
||||||
|
'key2': ['value1', 'value2']
|
||||||
|
}
|
||||||
|
|
||||||
|
this function will convert the value of each key to be a list of tuples of the
|
||||||
|
key and the value returning in the case of the above example::
|
||||||
|
|
||||||
|
{
|
||||||
|
'key': [('key', 'value')],
|
||||||
|
'key2': [('key2', 'value1'), ('key2', 'value2')]
|
||||||
|
}
|
||||||
|
|
||||||
|
"""
|
||||||
|
pairs_dict = {}
|
||||||
|
for name, options in field_type_kwargs.items():
|
||||||
|
pairs_dict[name] = []
|
||||||
|
if not isinstance(options, list):
|
||||||
|
options = [options]
|
||||||
|
for option in options:
|
||||||
|
pairs_dict[name].append((name, option))
|
||||||
|
return pairs_dict
|
||||||
|
|
||||||
|
|
||||||
|
def construct_all_possible_kwargs_for_field(
|
||||||
|
field_type_kwargs: FieldKeywordArgPossibilities,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Given a dictionary containing for each field type a dictionary of that
|
||||||
|
field types kwargs and all of their associated possible values::
|
||||||
|
|
||||||
|
{
|
||||||
|
'field_type_name' : {
|
||||||
|
'a_kwarg_for_this_field_type' : ['optionOne', 'optionTwo'],
|
||||||
|
'other_kwarg_for_this_field_type' : [True, False],
|
||||||
|
'final_kwarg_for_this_field_type' : 'constant'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
This function will create a list of instantiated kwargs per field type, the list
|
||||||
|
will contain every possible combination of the supplied possible values for the
|
||||||
|
fields kwargs. For example given the dict above this would be the result::
|
||||||
|
|
||||||
|
{
|
||||||
|
'field_type_name' : [
|
||||||
|
{
|
||||||
|
'a_kwarg_for_this_field_type' : 'optionOne'
|
||||||
|
'other_kwarg_for_this_field_type' : True,
|
||||||
|
'final_kwarg_for_this_field_type' : 'constant'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'a_kwarg_for_this_field_type' : 'optionTwo'
|
||||||
|
'other_kwarg_for_this_field_type' : True,
|
||||||
|
'final_kwarg_for_this_field_type' : 'constant'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'a_kwarg_for_this_field_type' : 'optionOne'
|
||||||
|
'other_kwarg_for_this_field_type' : False,
|
||||||
|
'final_kwarg_for_this_field_type' : 'constant'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'a_kwarg_for_this_field_type' : 'optionTwo'
|
||||||
|
'other_kwarg_for_this_field_type' : False,
|
||||||
|
'final_kwarg_for_this_field_type' : 'constant'
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
"""
|
||||||
|
dict_of_kwarg_value_pairs = map_dict_key_into_values(field_type_kwargs)
|
||||||
|
all_possible_kwargs = [
|
||||||
|
dict(pairwise_args)
|
||||||
|
for pairwise_args in itertools.product(*dict_of_kwarg_value_pairs.values())
|
||||||
|
]
|
||||||
|
|
||||||
|
return all_possible_kwargs
|
||||||
|
|
||||||
|
|
||||||
|
def construct_all_possible_field_kwargs(link_table) -> Dict[str, List[Dict[str, Any]]]:
|
||||||
|
"""
|
||||||
|
Some baserow field types have multiple different 'modes' which result in
|
||||||
|
different different database columns and modes of operation being
|
||||||
|
created. This function creates a dictionary of field type to a list of
|
||||||
|
kwarg dicts, one for each interesting possible 'subtype' of the field.
|
||||||
|
"""
|
||||||
|
extra_kwargs_for_type = {
|
||||||
|
"date": {
|
||||||
|
"date_include_time": [True, False],
|
||||||
|
},
|
||||||
|
"number": {
|
||||||
|
"number_type": [number_type for number_type, _ in NUMBER_TYPE_CHOICES],
|
||||||
|
"number_negative": [True, False],
|
||||||
|
},
|
||||||
|
"link_row": {"link_row_table": link_table},
|
||||||
|
}
|
||||||
|
|
||||||
|
all_possible_kwargs_per_type = {}
|
||||||
|
for field_type_name in field_type_registry.get_types():
|
||||||
|
extra_kwargs = extra_kwargs_for_type.get(field_type_name, {})
|
||||||
|
all_possible_kwargs = construct_all_possible_kwargs_for_field(extra_kwargs)
|
||||||
|
all_possible_kwargs_per_type[field_type_name] = all_possible_kwargs
|
||||||
|
|
||||||
|
return all_possible_kwargs_per_type
|
|
@ -7,6 +7,10 @@ from django.db.models import Max
|
||||||
|
|
||||||
from faker import Faker
|
from faker import Faker
|
||||||
|
|
||||||
|
from baserow.contrib.database.fields.field_helpers import (
|
||||||
|
construct_all_possible_field_kwargs,
|
||||||
|
)
|
||||||
|
from baserow.contrib.database.fields.handler import FieldHandler
|
||||||
from baserow.contrib.database.table.models import Table
|
from baserow.contrib.database.table.models import Table
|
||||||
from baserow.contrib.database.rows.handler import RowHandler
|
from baserow.contrib.database.rows.handler import RowHandler
|
||||||
|
|
||||||
|
@ -21,6 +25,12 @@ class Command(BaseCommand):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"limit", type=int, help="Amount of rows that need to be " "inserted."
|
"limit", type=int, help="Amount of rows that need to be " "inserted."
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--add-columns",
|
||||||
|
action="store_true",
|
||||||
|
help="Add a column for every field type other than link row to the table "
|
||||||
|
"before populating it.",
|
||||||
|
)
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
table_id = options["table_id"]
|
table_id = options["table_id"]
|
||||||
|
@ -37,6 +47,9 @@ class Command(BaseCommand):
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
if "add_columns" in options and options["add_columns"]:
|
||||||
|
self.create_a_column_for_every_type(table)
|
||||||
|
|
||||||
model = table.get_model()
|
model = table.get_model()
|
||||||
|
|
||||||
# Find out what the highest order is because we want to append the new rows.
|
# Find out what the highest order is because we want to append the new rows.
|
||||||
|
@ -69,3 +82,19 @@ class Command(BaseCommand):
|
||||||
getattr(instance, field_name).set(value)
|
getattr(instance, field_name).set(value)
|
||||||
|
|
||||||
self.stdout.write(self.style.SUCCESS(f"{limit} rows have been inserted."))
|
self.stdout.write(self.style.SUCCESS(f"{limit} rows have been inserted."))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def create_a_column_for_every_type(table):
|
||||||
|
field_handler = FieldHandler()
|
||||||
|
all_kwargs_per_type = construct_all_possible_field_kwargs(None)
|
||||||
|
for field_type_name, all_possible_kwargs in all_kwargs_per_type.items():
|
||||||
|
if field_type_name == "link_row":
|
||||||
|
continue
|
||||||
|
i = 0
|
||||||
|
for kwargs in all_possible_kwargs:
|
||||||
|
i = i + 1
|
||||||
|
postfix = str(i) if len(all_possible_kwargs) > 1 else ""
|
||||||
|
kwargs["name"] = field_type_name + postfix
|
||||||
|
field_handler.create_field(
|
||||||
|
table.database.group.users.first(), table, field_type_name, **kwargs
|
||||||
|
)
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import itertools
|
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
@ -15,6 +14,9 @@ from baserow.contrib.database.fields.exceptions import (
|
||||||
IncompatiblePrimaryFieldTypeError,
|
IncompatiblePrimaryFieldTypeError,
|
||||||
CannotChangeFieldType,
|
CannotChangeFieldType,
|
||||||
)
|
)
|
||||||
|
from baserow.contrib.database.fields.field_helpers import (
|
||||||
|
construct_all_possible_field_kwargs,
|
||||||
|
)
|
||||||
from baserow.contrib.database.fields.field_types import TextFieldType, LongTextFieldType
|
from baserow.contrib.database.fields.field_types import TextFieldType, LongTextFieldType
|
||||||
from baserow.contrib.database.fields.handler import FieldHandler
|
from baserow.contrib.database.fields.handler import FieldHandler
|
||||||
from baserow.contrib.database.fields.models import (
|
from baserow.contrib.database.fields.models import (
|
||||||
|
@ -24,33 +26,12 @@ from baserow.contrib.database.fields.models import (
|
||||||
BooleanField,
|
BooleanField,
|
||||||
SelectOption,
|
SelectOption,
|
||||||
LongTextField,
|
LongTextField,
|
||||||
NUMBER_TYPE_CHOICES,
|
|
||||||
)
|
)
|
||||||
from baserow.contrib.database.fields.registries import field_type_registry
|
from baserow.contrib.database.fields.registries import field_type_registry
|
||||||
from baserow.contrib.database.rows.handler import RowHandler
|
from baserow.contrib.database.rows.handler import RowHandler
|
||||||
from baserow.core.exceptions import UserNotInGroup
|
from baserow.core.exceptions import UserNotInGroup
|
||||||
|
|
||||||
|
|
||||||
def dict_to_pairs(field_type_kwargs):
|
|
||||||
pairs_dict = {}
|
|
||||||
for name, options in field_type_kwargs.items():
|
|
||||||
pairs_dict[name] = []
|
|
||||||
if not isinstance(options, list):
|
|
||||||
options = [options]
|
|
||||||
for option in options:
|
|
||||||
pairs_dict[name].append((name, option))
|
|
||||||
return pairs_dict
|
|
||||||
|
|
||||||
|
|
||||||
def construct_all_possible_kwargs(field_type_kwargs):
|
|
||||||
pairs_dict = dict_to_pairs(field_type_kwargs)
|
|
||||||
args = [
|
|
||||||
dict(pairwise_args) for pairwise_args in itertools.product(*pairs_dict.values())
|
|
||||||
]
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
|
|
||||||
# You must add --runslow to pytest to run this test, you can do this in intellij by
|
# You must add --runslow to pytest to run this test, you can do this in intellij by
|
||||||
# editing the run config for this test and adding --runslow to additional args.
|
# editing the run config for this test and adding --runslow to additional args.
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@ -82,22 +63,7 @@ def test_can_convert_between_all_fields(data_fixture):
|
||||||
# different conversion behaviour or entirely different database columns being
|
# different conversion behaviour or entirely different database columns being
|
||||||
# created. Here the kwargs which control these modes are enumerated so we can then
|
# created. Here the kwargs which control these modes are enumerated so we can then
|
||||||
# generate every possible type of conversion.
|
# generate every possible type of conversion.
|
||||||
extra_kwargs_for_type = {
|
all_possible_kwargs_per_type = construct_all_possible_field_kwargs(link_table)
|
||||||
"date": {
|
|
||||||
"date_include_time": [True, False],
|
|
||||||
},
|
|
||||||
"number": {
|
|
||||||
"number_type": [number_type for number_type, _ in NUMBER_TYPE_CHOICES],
|
|
||||||
"number_negative": [True, False],
|
|
||||||
},
|
|
||||||
"link_row": {"link_row_table": link_table},
|
|
||||||
}
|
|
||||||
|
|
||||||
all_possible_kwargs_per_type = {}
|
|
||||||
for field_type_name in field_type_registry.get_types():
|
|
||||||
extra_kwargs = extra_kwargs_for_type.get(field_type_name, {})
|
|
||||||
all_possible_kwargs = construct_all_possible_kwargs(extra_kwargs)
|
|
||||||
all_possible_kwargs_per_type[field_type_name] = all_possible_kwargs
|
|
||||||
|
|
||||||
i = 1
|
i = 1
|
||||||
for field_type_name, all_possible_kwargs in all_possible_kwargs_per_type.items():
|
for field_type_name, all_possible_kwargs in all_possible_kwargs_per_type.items():
|
||||||
|
|
|
@ -78,3 +78,37 @@ def test_fill_table_no_empty_table(data_fixture, test_limit):
|
||||||
# make sure the first row is still the same
|
# make sure the first row is still the same
|
||||||
assert first_row_value_before == first_row_value_after
|
assert first_row_value_before == first_row_value_after
|
||||||
assert len(results) == test_limit + row_length_before_random_insert
|
assert len(results) == test_limit + row_length_before_random_insert
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_fill_table_with_add_columns(
|
||||||
|
data_fixture,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Verify that the fill_table command correctly creates columns when passing in
|
||||||
|
the --add-columns flag.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# create a new empty table with a field
|
||||||
|
user = data_fixture.create_user()
|
||||||
|
table = data_fixture.create_database_table(user=user)
|
||||||
|
text_field = data_fixture.create_text_field(user=user, table=table)
|
||||||
|
|
||||||
|
model = table.get_model()
|
||||||
|
|
||||||
|
# create data in the previously created field
|
||||||
|
values = {f"field_{text_field.id}": "Some Text"}
|
||||||
|
model.objects.create(**values)
|
||||||
|
|
||||||
|
results = model.objects.all()
|
||||||
|
num_rows_before_fill_type = len(results)
|
||||||
|
num_columns_before_fill_table = table.field_set.count()
|
||||||
|
|
||||||
|
# execute the fill_table command
|
||||||
|
call_command("fill_table", table.id, 10, "--add-columns")
|
||||||
|
|
||||||
|
results = model.objects.all()
|
||||||
|
|
||||||
|
table.refresh_from_db()
|
||||||
|
assert len(results) > num_rows_before_fill_type
|
||||||
|
assert table.field_set.count() > num_columns_before_fill_table
|
||||||
|
|
|
@ -1,5 +1,10 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Unreleased
|
||||||
|
|
||||||
|
* Added `--add-columns` flag to the `fill_table` management command. It creates all the
|
||||||
|
field types before filling the table with random data.
|
||||||
|
|
||||||
## Released (2021-04-08)
|
## Released (2021-04-08)
|
||||||
|
|
||||||
* Added support for importing tables from XML files.
|
* Added support for importing tables from XML files.
|
||||||
|
|
Loading…
Add table
Reference in a new issue