From 7ba8d0bf6b6703958ff6dab8d48a9787159ed867 Mon Sep 17 00:00:00 2001 From: Davide Silvestri <davide@baserow.io> Date: Thu, 18 Jan 2024 08:36:56 +0000 Subject: [PATCH] =?UTF-8?q?1=EF=B8=8F=E2=83=A3=20Support=20for=20duration?= =?UTF-8?q?=20in=20formula:=20add=20formats=20with=20days?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/database/fields/field_helpers.py | 3 + .../contrib/database/fields/field_types.py | 78 +-- .../baserow/contrib/database/fields/fields.py | 6 +- .../contrib/database/fields/utils/duration.py | 512 +++++++++++++----- ...149_alter_durationfield_duration_format.py | 31 ++ backend/src/baserow/test_utils/helpers.py | 13 +- .../test_duration_field_serializers.py | 100 +++- .../database/api/rows/test_row_serializers.py | 3 + .../api/views/test_view_serializers.py | 15 +- .../database/export/test_export_handler.py | 7 +- .../field/test_duration_field_type.py | 104 +++- .../local_baserow/test_service_types.py | 21 + ...rmats_with_days_to_the_duration_field.json | 7 + .../export/test_premium_export_types.py | 12 + .../view/ViewFilterTypeDuration.vue | 4 +- web-frontend/modules/database/fieldTypes.js | 26 +- .../modules/database/mixins/durationField.js | 3 +- .../modules/database/utils/duration.js | 252 ++++++--- .../durationFieldTypeGetSort.spec.js | 18 +- .../test/unit/database/utils/duration.spec.js | 507 +++++++++++++++++ .../test/unit/database/utils/row.spec.js | 2 +- 21 files changed, 1355 insertions(+), 369 deletions(-) create mode 100644 backend/src/baserow/contrib/database/migrations/0149_alter_durationfield_duration_format.py rename backend/tests/baserow/contrib/database/{field => api/fields}/test_duration_field_serializers.py (52%) create mode 100644 changelog/entries/unreleased/feature/2217_add_formats_with_days_to_the_duration_field.json create mode 100644 web-frontend/test/unit/database/utils/duration.spec.js diff --git a/backend/src/baserow/contrib/database/fields/field_helpers.py b/backend/src/baserow/contrib/database/fields/field_helpers.py index e59b3a352..56854abb1 100644 --- a/backend/src/baserow/contrib/database/fields/field_helpers.py +++ b/backend/src/baserow/contrib/database/fields/field_helpers.py @@ -142,6 +142,9 @@ def construct_all_possible_field_kwargs( {"name": "duration_hms_s", "duration_format": "h:mm:ss.s"}, {"name": "duration_hms_ss", "duration_format": "h:mm:ss.ss"}, {"name": "duration_hms_sss", "duration_format": "h:mm:ss.sss"}, + {"name": "duration_dh", "duration_format": "d h"}, + {"name": "duration_dhm", "duration_format": "d h:mm"}, + {"name": "duration_dhms", "duration_format": "d h:mm:ss"}, ], "link_row": [ {"name": "link_row", "link_row_table": link_table}, diff --git a/backend/src/baserow/contrib/database/fields/field_types.py b/backend/src/baserow/contrib/database/fields/field_types.py index da7dc7d15..2cd4c36be 100755 --- a/backend/src/baserow/contrib/database/fields/field_types.py +++ b/backend/src/baserow/contrib/database/fields/field_types.py @@ -190,9 +190,12 @@ from .registries import ( field_type_registry, ) from .utils.duration import ( - DURATION_FORMAT_TOKENS, DURATION_FORMATS, - convert_duration_input_value_to_timedelta, + duration_value_sql_to_text, + duration_value_to_timedelta, + format_duration_value, + get_duration_search_expression, + is_duration_format_conversion_lossy, prepare_duration_value_for_db, ) @@ -1716,32 +1719,19 @@ class DurationFieldType(FieldType): return prepare_duration_value_for_db(value, instance.duration_format) def get_search_expression(self, field: Field, queryset: QuerySet) -> Expression: - search_exprs = [] - for token in field.duration_format.split(":"): - search_expr = DURATION_FORMAT_TOKENS[token]["search_expr"](field.db_column) - search_exprs.append(search_expr) - separators = [Value(" ")] * len(search_exprs) - # interleave a separator between each extract_expr - exprs = [expr for pair in zip(search_exprs, separators) for expr in pair][:-1] - return Func(*exprs, function="CONCAT") + return get_duration_search_expression(field) def random_value(self, instance, fake, cache): - random_seconds = fake.random.random() * 60 * 60 * 2 - return convert_duration_input_value_to_timedelta( - random_seconds, instance.duration_format - ) + random_seconds = fake.random.random() * 60 * 60 * 24 + # if we have days in the format, ensure the random value is picked accordingly + if "d" in instance.duration_format: + random_seconds *= 30 + return duration_value_to_timedelta(random_seconds, instance.duration_format) def get_alter_column_prepare_old_value(self, connection, from_field, to_field): to_field_type = field_type_registry.get_by_model(to_field) if to_field_type.type in (TextFieldType.type, LongTextFieldType.type): - format_func = " || ':' || ".join( - [ - DURATION_FORMAT_TOKENS[format_token]["sql_to_text"] - for format_token in from_field.duration_format.split(":") - ] - ) - - return f"p_in = {format_func};" + return f"p_in = {duration_value_sql_to_text(from_field)};" elif to_field_type.type == NumberFieldType.type: return "p_in = EXTRACT(EPOCH FROM CAST(p_in AS INTERVAL))::NUMERIC;" @@ -1768,19 +1758,7 @@ class DurationFieldType(FieldType): :return: The formatted string. """ - if value is None: - return None - - secs_in_a_min = 60 - secs_in_an_hour = 60 * 60 - - total_seconds = value.total_seconds() - hours = int(total_seconds / secs_in_an_hour) - minutes = int(total_seconds % secs_in_an_hour / secs_in_a_min) - seconds = total_seconds % secs_in_a_min - - format_func = DURATION_FORMATS[duration_format]["format_func"] - return format_func(hours, minutes, seconds) + return format_duration_value(value, duration_format) def get_export_value( self, @@ -1788,22 +1766,9 @@ class DurationFieldType(FieldType): field_object: "FieldObject", rich_value: bool = False, ) -> Optional[str]: - if value is None: - return None - - secs_in_a_min = 60 - secs_in_an_hour = 60 * 60 - - total_seconds = value.total_seconds() - - hours = int(total_seconds / secs_in_an_hour) - mins = int(total_seconds % secs_in_an_hour / secs_in_a_min) - secs = total_seconds % secs_in_a_min - field = field_object["field"] duration_format = field.duration_format - format_func = DURATION_FORMATS[duration_format]["format_func"] - return format_func(hours, mins, secs) + return self.format_duration(value, duration_format) def should_backup_field_data_for_same_type_update( self, old_field: DurationField, new_field_attrs: Dict[str, Any] @@ -1812,17 +1777,14 @@ class DurationFieldType(FieldType): "duration_format", old_field.duration_format ) - formats_needing_a_backup = DURATION_FORMATS[old_field.duration_format][ - "backup_field_if_changing_to" - ] - return new_duration_format in formats_needing_a_backup + return is_duration_format_conversion_lossy( + new_duration_format, old_field.duration_format + ) def force_same_type_alter_column(self, from_field, to_field): - curr_format = from_field.duration_format - formats_needing_alter_column = DURATION_FORMATS[curr_format][ - "backup_field_if_changing_to" - ] - return to_field.duration_format in formats_needing_alter_column + return is_duration_format_conversion_lossy( + to_field.duration_format, from_field.duration_format + ) def serialize_metadata_for_row_history( self, diff --git a/backend/src/baserow/contrib/database/fields/fields.py b/backend/src/baserow/contrib/database/fields/fields.py index 4e48a24bf..290f356f9 100644 --- a/backend/src/baserow/contrib/database/fields/fields.py +++ b/backend/src/baserow/contrib/database/fields/fields.py @@ -9,9 +9,7 @@ from django.db.models.fields.related_descriptors import ( ) from django.utils.functional import cached_property -from baserow.contrib.database.fields.utils.duration import ( - convert_duration_input_value_to_timedelta, -) +from baserow.contrib.database.fields.utils.duration import duration_value_to_timedelta from baserow.contrib.database.formula import BaserowExpression, FormulaHandler from baserow.core.fields import SyncedDateTimeField @@ -318,7 +316,7 @@ class DurationField(models.DurationField): super().__init__(*args, **kwargs) def get_prep_value(self, value: Any) -> Any: - value = convert_duration_input_value_to_timedelta(value, self.duration_format) + value = duration_value_to_timedelta(value, self.duration_format) return super().get_prep_value(value) diff --git a/backend/src/baserow/contrib/database/fields/utils/duration.py b/backend/src/baserow/contrib/database/fields/utils/duration.py index 1a02e87e6..6e1b2fa8b 100644 --- a/backend/src/baserow/contrib/database/fields/utils/duration.py +++ b/backend/src/baserow/contrib/database/fields/utils/duration.py @@ -1,5 +1,6 @@ +import re from datetime import timedelta -from typing import Optional, Union +from typing import List, Optional, Union from django.core.exceptions import ValidationError from django.db.models import ( @@ -10,144 +11,290 @@ from django.db.models import ( TextField, Value, ) -from django.db.models.functions import Cast, Extract +from django.db.models.functions import Cast, Extract, Mod + +H_M = "h:mm" +H_M_S = "h:mm:ss" +H_M_S_S = "h:mm:ss.s" +H_M_S_SS = "h:mm:ss.ss" +H_M_S_SSS = "h:mm:ss.sss" +D_H = "d h" +D_H_M = "d h:mm" +D_H_M_S = "d h:mm:ss" + +MOST_ACCURATE_DURATION_FORMAT = H_M_S_SSS + + +def total_secs(days=0, hours=0, mins=0, secs=0): + return int(days) * 86400 + int(hours) * 3600 + int(mins) * 60 + float(secs) + + +# These regexps are supposed to tokenize the provided duration value and to return a +# proper number of seconds based on format and the tokens. NOTE: Keep these in sync with +# web-frontend/modules/database/utils/duration.js:DURATION_REGEXPS +DURATION_REGEXPS = { + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+):(\d+):(\d+|\d+.\d+)$"): { + "default": lambda d, h, m, s: total_secs(days=d, hours=h, mins=m, secs=s), + }, + re.compile(r"^(\d+):(\d+):(\d+|\d+.\d+)$"): { + "default": lambda h, m, s: total_secs(hours=h, mins=m, secs=s), + }, + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+)h$"): { + "default": lambda d, h: total_secs(days=d, hours=h), + }, + re.compile(r"^(\d+)h$"): { + "default": lambda h: total_secs(hours=h), + }, + re.compile(r"^(\d+)d$"): { + "default": lambda d: total_secs(days=d), + }, + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+):(\d+)$"): { + H_M: lambda d, h, m: total_secs(days=d, hours=h, mins=m), + D_H: lambda d, h, m: total_secs(days=d, hours=h, mins=m), + D_H_M: lambda d, h, m: total_secs(days=d, hours=h, mins=m), + "default": lambda d, m, s: total_secs(days=d, mins=m, secs=s), + }, + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+):(\d+.\d+)$"): { + "default": lambda d, m, s: total_secs(days=d, mins=m, secs=s), + }, + re.compile(r"^(\d+):(\d+)$"): { + H_M: lambda h, m: total_secs(hours=h, mins=m), + D_H: lambda h, m: total_secs(hours=h, mins=m), + D_H_M: lambda h, m: total_secs(hours=h, mins=m), + "default": lambda m, s: total_secs(mins=m, secs=s), + }, + re.compile(r"^(\d+):(\d+.\d+)$"): { + "default": lambda m, s: total_secs(mins=m, secs=s), + }, + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+)$"): { + H_M: lambda d, m: total_secs(days=d, mins=m), + D_H: lambda d, h: total_secs(days=d, hours=h), + D_H_M: lambda d, m: total_secs(days=d, mins=m), + "default": lambda d, s: total_secs(days=d, secs=s), + }, + re.compile(r"^(\d+)(?:d\s*|\s+)(\d+.\d+)$"): { + "default": lambda d, s: total_secs(days=d, secs=s), + }, + re.compile(r"^(\d+)$"): { + H_M: lambda m: total_secs(mins=float(m)), + D_H: lambda h: total_secs(hours=float(h)), + D_H_M: lambda m: total_secs(mins=float(m)), + "default": lambda s: total_secs(secs=s), + }, + re.compile(r"^(\d+.\d+)$"): { + "default": lambda s: total_secs(secs=s), + }, +} + + +def rround(value: float, ndigits: int = 0) -> int: + """ + Rounds a float to the specified number of decimal places. Python round will round + 0.5 to 0, but we want it to round to 1. See + https://docs.python.org/3/library/functions.html#round for more info. + + :param value: the value to round + :param ndigits: the number of digits to round to + """ + + digit_value = 10**ndigits + return int(value * digit_value + 0.5) / digit_value + DURATION_FORMATS = { - "h:mm": { + H_M: { "name": "hours:minutes", - "backup_field_if_changing_to": set(), - "round_func": lambda value: round(value / 60, 0) * 60, + "round_func": lambda value: rround(value / 60) * 60, "sql_round_func": "(EXTRACT(EPOCH FROM p_in::INTERVAL) / 60)::int * 60", - "format_func": lambda hours, mins, _: "%d:%02d" % (hours, mins), + "format_func": lambda d, h, m, s: "%d:%02d" % (d * 24 + h, m), }, - "h:mm:ss": { + H_M_S: { "name": "hours:minutes:seconds", - "backup_field_if_changing_to": {"h:mm"}, - "round_func": lambda value: round(value, 0), + "round_func": lambda value: rround(value, 0), "sql_round_func": "EXTRACT(EPOCH FROM p_in::INTERVAL)::int", - "format_func": lambda hours, mins, secs: "%d:%02d:%02d" % (hours, mins, secs), + "format_func": lambda d, h, m, s: "%d:%02d:%02d" % (d * 24 + h, m, s), }, - "h:mm:ss.s": { + H_M_S_S: { "name": "hours:minutes:seconds:deciseconds", - "round_func": lambda value: round(value, 1), + "round_func": lambda value: rround(value, 1), "sql_round_func": "ROUND(EXTRACT(EPOCH FROM p_in::INTERVAL)::NUMERIC, 1)", - "backup_field_if_changing_to": {"h:mm", "h:mm:ss"}, - "format_func": lambda hours, mins, secs: "%d:%02d:%04.1f" % (hours, mins, secs), + "format_func": lambda d, h, m, s: "%d:%02d:%04.1f" % (d * 24 + h, m, s), }, - "h:mm:ss.ss": { + H_M_S_SS: { "name": "hours:minutes:seconds:centiseconds", - "backup_field_if_changing_to": {"h:mm", "h:mm:ss", "h:mm:ss.s"}, - "round_func": lambda value: round(value, 2), + "round_func": lambda value: rround(value, 2), "sql_round_func": "ROUND(EXTRACT(EPOCH FROM p_in::INTERVAL)::NUMERIC, 2)", - "format_func": lambda hours, mins, secs: "%d:%02d:%05.2f" % (hours, mins, secs), + "format_func": lambda d, h, m, s: "%d:%02d:%05.2f" % (d * 24 + h, m, s), }, - "h:mm:ss.sss": { + H_M_S_SSS: { "name": "hours:minutes:seconds:milliseconds", - "backup_field_if_changing_to": { - "h:mm", - "h:mm:ss", - "h:mm:ss.s", - "h:mm:ss.ss", - }, - "round_func": lambda value: round(value, 3), + "round_func": lambda value: rround(value, 3), "sql_round_func": "ROUND(EXTRACT(EPOCH FROM p_in::INTERVAL)::NUMERIC, 3)", - "format_func": lambda hours, mins, secs: "%d:%02d:%06.3f" % (hours, mins, secs), + "format_func": lambda d, h, m, s: "%d:%02d:%06.3f" % (d * 24 + h, m, s), + }, + D_H: { + "name": "days:hours", + "round_func": lambda value: rround(value / 3600) * 3600, + "sql_round_func": "(EXTRACT(EPOCH FROM p_in::INTERVAL) / 3600)::int * 3600", + "format_func": lambda d, h, m, s: "%dd %dh" % (d, h), + }, + D_H_M: { + "name": "days:hours:minutes", + "round_func": lambda value: rround(value / 60) * 60, + "sql_round_func": "(EXTRACT(EPOCH FROM p_in::INTERVAL) / 60)::int * 60", + "format_func": lambda d, h, m, s: "%dd %d:%02d" % (d, h, m), + }, + D_H_M_S: { + "name": "days:hours:minutes:seconds", + "round_func": lambda value: rround(value, 0), + "sql_round_func": "EXTRACT(EPOCH FROM p_in::INTERVAL)::int", + "format_func": lambda d, h, m, s: "%dd %d:%02d:%02d" % (d, h, m, s), }, } -# The tokens used in the format strings and their utility functions. +HOURS_WITH_DAYS_SQL_TO_TEXT = ( + "(EXTRACT(HOUR FROM CAST(p_in AS INTERVAL))::INTEGER %% 24)" +) + + +def hours_with_days_search_expr(field_name): + return Mod( + Cast(Extract(field_name, "hour"), output_field=IntegerField()), Value(24) + ) + + +# The tokens used in the format strings and their utility functions. NOTE: the order of +# duration units is crucial, as it ranges from the largest to the smallest unit. This +# order is significant because it helps determine the precision of different duration +# formats (see is_duration_format_conversion_lossy down below), which are created by +# combining various tokens. If there's a need to convert between these formats, it might +# be necessary to backup data beforehand to prevent loss of precision. DURATION_FORMAT_TOKENS = { - "h": { - "multiplier": 3600, - "parse_func": int, - "sql_to_text": "TRUNC(EXTRACT(EPOCH FROM CAST(p_in AS INTERVAL))::INTEGER / 3600)", - "search_expr": lambda field_name: Cast( - Func( - Extract(field_name, "epoch", output_field=IntegerField()) / Value(3600), - function="TRUNC", - output_field=IntegerField(), + "d": { + "sql_to_text": { + "default": "CASE WHEN p_in IS null THEN null ELSE CONCAT(TRUNC(EXTRACT(EPOCH FROM CAST(p_in AS INTERVAL))::INTEGER / 86400), 'd') END", + }, + "search_expr": { + "default": lambda field_name: Func( + Cast( + Func( + Extract(field_name, "epoch", output_field=IntegerField()) + / Value(86400), + function="TRUNC", + output_field=IntegerField(), + ), + output_field=TextField(), + ), + Value("d"), + function="CONCAT", ), - output_field=TextField(), - ), + }, + }, + "h": { + "sql_to_text": { + D_H: f"CASE WHEN p_in IS null THEN null ELSE CONCAT({HOURS_WITH_DAYS_SQL_TO_TEXT}, 'h') END", + D_H_M: HOURS_WITH_DAYS_SQL_TO_TEXT, + D_H_M_S: HOURS_WITH_DAYS_SQL_TO_TEXT, + "default": "TRUNC(EXTRACT(EPOCH FROM CAST(p_in AS INTERVAL))::INTEGER / 3600)", + }, + "search_expr": { + D_H: lambda field_name: Func( + hours_with_days_search_expr(field_name), Value("h"), function="CONCAT" + ), + D_H_M: hours_with_days_search_expr, + D_H_M_S: hours_with_days_search_expr, + "default": lambda field_name: Cast( + Func( + Extract(field_name, "epoch", output_field=IntegerField()) + / Value(3600), + function="TRUNC", + output_field=IntegerField(), + ), + output_field=TextField(), + ), + }, }, "mm": { - "multiplier": 60, - "parse_func": int, - "sql_to_text": "TO_CHAR(EXTRACT(MINUTE FROM CAST(p_in AS INTERVAL))::INTEGER, 'FM00')", - "search_expr": lambda field_name: Func( - Extract(field_name, "minutes", output_field=IntegerField()), - Value("FM00"), - function="TO_CHAR", - output_field=TextField(), - ), + "sql_to_text": { + "default": "TO_CHAR(EXTRACT(MINUTE FROM CAST(p_in AS INTERVAL))::INTEGER, 'FM00')", + }, + "search_expr": { + "default": lambda field_name: Func( + Extract(field_name, "minutes", output_field=IntegerField()), + Value("FM00"), + function="TO_CHAR", + output_field=TextField(), + ), + }, }, "ss": { - "multiplier": 1, - "parse_func": lambda value: round(float(value), 0), - "sql_to_text": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 0)), 'FM00')", - "search_expr": lambda field_name: Func( - Cast( - Extract(field_name, "seconds", output_field=FloatField()), - output_field=DecimalField(max_digits=15, decimal_places=0), + "sql_to_text": { + "default": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 0)), 'FM00')", + }, + "search_expr": { + "default": lambda field_name: Func( + Cast( + Extract(field_name, "seconds", output_field=FloatField()), + output_field=DecimalField(max_digits=15, decimal_places=0), + ), + Value("FM00"), + function="TO_CHAR", + output_field=TextField(), ), - Value("FM00"), - function="TO_CHAR", - output_field=TextField(), - ), + }, }, "ss.s": { - "multiplier": 1, - "parse_func": lambda value: round(float(value), 1), - "sql_to_text": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 1)), 'FM00.0')", - "search_expr": lambda field_name: Func( - Cast( - Extract(field_name, "seconds", output_field=DecimalField()), - output_field=DecimalField(max_digits=15, decimal_places=1), - ), - Value("FM00.0"), - function="TO_CHAR", - output_field=TextField(), - ), + "sql_to_text": { + "default": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 1)), 'FM00.0')" + }, + "search_expr": { + "default": lambda field_name: Func( + Cast( + Extract(field_name, "seconds", output_field=DecimalField()), + output_field=DecimalField(max_digits=15, decimal_places=1), + ), + Value("FM00.0"), + function="TO_CHAR", + output_field=TextField(), + ) + }, }, "ss.ss": { - "multiplier": 1, - "parse_func": lambda value: round(float(value), 2), - "sql_to_text": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 2)), 'FM00.00')", - "search_expr": lambda field_name: Func( - Cast( - Extract(field_name, "seconds", output_field=DecimalField()), - output_field=DecimalField(max_digits=15, decimal_places=2), - ), - Value("FM00.00"), - function="TO_CHAR", - output_field=TextField(), - ), + "sql_to_text": { + "default": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 2)), 'FM00.00')" + }, + "search_expr": { + "default": lambda field_name: Func( + Cast( + Extract(field_name, "seconds", output_field=DecimalField()), + output_field=DecimalField(max_digits=15, decimal_places=2), + ), + Value("FM00.00"), + function="TO_CHAR", + output_field=TextField(), + ) + }, }, "ss.sss": { - "multiplier": 1, - "parse_func": lambda value: round(float(value), 3), - "sql_to_text": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 3)), 'FM00.000')", - "search_expr": lambda field_name: Func( - Cast( - Extract(field_name, "seconds", output_field=DecimalField()), - output_field=DecimalField(max_digits=15, decimal_places=3), - ), - Value("FM00.000"), - function="TO_CHAR", - output_field=TextField(), - ), + "sql_to_text": { + "default": "TO_CHAR(CAST(EXTRACT(SECOND FROM CAST(p_in AS INTERVAL)) AS NUMERIC(15, 3)), 'FM00.000')" + }, + "search_expr": { + "default": lambda field_name: Func( + Cast( + Extract(field_name, "seconds", output_field=DecimalField()), + output_field=DecimalField(max_digits=15, decimal_places=3), + ), + Value("FM00.000"), + function="TO_CHAR", + output_field=TextField(), + ) + }, }, } -MOST_ACCURATE_DURATION_FORMAT = "h:mm:ss.sss" -MAX_NUMBER_OF_TOKENS_IN_DURATION_FORMAT = len(MOST_ACCURATE_DURATION_FORMAT.split(":")) -# Keep this function in sync with the one in the frontend: -# web-frontend/modules/database/utils/duration.js -> -# guessDurationValueFromString -def parse_formatted_duration( - formatted_value: str, format: str, strict: bool = False -) -> float: +def parse_duration_value(formatted_value: str, format: str) -> float: """ Parses a formatted duration string into a number of seconds according to the provided format. If the format doesn't match exactly, it will still try to @@ -155,46 +302,24 @@ def parse_formatted_duration( :param formatted_value: The formatted duration string. :param format: The format of the duration string. - :param strict: If True, the formatted value must match the format exactly or - a ValueError will be raised. If False, the formatted value can be any - acceptable format and will be parsed as best as possible. - :return: The number of seconds. + :return: The total number of seconds for the given formatted duration. :raises ValueError: If the format is invalid. """ if format not in DURATION_FORMATS: raise ValueError(f"{format} is not a valid duration format.") - tokens = format.split(":") - parts = formatted_value.split(":") - if len(parts) > len(tokens): - if strict or len(parts) > MAX_NUMBER_OF_TOKENS_IN_DURATION_FORMAT: - raise ValueError( - f"Too many parts in formatted value {formatted_value} for format {format}." - ) - else: - # In this case we still want to parse the value as best as possible, - # so use the most accurate format and round the value later. - tokens = MOST_ACCURATE_DURATION_FORMAT.split(":") + for regex, format_funcs in DURATION_REGEXPS.items(): + match = regex.match(formatted_value) + if match: + format_func = format_funcs.get(format, format_funcs["default"]) + return format_func(*match.groups()) - total_seconds = 0 - for i, token in enumerate(reversed(tokens)): - if len(parts) <= i: - break - # pick the corresponding part from the end of the list - part = parts[-(i + 1)] - - token_options = DURATION_FORMAT_TOKENS[token] - multiplier = token_options["multiplier"] - parse_func = token_options["parse_func"] - - total_seconds += parse_func(part) * multiplier - - total_seconds = DURATION_FORMATS[format]["round_func"](total_seconds) - return total_seconds + # None of the regexps matches the formatted value + raise ValueError(f"{formatted_value} is not a valid duration string.") -def convert_duration_input_value_to_timedelta( +def duration_value_to_timedelta( value: Union[int, float, timedelta, str, None], format: str ) -> Optional[timedelta]: """ @@ -207,8 +332,7 @@ def convert_duration_input_value_to_timedelta( :param format: The format to use when parsing the value if it is a string and to round the value to. :return: The timedelta object. - :raises ValueError: If the value is not a valid integer or string according - to the format. + :raises ValueError: If the value has an invalid type. :raise OverflowError: If the value is too big to be converted to a timedelta. """ @@ -220,10 +344,9 @@ def convert_duration_input_value_to_timedelta( elif isinstance(value, timedelta): return value - # Since our view_filters are storing the number of seconds as string, let's try to - # convert it to a float first. Please note that this is different in the frontend - # where the input value is a string and immediately use the field format to parse - # it. + # Since our view_filters are storing the number of seconds as string, let's try + # to convert it to a number first. Please note that this is different in the + # frontend where the input value is parsed accordingly to the field format. try: value = float(value) except ValueError: @@ -232,8 +355,7 @@ def convert_duration_input_value_to_timedelta( if isinstance(value, (int, float)) and value >= 0: total_seconds = value elif isinstance(value, str): - formatted_duration = value - total_seconds = parse_formatted_duration(formatted_duration, format) + total_seconds = parse_duration_value(value, format) else: raise ValueError( "The provided value should be a valid integer or string according to the " @@ -262,7 +384,7 @@ def prepare_duration_value_for_db( """ try: - return convert_duration_input_value_to_timedelta(value, duration_format) + return duration_value_to_timedelta(value, duration_format) except ValueError: raise default_exc( f"The value {value} is not a valid duration.", @@ -273,3 +395,107 @@ def prepare_duration_value_for_db( f"Value {value} is too large. The maximum is {timedelta.max}.", code="invalid", ) + + +def format_duration_value( + duration: Optional[timedelta], duration_format +) -> Optional[str]: + """ + Format a duration value according to the provided format. + + :param duration: The duration to format. + :param duration_format: The format to use. + :return: The formatted duration. + """ + + if duration is None: + return None + + days = duration.days + hours = duration.seconds // 3600 + mins = duration.seconds % 3600 // 60 + secs = duration.seconds % 60 + duration.microseconds / 10**6 + + format_func = DURATION_FORMATS[duration_format]["format_func"] + return format_func(days, hours, mins, secs) + + +def tokenize_formatted_duration(duration_format: str) -> List[str]: + """ + Tokenize a formatted duration format returning a list of tokens. + + :param formatted_value: The formatted duration string. + :return: A list with tokens. + """ + + return re.split("[: ]", duration_format) + + +def is_duration_format_conversion_lossy(new_format, old_format): + """ + Returns True if converting from starting_format to ending_format will result in a + loss of precision. + + :param starting_format: The format to convert from. + :param ending_format: The format to convert to. + """ + + ordered_tokens = list(DURATION_FORMAT_TOKENS.keys()) + + new_lst = tokenize_formatted_duration(new_format)[-1] + old_lst = tokenize_formatted_duration(old_format)[-1] + + return ordered_tokens.index(old_lst) > ordered_tokens.index(new_lst) + + +def get_duration_search_expression(field) -> Func: + """ + Returns a search expression that can be used to search the field as a formatted + string. + + :param field: The field to get the search expression for. + :return: A search expression that can be used to search the field as a formatted + string. + """ + + search_exprs = [] + for token in tokenize_formatted_duration(field.duration_format): + search_exp_functions = DURATION_FORMAT_TOKENS[token]["search_expr"] + search_expr_func = search_exp_functions.get( + field.duration_format, search_exp_functions["default"] + ) + search_exprs.append(search_expr_func(field.db_column)) + separators = [Value(" ")] * len(search_exprs) + # interleave a separator between each extract_expr + exprs = [expr for pair in zip(search_exprs, separators) for expr in pair][:-1] + return Func(*exprs, function="CONCAT") + + +def duration_value_sql_to_text(field) -> str: + """ + Returns a SQL expression that can be used to convert the duration value to a + formatted string. + + :param field: The field to get the SQL expression for. + :return: A string containing the SQL expression that can be used to convert the + duration value to a formatted string. + """ + + format_func = "" + tokens = tokenize_formatted_duration(field.duration_format) + for i, format_token in enumerate(tokens): + sql_to_text_funcs = DURATION_FORMAT_TOKENS[format_token]["sql_to_text"] + sql_to_text = sql_to_text_funcs.get( + field.duration_format, sql_to_text_funcs["default"] + ) + format_func += sql_to_text + + # Add the proper separator between each token, if it's not the last one + if i == len(tokens) - 1: + break + elif format_token == "d": # nosec b105 + format_func += " || ' ' || " + else: + format_func += " || ':' || " + (format_func) + return format_func diff --git a/backend/src/baserow/contrib/database/migrations/0149_alter_durationfield_duration_format.py b/backend/src/baserow/contrib/database/migrations/0149_alter_durationfield_duration_format.py new file mode 100644 index 000000000..2413b8335 --- /dev/null +++ b/backend/src/baserow/contrib/database/migrations/0149_alter_durationfield_duration_format.py @@ -0,0 +1,31 @@ +# Generated by Django 4.0.10 on 2024-01-03 15:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0148_add_formula_button_type"), + ] + + operations = [ + migrations.AlterField( + model_name="durationfield", + name="duration_format", + field=models.CharField( + choices=[ + ("h:mm", "hours:minutes"), + ("h:mm:ss", "hours:minutes:seconds"), + ("h:mm:ss.s", "hours:minutes:seconds:deciseconds"), + ("h:mm:ss.ss", "hours:minutes:seconds:centiseconds"), + ("h:mm:ss.sss", "hours:minutes:seconds:milliseconds"), + ("d h", "days:hours"), + ("d h:mm", "days:hours:minutes"), + ("d h:mm:ss", "days:hours:minutes:seconds"), + ], + default="h:mm", + help_text="The format of the duration.", + max_length=32, + ), + ), + ] diff --git a/backend/src/baserow/test_utils/helpers.py b/backend/src/baserow/test_utils/helpers.py index 672858fb7..facbc6706 100644 --- a/backend/src/baserow/test_utils/helpers.py +++ b/backend/src/baserow/test_utils/helpers.py @@ -195,11 +195,14 @@ def setup_interesting_test_table( "created_on_datetime_eu_tzone": None, "last_modified_by": None, "created_by": None, - "duration_hm": timedelta(seconds=3660), - "duration_hms": timedelta(seconds=3666), - "duration_hms_s": timedelta(seconds=3666.6), - "duration_hms_ss": timedelta(seconds=3666.66), - "duration_hms_sss": timedelta(seconds=3666.666), + "duration_hm": timedelta(hours=1, minutes=1), + "duration_hms": timedelta(hours=1, minutes=1, seconds=6), + "duration_hms_s": timedelta(hours=1, minutes=1, seconds=6.6), + "duration_hms_ss": timedelta(hours=1, minutes=1, seconds=6.66), + "duration_hms_sss": timedelta(hours=1, minutes=1, seconds=6.666), + "duration_dh": timedelta(days=1, hours=1), + "duration_dhm": timedelta(days=1, hours=1, minutes=1), + "duration_dhms": timedelta(days=1, hours=1, minutes=1, seconds=6), # We will setup link rows manually later "link_row": None, "self_link_row": None, diff --git a/backend/tests/baserow/contrib/database/field/test_duration_field_serializers.py b/backend/tests/baserow/contrib/database/api/fields/test_duration_field_serializers.py similarity index 52% rename from backend/tests/baserow/contrib/database/field/test_duration_field_serializers.py rename to backend/tests/baserow/contrib/database/api/fields/test_duration_field_serializers.py index 5b2262ce3..ea9f3f584 100644 --- a/backend/tests/baserow/contrib/database/field/test_duration_field_serializers.py +++ b/backend/tests/baserow/contrib/database/api/fields/test_duration_field_serializers.py @@ -7,13 +7,14 @@ from baserow.contrib.database.api.fields.serializers import DurationFieldSeriali from baserow.contrib.database.fields.utils.duration import ( DURATION_FORMAT_TOKENS, DURATION_FORMATS, + tokenize_formatted_duration, ) @pytest.mark.parametrize( "duration_format,user_input,saved_value", [ - # Normal input: + # number input: ("h:mm", 0, timedelta(seconds=0)), ("h:mm", 3660, timedelta(seconds=3660)), ("h:mm", 86400, timedelta(days=1)), @@ -23,6 +24,14 @@ from baserow.contrib.database.fields.utils.duration import ( ("h:mm:ss.ss", 3661.12, timedelta(seconds=3661.12)), ("h:mm:ss.sss", 0, timedelta(seconds=0)), ("h:mm:ss.sss", 3661.123, timedelta(seconds=3661.123)), + ("d h", 0, timedelta(seconds=0)), + ("d h", 3600, timedelta(hours=1)), + ("d h", 90000, timedelta(days=1, hours=1)), + ("d h:mm", 0, timedelta(seconds=0)), + ("d h:mm", 3660, timedelta(hours=1, minutes=1)), + ("d h:mm", 86400, timedelta(days=1)), + ("d h:mm:ss", 0, timedelta(seconds=0)), + ("d h:mm:ss", 3661, timedelta(seconds=3661)), # Rounding: ("h:mm", 3661.123, timedelta(seconds=3660)), ("h:mm", 3661.999, timedelta(seconds=3660)), @@ -34,12 +43,39 @@ from baserow.contrib.database.fields.utils.duration import ( ("h:mm:ss.ss", 3661.789, timedelta(seconds=3661.79)), ("h:mm:ss.sss", 3661.1234, timedelta(seconds=3661.123)), ("h:mm:ss.sss", 3661.6789, timedelta(seconds=3661.679)), + ("d h", 86400 + 3600.123, timedelta(days=1, hours=1)), + ("d h", 29.9 * 60, timedelta(seconds=0)), + ("d h", 30 * 60, timedelta(hours=1)), + ("d h:mm", 29, timedelta(seconds=0)), + ("d h:mm", 30, timedelta(minutes=1)), + ("d h:mm:ss", 0.49, timedelta(seconds=0)), + ("d h:mm:ss", 0.5, timedelta(seconds=1)), # String input: ("h:mm", "1:01", timedelta(seconds=3660)), ("h:mm:ss", "1:01:01", timedelta(seconds=3661)), ("h:mm:ss.s", "1:01:01.1", timedelta(seconds=3661.1)), ("h:mm:ss.ss", "1:01:01.12", timedelta(seconds=3661.12)), ("h:mm:ss.sss", "1:01:01.123", timedelta(seconds=3661.123)), + ("d h", "1 1", timedelta(days=1, hours=1)), + ("d h", "1d1h", timedelta(days=1, hours=1)), + ("d h", "1d 1h", timedelta(days=1, hours=1)), + ("d h", "1 1h", timedelta(days=1, hours=1)), + ("d h", "1d 1", timedelta(days=1, hours=1)), + ("d h", "1d1", timedelta(days=1, hours=1)), + ("d h", "12h", timedelta(hours=12)), + ("d h", "3d", timedelta(days=3)), + ("d h:mm", "1 1:01", timedelta(days=1, seconds=3660)), + ("d h:mm", "1d1:01", timedelta(days=1, seconds=3660)), + ("d h:mm", "1d 1:01", timedelta(days=1, seconds=3660)), + ("d h:mm", "1 1:1", timedelta(days=1, seconds=3660)), + ("d h:mm", "1d1:1", timedelta(days=1, seconds=3660)), + ("d h:mm", "1d 1:1", timedelta(days=1, seconds=3660)), + ("d h:mm:ss", "1 1:01:01", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1d1:01:01", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1d 1:01:01", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1 1:1:1", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1d1:1:1", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1d 1:1:1", timedelta(days=1, seconds=3661)), # String input with rounding: ("h:mm", "1:01:01.123", timedelta(seconds=3660)), ("h:mm", "1:01:01.999", timedelta(seconds=3660)), @@ -51,18 +87,35 @@ from baserow.contrib.database.fields.utils.duration import ( ("h:mm:ss.ss", "1:01:01.789", timedelta(seconds=3661.79)), ("h:mm:ss.sss", "1:01:01.1234", timedelta(seconds=3661.123)), ("h:mm:ss.sss", "1:01:01.6789", timedelta(seconds=3661.679)), + ("d h", "1 1:01:01.123", timedelta(days=1, hours=1)), + ("d h", "1.123", timedelta(seconds=0)), + ("d h", "29:59.999", timedelta(seconds=0)), + ("d h", "30:00", timedelta(hours=30)), + ("d h", "30:00.001", timedelta(hours=1)), + ("d h:mm", "1 1:01:01.123", timedelta(days=1, hours=1, minutes=1)), + ("d h:mm", "1.123", timedelta(seconds=0)), + ("d h:mm", "29.999", timedelta(seconds=0)), + ("d h:mm", "30", timedelta(minutes=1)), + ("d h:mm", "30.001", timedelta(minutes=1)), + ("d h:mm:ss", "1 1:01:01.123", timedelta(days=1, seconds=3661)), + ("d h:mm:ss", "1.123", timedelta(seconds=1)), + ("d h:mm:ss", "29.999", timedelta(seconds=30)), + ("d h:mm:ss", "70", timedelta(seconds=70)), + ("d h:mm:ss", "0.5", timedelta(seconds=1)), # None should be None in every format: ("h:mm", None, None), ("h:mm:ss", None, None), ("h:mm:ss.s", None, None), ("h:mm:ss.ss", None, None), ("h:mm:ss.sss", None, None), + ("d h", None, None), + ("d h:mm", None, None), + ("d h:mm:ss", None, None), ], ) -@pytest.mark.django_db @pytest.mark.field_duration def test_duration_serializer_to_internal_value( - data_fixture, duration_format, user_input, saved_value + duration_format, user_input, saved_value ): """ Tests that for the Duration Serializer, the value is always serialized as @@ -88,12 +141,15 @@ def test_duration_serializer_to_internal_value( ("h:mm:ss.s", "aaaaaaa"), ("invalid format", 1), ("h:m", timedelta.max.total_seconds() + 1), # Overflow + ("d h", "1dd"), + ("d h", "1hh"), + ("d h", "1d1d"), + ("d h", "1h1h"), ], ) -@pytest.mark.django_db @pytest.mark.field_duration def test_duration_serializer_to_internal_value_with_invalid_values( - data_fixture, duration_format, user_input + duration_format, user_input ): serializer = DurationFieldSerializer(duration_format=duration_format) with pytest.raises(serializers.ValidationError): @@ -103,38 +159,34 @@ def test_duration_serializer_to_internal_value_with_invalid_values( @pytest.mark.parametrize( "duration_format,user_input,returned_value", [ - ("h:mm", 3660, 3660), - ("h:mm:ss", 3661, 3661), - ("h:mm:ss.s", 3661.1, 3661.1), - ("h:mm:ss.ss", 3661.12, 3661.12), - ("h:mm:ss.sss", 3661.123, 3661.123), - ("h:mm:ss.sss", 3661.1234, 3661.1234), + ("h:mm", timedelta(seconds=0), 0), + ("h:mm", timedelta(hours=1, minutes=1), 3660), + ("h:mm:ss", timedelta(hours=1, minutes=1, seconds=1), 3661), + ("h:mm:ss.s", timedelta(hours=1, minutes=1, seconds=1.1), 3661.1), + ("h:mm:ss.ss", timedelta(hours=1, minutes=1, seconds=1.12), 3661.12), + ("h:mm:ss.sss", timedelta(hours=1, minutes=1, seconds=1.123), 3661.123), + ("h:mm:ss.sss", timedelta(hours=1, minutes=1, seconds=1.1234), 3661.1234), + ("d h", timedelta(days=1, hours=1), 90000), + ("d h:mm", timedelta(days=1, hours=1, minutes=1), 90060), + ("d h:mm:ss", timedelta(days=1, hours=1, minutes=1, seconds=1), 90061), ], ) -@pytest.mark.django_db @pytest.mark.field_duration def test_duration_serializer_to_representation( - data_fixture, duration_format, user_input, returned_value + duration_format, user_input, returned_value ): - """ - Tests that for the Duration Serializer, the representation is returned in - seconds (value.total_seconds()) from the database for every duration format. - """ - serializer = DurationFieldSerializer(duration_format=duration_format) - - assert serializer.to_representation(timedelta(seconds=user_input)) == returned_value + assert serializer.to_representation(user_input) == returned_value -@pytest.mark.django_db @pytest.mark.field_duration -def test_duration_token_options(data_fixture): +def test_duration_token_options(): """ - Tests that the token options are correct for the duration field. + Tests that format are made of tokens that are in DURATION_FORMAT_TOKENS. """ for format in DURATION_FORMATS.keys(): - for token in format.split(":"): + for token in tokenize_formatted_duration(format): assert token in DURATION_FORMAT_TOKENS, ( f"{token} not in DURATION_FORMAT_TOKENS. Please add it with the correct " "options to be able to convert a formatted string to a timedelta." diff --git a/backend/tests/baserow/contrib/database/api/rows/test_row_serializers.py b/backend/tests/baserow/contrib/database/api/rows/test_row_serializers.py index e44315097..c972666c2 100644 --- a/backend/tests/baserow/contrib/database/api/rows/test_row_serializers.py +++ b/backend/tests/baserow/contrib/database/api/rows/test_row_serializers.py @@ -250,6 +250,9 @@ def test_get_row_serializer_with_user_field_names(data_fixture): "duration_hms_s": 3666.6, "duration_hms_ss": 3666.66, "duration_hms_sss": 3666.666, + "duration_dh": 90000, + "duration_dhm": 90060, + "duration_dhms": 90066, "email": "test@example.com", "file": [ { diff --git a/backend/tests/baserow/contrib/database/api/views/test_view_serializers.py b/backend/tests/baserow/contrib/database/api/views/test_view_serializers.py index 4101f595b..082df2e61 100644 --- a/backend/tests/baserow/contrib/database/api/views/test_view_serializers.py +++ b/backend/tests/baserow/contrib/database/api/views/test_view_serializers.py @@ -1,4 +1,5 @@ import pytest +from pytest_unordered import unordered from baserow.contrib.database.api.views.serializers import serialize_group_by_metadata from baserow.contrib.database.fields.models import Field @@ -90,7 +91,7 @@ def test_serialize_group_by_metadata_on_all_fields_in_interesting_table(data_fix # comparison. for result in serialized: result[f"field_{field.name}"] = result.pop(f"field_{str(field.id)}") - actual_result_per_field_name[field.name] = serialized + actual_result_per_field_name[field.name] = unordered(serialized) assert actual_result_per_field_name == { "text": [ @@ -237,4 +238,16 @@ def test_serialize_group_by_metadata_on_all_fields_in_interesting_table(data_fix {"count": 1, "field_duration_hms_sss": 3666.666}, {"count": 1, "field_duration_hms_sss": None}, ], + "duration_dh": [ + {"count": 1, "field_duration_dh": 90000.0}, + {"count": 1, "field_duration_dh": None}, + ], + "duration_dhm": [ + {"count": 1, "field_duration_dhm": 90060.0}, + {"count": 1, "field_duration_dhm": None}, + ], + "duration_dhms": [ + {"count": 1, "field_duration_dhms": 90066.0}, + {"count": 1, "field_duration_dhms": None}, + ], } diff --git a/backend/tests/baserow/contrib/database/export/test_export_handler.py b/backend/tests/baserow/contrib/database/export/test_export_handler.py index f956f88c5..91e0062a7 100755 --- a/backend/tests/baserow/contrib/database/export/test_export_handler.py +++ b/backend/tests/baserow/contrib/database/export/test_export_handler.py @@ -227,14 +227,15 @@ def test_can_export_every_interesting_different_field_to_csv( "last_modified_date_eu,last_modified_datetime_eu_tzone,created_on_datetime_us," "created_on_date_us,created_on_datetime_eu,created_on_date_eu,created_on_datetime_eu_tzone," "last_modified_by,created_by,duration_hm,duration_hms,duration_hms_s,duration_hms_ss," - "duration_hms_sss,link_row,self_link_row,link_row_without_related,decimal_link_row," + "duration_hms_sss,duration_dh,duration_dhm,duration_dhms," + "link_row,self_link_row,link_row_without_related,decimal_link_row," "file_link_row,file,single_select,multiple_select,multiple_collaborators," "phone_number,formula_text,formula_int,formula_bool,formula_decimal,formula_dateinterval," "formula_date,formula_singleselect,formula_email,formula_link_with_label," "formula_link_url_only,formula_multipleselect,count,rollup,lookup,uuid,autonumber\r\n" "1,,,,,,,,,0,False,,,,,,,01/02/2021 12:00,01/02/2021,02/01/2021 12:00,02/01/2021," "02/01/2021 13:00,01/02/2021 12:00,01/02/2021,02/01/2021 12:00,02/01/2021,02/01/2021 13:00," - "user@example.com,user@example.com,,,,,,,,,,,,,,,,test FORMULA,1,True,33.3333333333," + "user@example.com,user@example.com,,,,,,,,,,,,,,,,,,,test FORMULA,1,True,33.3333333333," "1 day,2020-01-01,,,label (https://google.com),https://google.com,,0,0.000,," "00000000-0000-4000-8000-000000000001,1\r\n" "2,text,long_text,https://www.google.com,test@example.com,-1,1,-1.2,1.2,3,True," @@ -242,7 +243,7 @@ def test_can_export_every_interesting_different_field_to_csv( "01/02/2020 02:23,01/02/2021 12:00,01/02/2021,02/01/2021 12:00,02/01/2021," "02/01/2021 13:00,01/02/2021 12:00,01/02/2021,02/01/2021 12:00,02/01/2021," "02/01/2021 13:00,user@example.com,user@example.com," - "1:01,1:01:06,1:01:06.6,1:01:06.66,1:01:06.666," + "1:01,1:01:06,1:01:06.6,1:01:06.66,1:01:06.666,1d 1h,1d 1:01,1d 1:01:06," '"linked_row_1,linked_row_2,unnamed row 3",unnamed row 1,' '"linked_row_1,linked_row_2","1.234,-123.456,unnamed row 3",' '"name.txt (http://localhost:8000/media/user_files/test_hash.txt),unnamed row 2",' diff --git a/backend/tests/baserow/contrib/database/field/test_duration_field_type.py b/backend/tests/baserow/contrib/database/field/test_duration_field_type.py index f644fc4d5..27424cff2 100644 --- a/backend/tests/baserow/contrib/database/field/test_duration_field_type.py +++ b/backend/tests/baserow/contrib/database/field/test_duration_field_type.py @@ -196,6 +196,15 @@ def test_remove_duration_field(data_fixture): (timedelta(seconds=3665.5), "h:mm:ss.s", "1:01:05.5"), (timedelta(seconds=3665.55), "h:mm:ss.ss", "1:01:05.55"), (timedelta(seconds=3665.555), "h:mm:ss.sss", "1:01:05.555"), + (timedelta(days=1, hours=2), "d h", "1d 2h"), + (timedelta(days=1, hours=1, minutes=2), "d h:mm", "1d 1:02"), + (timedelta(days=1, hours=1, minutes=2, seconds=3), "d h:mm:ss", "1d 1:02:03"), + (None, "h:mm", None), + (None, "d h", None), + (None, "d h:mm", None), + (None, "d h:mm:ss", None), + (0, "h:mm", "0:00"), + (0, "d h", "0d 0h"), ], ) @pytest.mark.django_db @@ -261,6 +270,21 @@ def test_convert_duration_field_to_text_field( "1:01:05.555", timedelta(seconds=3665.555), ), + ( + "d h", + "1d 2h", + timedelta(days=1, hours=2), + ), + ( + "d h:mm", + "1d 1:02", + timedelta(days=1, hours=1, minutes=2), + ), + ( + "d h:mm:ss", + "1d 1:02:03", + timedelta(days=1, hours=1, minutes=2, seconds=3), + ), ], ) @pytest.mark.django_db @@ -315,11 +339,38 @@ def test_convert_text_field_to_duration_field( @pytest.mark.parametrize( "expected,field_kwargs", [ - ([0, 0, 60, 120, 120], {"duration_format": "h:mm"}), - ([1, 10, 51, 100, 122], {"duration_format": "h:mm:ss"}), - ([1.2, 10.1, 50.7, 100.1, 122], {"duration_format": "h:mm:ss.s"}), - ([1.20, 10.11, 50.68, 100.1, 122], {"duration_format": "h:mm:ss.ss"}), - ([1.199, 10.11, 50.679, 100.1, 122], {"duration_format": "h:mm:ss.sss"}), + ( + [0, 0, 60, 120, 120, 86460, 90000], + {"duration_format": "h:mm"}, + ), + ( + [1, 10, 51, 100, 122, 86461, 90002], + {"duration_format": "h:mm:ss"}, + ), + ( + [1.2, 10.1, 50.7, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.s"}, + ), + ( + [1.20, 10.11, 50.68, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.ss"}, + ), + ( + [1.199, 10.11, 50.679, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.sss"}, + ), + ( + [0, 0, 0, 0, 0, 86400, 90000], + {"duration_format": "d h"}, + ), + ( + [0, 0, 60, 120, 120, 86460, 90000], + {"duration_format": "d h:mm"}, + ), + ( + [1, 10, 51, 100, 122, 86461, 90002], + {"duration_format": "d h:mm:ss"}, + ), ], ) def test_alter_duration_format(expected, field_kwargs, data_fixture): @@ -329,7 +380,7 @@ def test_alter_duration_format(expected, field_kwargs, data_fixture): user=user, table=table, duration_format="h:mm:ss.sss" ) - original_values = [1.199, 10.11, 50.6789, 100.1, 122] + original_values = [1.199, 10.11, 50.6789, 100.1, 122, 86461, 90001.8] RowHandler().create_rows( user, @@ -353,11 +404,38 @@ def test_alter_duration_format(expected, field_kwargs, data_fixture): @pytest.mark.parametrize( "expected,field_kwargs", [ - ([0, 0, 60, 120, 120], {"duration_format": "h:mm"}), - ([1, 10, 51, 100, 122], {"duration_format": "h:mm:ss"}), - ([1.2, 10.1, 50.7, 100.1, 122], {"duration_format": "h:mm:ss.s"}), - ([1.20, 10.11, 50.68, 100.1, 122], {"duration_format": "h:mm:ss.ss"}), - ([1.199, 10.11, 50.679, 100.1, 122], {"duration_format": "h:mm:ss.sss"}), + ( + [0, 0, 60, 120, 120, 86460, 90000], + {"duration_format": "h:mm"}, + ), + ( + [1, 10, 51, 100, 122, 86461, 90002], + {"duration_format": "h:mm:ss"}, + ), + ( + [1.2, 10.1, 50.7, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.s"}, + ), + ( + [1.20, 10.11, 50.68, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.ss"}, + ), + ( + [1.199, 10.11, 50.679, 100.1, 122, 86461, 90001.8], + {"duration_format": "h:mm:ss.sss"}, + ), + ( + [0, 0, 0, 0, 0, 86400, 90000], + {"duration_format": "d h"}, + ), + ( + [0, 0, 60, 120, 120, 86460, 90000], + {"duration_format": "d h:mm"}, + ), + ( + [1, 10, 51, 100, 122, 86461, 90002], + {"duration_format": "d h:mm:ss"}, + ), ], ) def test_alter_duration_format_can_be_undone(expected, field_kwargs, data_fixture): @@ -368,7 +446,7 @@ def test_alter_duration_format_can_be_undone(expected, field_kwargs, data_fixtur user=user, table=table, duration_format="h:mm:ss.sss" ) - original_values = [1.199, 10.11, 50.679, 100.1, 122] + original_values = [1.199, 10.11, 50.679, 100.1, 122, 86461, 90001.8] RowHandler().create_rows( user, @@ -416,7 +494,7 @@ def test_duration_field_view_filters(data_fixture): {field.db_column: 1.123}, {field.db_column: 60}, # 1min {field.db_column: "24:0:0"}, # 1day - {field.db_column: 86400}, # 1day + {field.db_column: "1 0"}, # 1day {field.db_column: 3601}, # 1hour 1sec {field.db_column: "1:0:0"}, # 1 hour ], diff --git a/backend/tests/baserow/contrib/integrations/local_baserow/test_service_types.py b/backend/tests/baserow/contrib/integrations/local_baserow/test_service_types.py index 57ac52917..bf63424c5 100644 --- a/backend/tests/baserow/contrib/integrations/local_baserow/test_service_types.py +++ b/backend/tests/baserow/contrib/integrations/local_baserow/test_service_types.py @@ -1563,6 +1563,27 @@ def test_local_baserow_table_service_generate_schema_with_interesting_test_table "metadata": {}, "type": "string", }, + field_db_column_by_name["duration_dh"]: { + "title": "duration_dh", + "default": None, + "original_type": "duration", + "metadata": {}, + "type": "string", + }, + field_db_column_by_name["duration_dhm"]: { + "title": "duration_dhm", + "default": None, + "original_type": "duration", + "metadata": {}, + "type": "string", + }, + field_db_column_by_name["duration_dhms"]: { + "title": "duration_dhms", + "default": None, + "original_type": "duration", + "metadata": {}, + "type": "string", + }, "id": {"metadata": {}, "type": "number", "title": "Id"}, } diff --git a/changelog/entries/unreleased/feature/2217_add_formats_with_days_to_the_duration_field.json b/changelog/entries/unreleased/feature/2217_add_formats_with_days_to_the_duration_field.json new file mode 100644 index 000000000..7046003e7 --- /dev/null +++ b/changelog/entries/unreleased/feature/2217_add_formats_with_days_to_the_duration_field.json @@ -0,0 +1,7 @@ +{ + "type": "feature", + "message": "Add formats with days to the duration field.", + "issue_number": 2217, + "bullet_points": [], + "created_at": "2024-01-04" +} \ No newline at end of file diff --git a/premium/backend/tests/baserow_premium_tests/export/test_premium_export_types.py b/premium/backend/tests/baserow_premium_tests/export/test_premium_export_types.py index c2062b1da..da74a1af2 100644 --- a/premium/backend/tests/baserow_premium_tests/export/test_premium_export_types.py +++ b/premium/backend/tests/baserow_premium_tests/export/test_premium_export_types.py @@ -71,6 +71,9 @@ def test_can_export_every_interesting_different_field_to_json( "duration_hms_s": "", "duration_hms_ss": "", "duration_hms_sss": "", + "duration_dh": "", + "duration_dhm": "", + "duration_dhms": "", "link_row": [], "self_link_row": [], "link_row_without_related": [], @@ -138,6 +141,9 @@ def test_can_export_every_interesting_different_field_to_json( "duration_hms_s": "1:01:06.6", "duration_hms_ss": "1:01:06.66", "duration_hms_sss": "1:01:06.666", + "duration_dh": "1d 1h", + "duration_dhm": "1d 1:01", + "duration_dhms": "1d 1:01:06", "link_row": [ "linked_row_1", "linked_row_2", @@ -328,6 +334,9 @@ def test_can_export_every_interesting_different_field_to_xml( <duration-hms-s/> <duration-hms-ss/> <duration-hms-sss/> + <duration-dh/> + <duration-dhm/> + <duration-dhms/> <link-row/> <self-link-row/> <link-row-without-related/> @@ -395,6 +404,9 @@ def test_can_export_every_interesting_different_field_to_xml( <duration-hms-s>1:01:06.6</duration-hms-s> <duration-hms-ss>1:01:06.66</duration-hms-ss> <duration-hms-sss>1:01:06.666</duration-hms-sss> + <duration-dh>1d 1h</duration-dh> + <duration-dhm>1d 1:01</duration-dhm> + <duration-dhms>1d 1:01:06</duration-dhms> <link-row> <item>linked_row_1</item> <item>linked_row_2</item> diff --git a/web-frontend/modules/database/components/view/ViewFilterTypeDuration.vue b/web-frontend/modules/database/components/view/ViewFilterTypeDuration.vue index dcbed3213..b0e0c6851 100644 --- a/web-frontend/modules/database/components/view/ViewFilterTypeDuration.vue +++ b/web-frontend/modules/database/components/view/ViewFilterTypeDuration.vue @@ -38,7 +38,9 @@ export default { setCopyAndDelayedUpdate(value, immediately = false) { const newValue = this.updateCopy(this.field, value) if (newValue !== undefined) { - this.delayedUpdate(this.copy, immediately) + // a filter Value cannot be null, so send an empty string in case. + const filterValue = newValue === null ? '' : newValue + this.delayedUpdate(filterValue, immediately) } }, getValidationError(value) { diff --git a/web-frontend/modules/database/fieldTypes.js b/web-frontend/modules/database/fieldTypes.js index 60efa156a..4be2511d2 100644 --- a/web-frontend/modules/database/fieldTypes.js +++ b/web-frontend/modules/database/fieldTypes.js @@ -1,6 +1,6 @@ import BigNumber from 'bignumber.js' import { - formatDuration, + formatDurationValue, parseDurationValue, roundDurationValueToFormat, DURATION_FORMATS, @@ -2285,31 +2285,17 @@ export class DurationFieldType extends FieldType { getSort(name, order) { return (a, b) => { - const aValue = parseDurationValue(a[name]) - const bValue = parseDurationValue(b[name]) + const aValue = a[name] + const bValue = b[name] if (aValue === bValue) { return 0 } - if ( - (aValue === null && order === 'ASC') || - (bValue === null && order === 'DESC') - ) { - return -1 - } - - if ( - (bValue === null && order === 'ASC') || - (aValue === null && order === 'DESC') - ) { - return 1 - } - if (order === 'ASC') { - return aValue < bValue ? -1 : 1 + return aValue === null || (bValue !== null && aValue < bValue) ? -1 : 1 } else { - return bValue < aValue ? -1 : 1 + return bValue === null || (aValue !== null && bValue < aValue) ? -1 : 1 } } } @@ -2361,7 +2347,7 @@ export class DurationFieldType extends FieldType { } static formatValue(field, value) { - return formatDuration(value, field.duration_format) + return formatDurationValue(value, field.duration_format) } static parseInputValue(field, value) { diff --git a/web-frontend/modules/database/mixins/durationField.js b/web-frontend/modules/database/mixins/durationField.js index 5589f0266..dd9918cab 100644 --- a/web-frontend/modules/database/mixins/durationField.js +++ b/web-frontend/modules/database/mixins/durationField.js @@ -48,7 +48,8 @@ export default { } }, onKeyPress(field, event) { - if (!/\d/.test(event.key) && event.key !== '.' && event.key !== ':') { + const allowedChars = ['.', ':', ' ', 'd', 'h'] + if (!/\d/.test(event.key) && !allowedChars.includes(event.key)) { return event.preventDefault() } }, diff --git a/web-frontend/modules/database/utils/duration.js b/web-frontend/modules/database/utils/duration.js index 20340cc3f..f878c0b45 100644 --- a/web-frontend/modules/database/utils/duration.js +++ b/web-frontend/modules/database/utils/duration.js @@ -1,28 +1,122 @@ const MOST_ACCURATE_DURATION_FORMAT = 'h:mm:ss.sss' -const MAX_NUMBER_OF_TOKENS_IN_DURATION_FORMAT = - MOST_ACCURATE_DURATION_FORMAT.split(':').length -export const MAX_BACKEND_DURATION_VALUE_NUMBER_OF_SECS = 86400000000000 // taken from backend timedelta.max.total_seconds() +// taken from backend timedelta.max.total_seconds() == 1_000_000_000 days +export const MAX_BACKEND_DURATION_VALUE_NUMBER_OF_SECS = 86400000000000 +export const DEFAULT_DURATION_FORMAT = 'h:mm' + +const D_H = 'd h' +const D_H_M = 'd h:mm' +const D_H_M_S = 'd h:mm:ss' +const H_M = 'h:mm' +const H_M_S = 'h:mm:ss' +const H_M_S_S = 'h:mm:ss.s' +const H_M_S_SS = 'h:mm:ss.ss' +const H_M_S_SSS = 'h:mm:ss.sss' + +const SECS_IN_DAY = 86400 +const SECS_IN_HOUR = 3600 +const SECS_IN_MIN = 60 + +function totalSecs({ secs = 0, mins = 0, hours = 0, days = 0 }) { + return ( + parseInt(days) * SECS_IN_DAY + + parseInt(hours) * SECS_IN_HOUR + + parseInt(mins) * SECS_IN_MIN + + parseFloat(secs) + ) +} + +const DURATION_REGEXPS = new Map([ + [ + /^(\d+)(?:d\s*|\s+)(\d+):(\d+):(\d+|\d+\.\d+)$/, + { + default: (days, hours, mins, secs) => + totalSecs({ days, hours, mins, secs }), + }, + ], + [ + /^(\d+):(\d+):(\d+|\d+\.\d+)$/, + { + default: (hours, mins, secs) => totalSecs({ hours, mins, secs }), + }, + ], + [ + /^(\d+)(?:d\s*|\s+)(\d+):(\d+)$/, + { + [D_H]: (days, hours, mins) => totalSecs({ days, hours, mins }), + [D_H_M]: (days, hours, mins) => totalSecs({ days, hours, mins }), + default: (days, mins, secs) => totalSecs({ days, mins, secs }), + }, + ], + [ + /^(\d+)(?:d\s*|\s+)(\d+):(\d+\.\d+)$/, + { default: (days, mins, secs) => totalSecs({ days, mins, secs }) }, + ], + [/^(\d+)h$/, { default: (hours) => totalSecs({ hours }) }], + [ + /^(\d+)(?:d\s*|\s+)(\d+)h$/, + { + default: (days, hours) => totalSecs({ days, hours }), + }, + ], + [/^(\d+)d$/, { default: (days) => totalSecs({ days }) }], + [ + /^(\d+)(?:d\s*|\s+)(\d+)$/, + { + [D_H]: (days, hours) => totalSecs({ days, hours }), + [D_H_M]: (days, mins) => totalSecs({ days, mins }), + [H_M]: (days, mins) => totalSecs({ days, mins }), + default: (days, secs) => totalSecs({ days, secs }), + }, + ], + [ + /^(\d+)(?:d\s*|\s+)(\d+\.\d+)$/, + { default: (days, secs) => totalSecs({ days, secs }) }, + ], + [ + /^(\d+):(\d+)$/, + { + [D_H]: (hours, mins) => totalSecs({ hours, mins }), + [D_H_M]: (hours, mins) => totalSecs({ hours, mins }), + [H_M]: (hours, mins) => totalSecs({ hours, mins }), + default: (mins, secs) => totalSecs({ mins, secs }), + }, + ], + [ + /^(\d+):(\d+\.\d+)$/, + { default: (mins, secs) => totalSecs({ mins, secs }) }, + ], + [/^(\d+\.\d+)$/, { default: (secs) => totalSecs({ secs }) }], + [ + /^(\d+)$/, + { + [D_H]: (hours) => totalSecs({ hours }), + [D_H_M]: (mins) => totalSecs({ mins }), + [H_M]: (mins) => totalSecs({ mins }), + default: (secs) => totalSecs({ secs }), + }, + ], +]) // Map guarantees the order of the entries export const DURATION_FORMATS = new Map([ [ - 'h:mm', + H_M, { description: 'h:mm (1:23)', example: '1:23', - toString(hours, minutes) { - return `${hours}:${minutes.toString().padStart(2, '0')}` + toString(d, h, m, s) { + return `${d * 24 + h}:${m.toString().padStart(2, '0')}` }, round: (value) => Math.round(value / 60) * 60, }, ], [ - 'h:mm:ss', + H_M_S, { description: 'h:mm:ss (1:23:40)', example: '1:23:40', - toString(hours, minutes, seconds) { - return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds + toString(d, h, m, s) { + return `${d * 24 + h}:${m.toString().padStart(2, '0')}:${s .toString() .padStart(2, '0')}` }, @@ -30,12 +124,12 @@ export const DURATION_FORMATS = new Map([ }, ], [ - 'h:mm:ss.s', + H_M_S_S, { description: 'h:mm:ss.s (1:23:40.0)', example: '1:23:40.0', - toString(hours, minutes, seconds) { - return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds + toString(d, h, m, s) { + return `${d * 24 + h}:${m.toString().padStart(2, '0')}:${s .toFixed(1) .padStart(4, '0')}` }, @@ -43,12 +137,12 @@ export const DURATION_FORMATS = new Map([ }, ], [ - 'h:mm:ss.ss', + H_M_S_SS, { description: 'h:mm:ss.ss (1:23:40.00)', example: '1:23:40.00', - toString(hours, minutes, seconds) { - return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds + toString(d, h, m, s) { + return `${d * 24 + h}:${m.toString().padStart(2, '0')}:${s .toFixed(2) .padStart(5, '0')}` }, @@ -56,54 +150,65 @@ export const DURATION_FORMATS = new Map([ }, ], [ - 'h:mm:ss.sss', + H_M_S_SSS, { description: 'h:mm:ss.sss (1:23:40.000)', example: '1:23:40.000', - toString(hours, minutes, seconds) { - return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds + toString(d, h, m, s) { + return `${d * 24 + h}:${m.toString().padStart(2, '0')}:${s .toFixed(3) .padStart(6, '0')}` }, round: (value) => Math.round(value * 1000) / 1000, }, ], + [ + D_H, + { + description: 'd h (1d 2h)', + example: '1d 2h', + toString(d, h, m, s) { + return `${d}d ${h}h` + }, + round: (value) => Math.round(value / 3600) * 3600, + }, + ], + [ + D_H_M, + { + description: 'd h:mm (1d 2:34)', + example: '1d 2:34', + toString(d, h, m, s) { + return `${d}d ${h}:${m.toString().padStart(2, '0')}` + }, + round: (value) => Math.round(value / 60) * 60, + }, + ], + [ + D_H_M_S, + { + description: 'd h:mm:ss (1d 2:34:56)', + example: '1d 2:34:56', + toString(d, h, m, s) { + return `${d}d ${h}:${m.toString().padStart(2, '0')}:${s + .toString() + .padStart(2, '0')}` + }, + round: (value) => Math.round(value), + }, + ], ]) -export const DURATION_TOKENS = { - h: { - multiplier: 3600, - parse: (value) => parseInt(value), - }, - mm: { - multiplier: 60, - parse: (value) => parseInt(value), - }, - ss: { - multiplier: 1, - parse: (value) => parseInt(value), - }, - 'ss.s': { - multiplier: 1, - parse: (value) => Math.round(parseFloat(value) * 10) / 10, - }, - 'ss.ss': { - multiplier: 1, - parse: (value) => Math.round(parseFloat(value) * 100) / 100, - }, - 'ss.sss': { - multiplier: 1, - parse: (value) => Math.round(parseFloat(value) * 1000) / 1000, - }, -} - export const roundDurationValueToFormat = (value, format) => { if (value === null) { return null } - const roundFunc = DURATION_FORMATS.get(format).round - return roundFunc(value) + const durationFormatOptions = DURATION_FORMATS.get(format) + if (!durationFormatOptions) { + throw new Error(`Unknown duration format ${format}`) + } + return durationFormatOptions.round(value) } /** @@ -113,65 +218,40 @@ export const roundDurationValueToFormat = (value, format) => { */ export const parseDurationValue = ( inputValue, - format = MOST_ACCURATE_DURATION_FORMAT, - strict = false + format = MOST_ACCURATE_DURATION_FORMAT ) => { if (inputValue === null || inputValue === undefined || inputValue === '') { return null } - // If the input value is a number, we assume it is in seconds. + // If the value is a number, we assume it's already in seconds (i.e. from the backend). if (Number.isFinite(inputValue)) { return inputValue > 0 ? inputValue : null } - const parts = inputValue.split(':').reverse() - let tokens = format.split(':').reverse() - if (parts.length > tokens.length) { - if (strict || parts.length > MAX_NUMBER_OF_TOKENS_IN_DURATION_FORMAT) { - throw new Error( - `Invalid duration format: ${inputValue} does not match ${format}` - ) - } else { - tokens = MOST_ACCURATE_DURATION_FORMAT.split(':').reverse() + for (const [fmtRegExp, formatFuncs] of DURATION_REGEXPS) { + const match = inputValue.match(fmtRegExp) + if (match) { + const formatFunc = formatFuncs[format] || formatFuncs.default + return formatFunc(...match.slice(1)) } } - - try { - return tokens.reduce((acc, token, index) => { - if (index >= parts.length) { - return acc - } - const part = parts[index] - const parseFunc = DURATION_TOKENS[token].parse - const number = parseFunc(part) - - if (isNaN(number) || number < 0) { - throw new Error( - `Invalid duration format: ${inputValue} does not match ${format}` - ) - } - - const multiplier = DURATION_TOKENS[token].multiplier - return acc + number * multiplier - }, 0) - } catch (e) { - return null - } + return null } /** * It formats the given duration value using the given format. */ -export const formatDuration = (value, format) => { +export const formatDurationValue = (value, format) => { if (value === null || value === undefined || value === '') { return '' } - const hours = Math.floor(value / 3600) - const mins = Math.floor((value - hours * 3600) / 60) - const secs = value - hours * 3600 - mins * 60 + const days = Math.floor(value / 86400) + const hours = Math.floor((value % 86400) / 3600) + const mins = Math.floor((value % 3600) / 60) + const secs = value % 60 const formatFunc = DURATION_FORMATS.get(format).toString - return formatFunc(hours, mins, secs) + return formatFunc(days, hours, mins, secs) } diff --git a/web-frontend/test/unit/database/fieldTypes/durationFieldTypeGetSort.spec.js b/web-frontend/test/unit/database/fieldTypes/durationFieldTypeGetSort.spec.js index 9c710aa09..d135a44dd 100644 --- a/web-frontend/test/unit/database/fieldTypes/durationFieldTypeGetSort.spec.js +++ b/web-frontend/test/unit/database/fieldTypes/durationFieldTypeGetSort.spec.js @@ -12,7 +12,7 @@ const tableRows = [ { id: 2, order: '2.00000000000000000000', - field: '0:1:0', + field: 60, }, { id: 3, @@ -32,12 +32,12 @@ const tableRows = [ { id: 6, order: '5.00000000000000000000', - field: '2:0:0.123', + field: 7200.123, }, { id: 7, order: '6.00000000000000000000', - field: '1.12' + field: 1.12 } ] @@ -71,11 +71,11 @@ describe('LastModifiedByFieldType.getSort()', () => { ) const expected = [ null, - '1.12', - '0:1:0', + 1.12, + 60, 120, 3600, - '2:0:0.123', + 7200.123, 86400, ] @@ -89,11 +89,11 @@ describe('LastModifiedByFieldType.getSort()', () => { const expectedReversed = [ 86400, - '2:0:0.123', + 7200.123, 3600, 120, - '0:1:0', - '1.12', + 60, + 1.12, null, ] diff --git a/web-frontend/test/unit/database/utils/duration.spec.js b/web-frontend/test/unit/database/utils/duration.spec.js new file mode 100644 index 000000000..2d6623c61 --- /dev/null +++ b/web-frontend/test/unit/database/utils/duration.spec.js @@ -0,0 +1,507 @@ +import { + parseDurationValue, + formatDurationValue, + roundDurationValueToFormat, +} from '@baserow/modules/database/utils/duration' + +const SECS_IN_MIN = 60 +const SECS_IN_HOUR = 60 * 60 + +const VALID_DURATION_VALUES = { + 'd h': [ + ['1d 2h', 26 * SECS_IN_HOUR], + ['2 1h', 49 * SECS_IN_HOUR], + ['2 2', 50 * SECS_IN_HOUR], + ['1 8', 32 * SECS_IN_HOUR], + ['1d0h', 24 * SECS_IN_HOUR], + ['1d25', 49 * SECS_IN_HOUR], + ['5h', 5 * SECS_IN_HOUR], + ['2d', 48 * SECS_IN_HOUR], + ['10', 10 * SECS_IN_HOUR], + ['3600.0', 1 * SECS_IN_HOUR], + ['60:0.0', 1 * SECS_IN_HOUR], + ['0', 0], + ], + 'd h:mm': [ + ['1d 2:30', 26 * SECS_IN_HOUR + 30 * 60], + ['2 1:30', 49 * SECS_IN_HOUR + 30 * 60], + ['2 1:61', 50 * SECS_IN_HOUR + 1 * 60], + ['1 8:30', 32 * SECS_IN_HOUR + 30 * 60], + ['1d0:61', 25 * SECS_IN_HOUR + 1 * 60], + ['1d0:30', 24 * SECS_IN_HOUR + 30 * 60], + ['1d1:30', 25 * SECS_IN_HOUR + 30 * 60], + ['5:30', 5 * SECS_IN_HOUR + 30 * 60], + ['10:30', 10 * SECS_IN_HOUR + 30 * 60], + ['0:30', 30 * 60], + ['0:0', 0], + ], + 'd h:mm:ss': [ + ['1d 2:30:45', 26 * SECS_IN_HOUR + 30 * 60 + 45], + ['2 2:30:45', 50 * SECS_IN_HOUR + 30 * 60 + 45], + ['1 8:30:45', 32 * SECS_IN_HOUR + 30 * 60 + 45], + ['1d0:30:45', 24 * SECS_IN_HOUR + 30 * 60 + 45], + ['1d1:30:45', 25 * SECS_IN_HOUR + 30 * 60 + 45], + ['1d2:30:61', 26 * SECS_IN_HOUR + 31 * 60 + 1], + ['5:30:45', 5 * SECS_IN_HOUR + 30 * 60 + 45], + ['10:30:45', 10 * SECS_IN_HOUR + 30 * 60 + 45], + ['0:30:45', 30 * 60 + 45], + ['0:0:45', 45], + ['0:0:0', 0], + ], + 'h:mm': [ + ['2:30', 2 * SECS_IN_HOUR + 30 * 60], + ['1:30', 1 * SECS_IN_HOUR + 30 * 60], + ['2:30', 2 * SECS_IN_HOUR + 30 * 60], + ['8:30', 8 * SECS_IN_HOUR + 30 * 60], + ['0:30', 30 * 60], + ['0:61', 61 * 60], + ['0:0', 0], + ['0', 0], + ], + 'h:mm:ss': [ + ['2:30:45', 2 * SECS_IN_HOUR + 30 * 60 + 45], + ['1:30:45', 1 * SECS_IN_HOUR + 30 * 60 + 45], + ['2:30:45', 2 * SECS_IN_HOUR + 30 * 60 + 45], + ['8:30:45', 8 * SECS_IN_HOUR + 30 * 60 + 45], + ['0:30:45', 30 * 60 + 45], + ['0:0:45', 45], + ['0:0:61', 61], + ['0:0:0', 0], + ], + 'h:mm:ss.s': [ + ['2:30:45.1', 2 * SECS_IN_HOUR + 30 * 60 + 45.1], + ['1:30:45.2', 1 * SECS_IN_HOUR + 30 * 60 + 45.2], + ['2:30:45.3', 2 * SECS_IN_HOUR + 30 * 60 + 45.3], + ['8:30:45.9', 8 * SECS_IN_HOUR + 30 * 60 + 45.9], + ['0:30:45.5', 30 * 60 + 45.5], + ['0:0:45.0', 45], + ['0:0:59.9', 59.9], + ['0:0:0.0', 0], + ], + 'h:mm:ss.ss': [ + ['2:30:45.11', 2 * SECS_IN_HOUR + 30 * 60 + 45.11], + ['1:30:45.22', 1 * SECS_IN_HOUR + 30 * 60 + 45.22], + ['2:30:45.33', 2 * SECS_IN_HOUR + 30 * 60 + 45.33], + ['8:30:45.46', 8 * SECS_IN_HOUR + 30 * 60 + 45.46], + ['0:30:45.50', 30 * 60 + 45.5], + ['0:0:45.00', 45], + ['0:0:59.99', 59.99], + ['0', 0], + ], + 'h:mm:ss.sss': [ + ['2:30:45.111', 2 * SECS_IN_HOUR + 30 * 60 + 45.111], + ['1:30:45.222', 1 * SECS_IN_HOUR + 30 * 60 + 45.222], + ['2:30:45.333', 2 * SECS_IN_HOUR + 30 * 60 + 45.333], + ['8:30:45.456', 8 * SECS_IN_HOUR + 30 * 60 + 45.456], + ['0:30:45.500', 30 * 60 + 45.5], + ['0:0:45.000', 45], + ['0:0:59.999', 59.999], + ['0', 0], + ], +} + +const INVALID_DURATION_VALUES = [ + 'd', + 'h', + '1d 2h 3m', + '1d 2h 3s', + '1day', + '2 days', + '1 hour', + '8 hours', + '1:2:3:4', + 'aaaaa', + '1dd', + '2hh', + '1h2m', +] + +const DURATION_FORMATTED_VALUES = { + 'd h': [ + [0, '0d 0h'], + [1 * SECS_IN_HOUR, '0d 1h'], + [24 * SECS_IN_HOUR, '1d 0h'], + [25 * SECS_IN_HOUR, '1d 1h'], + [49 * SECS_IN_HOUR, '2d 1h'], + [50 * SECS_IN_HOUR, '2d 2h'], + [32 * SECS_IN_HOUR, '1d 8h'], + [5 * SECS_IN_HOUR, '0d 5h'], + ], + 'd h:mm': [ + [0, '0d 0:00'], + [1 * SECS_IN_HOUR, '0d 1:00'], + [24 * SECS_IN_HOUR, '1d 0:00'], + [2 * SECS_IN_HOUR, '0d 2:00'], + [49 * SECS_IN_HOUR, '2d 1:00'], + [50 * SECS_IN_HOUR, '2d 2:00'], + [32 * SECS_IN_HOUR, '1d 8:00'], + [5 * SECS_IN_HOUR, '0d 5:00'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '0d 5:30'], + ], + 'd h:mm:ss': [ + [0, '0d 0:00:00'], + [1 * SECS_IN_HOUR, '0d 1:00:00'], + [24 * SECS_IN_HOUR, '1d 0:00:00'], + [25 * SECS_IN_HOUR, '1d 1:00:00'], + [49 * SECS_IN_HOUR, '2d 1:00:00'], + [50 * SECS_IN_HOUR, '2d 2:00:00'], + [32 * SECS_IN_HOUR, '1d 8:00:00'], + [5 * SECS_IN_HOUR, '0d 5:00:00'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '0d 5:30:00'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5, '0d 5:05:05'], + ], + 'h:mm': [ + [0, '0:00'], + [1 * SECS_IN_MIN, '0:01'], + [61 * SECS_IN_MIN, '1:01'], + [24 * SECS_IN_HOUR, '24:00'], + [25 * SECS_IN_HOUR, '25:00'], + [49 * SECS_IN_HOUR, '49:00'], + [50 * SECS_IN_HOUR, '50:00'], + [32 * SECS_IN_HOUR, '32:00'], + [5 * SECS_IN_HOUR, '5:00'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '5:30'], + ], + 'h:mm:ss': [ + [0, '0:00:00'], + [1, '0:00:01'], + [61, '0:01:01'], + [1 * SECS_IN_MIN, '0:01:00'], + [61 * SECS_IN_MIN, '1:01:00'], + [24 * SECS_IN_HOUR, '24:00:00'], + [25 * SECS_IN_HOUR, '25:00:00'], + [49 * SECS_IN_HOUR, '49:00:00'], + [50 * SECS_IN_HOUR, '50:00:00'], + [32 * SECS_IN_HOUR, '32:00:00'], + [5 * SECS_IN_HOUR, '5:00:00'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '5:30:00'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5, '5:05:05'], + ], + 'h:mm:ss.s': [ + [0, '0:00:00.0'], + [1.1, '0:00:01.1'], + [61.9, '0:01:01.9'], + [1 * SECS_IN_MIN + 1.1, '0:01:01.1'], + [61 * SECS_IN_MIN + 2.2, '1:01:02.2'], + [24 * SECS_IN_HOUR + 3.3, '24:00:03.3'], + [25 * SECS_IN_HOUR + 9.9, '25:00:09.9'], + [49 * SECS_IN_HOUR, '49:00:00.0'], + [50 * SECS_IN_HOUR, '50:00:00.0'], + [32 * SECS_IN_HOUR, '32:00:00.0'], + [5 * SECS_IN_HOUR, '5:00:00.0'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '5:30:00.0'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5, '5:05:05.0'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5.1, '5:05:05.1'], + ], + 'h:mm:ss.ss': [ + [0, '0:00:00.00'], + [1.11, '0:00:01.11'], + [61.22, '0:01:01.22'], + [1 * SECS_IN_MIN + 1.11, '0:01:01.11'], + [61 * SECS_IN_MIN + 2.22, '1:01:02.22'], + [24 * SECS_IN_HOUR + 3.33, '24:00:03.33'], + [25 * SECS_IN_HOUR + 9.99, '25:00:09.99'], + [49 * SECS_IN_HOUR, '49:00:00.00'], + [50 * SECS_IN_HOUR, '50:00:00.00'], + [32 * SECS_IN_HOUR, '32:00:00.00'], + [5 * SECS_IN_HOUR, '5:00:00.00'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '5:30:00.00'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5, '5:05:05.00'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5.1, '5:05:05.10'], + ], + 'h:mm:ss.sss': [ + [0, '0:00:00.000'], + [1.111, '0:00:01.111'], + [61.222, '0:01:01.222'], + [1 * SECS_IN_MIN + 1.111, '0:01:01.111'], + [61 * SECS_IN_MIN + 2.222, '1:01:02.222'], + [24 * SECS_IN_HOUR + 3.333, '24:00:03.333'], + [25 * SECS_IN_HOUR + 9.999, '25:00:09.999'], + [49 * SECS_IN_HOUR, '49:00:00.000'], + [50 * SECS_IN_HOUR, '50:00:00.000'], + [32 * SECS_IN_HOUR, '32:00:00.000'], + [5 * SECS_IN_HOUR, '5:00:00.000'], + [5 * SECS_IN_HOUR + 30 * SECS_IN_MIN, '5:30:00.000'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5, '5:05:05.000'], + [5 * SECS_IN_HOUR + 5 * SECS_IN_MIN + 5.1, '5:05:05.100'], + ], +} + +const DURATION_ROUNDING_VALUES = { + 'd h': [ + [null, null], + [0, 0], + [1.9, 0], + [30, 0], + [60, 0], + [29 * SECS_IN_MIN, 0], + [30 * SECS_IN_MIN, 1 * SECS_IN_HOUR], + [59 * SECS_IN_MIN, 1 * SECS_IN_HOUR], + [1 * SECS_IN_HOUR + 29 * SECS_IN_MIN, 1 * SECS_IN_HOUR], + [1 * SECS_IN_HOUR + 30 * SECS_IN_MIN, 2 * SECS_IN_HOUR], + ], + 'd h:mm': [ + [null, null], + [0, 0], + [1, 0], + [29, 0], + [30, 1 * SECS_IN_MIN], + [59, 1 * SECS_IN_MIN], + [29 * SECS_IN_MIN + 29.9, 29 * SECS_IN_MIN], + [29 * SECS_IN_MIN + 30, 30 * SECS_IN_MIN], + ], + 'd h:mm:ss': [ + [null, null], + [0, 0], + [0.499, 0], + [0.5, 1], + [0.9, 1], + [1, 1], + [1.5, 2], + ], + 'h:mm': [ + [null, null], + [0, 0], + [1, 0], + [29, 0], + [30, 1 * SECS_IN_MIN], + [59, 1 * SECS_IN_MIN], + [29 * SECS_IN_MIN + 29.9, 29 * SECS_IN_MIN], + [29 * SECS_IN_MIN + 30, 30 * SECS_IN_MIN], + ], + 'h:mm:ss': [ + [null, null], + [0, 0], + [0.499, 0], + [0.5, 1], + [0.9, 1], + [1, 1], + [1.5, 2], + ], + 'h:mm:ss.s': [ + [null, null], + [0, 0], + [0.449, 0.4], + [0.45, 0.5], + [0.99, 1], + [1.12, 1.1], + [1.9, 1.9], + [1.99, 2], + ], + 'h:mm:ss.ss': [ + [null, null], + [0, 0], + [0.49, 0.49], + [0.494, 0.49], + [0.499, 0.5], + [0.999, 1], + [1.123, 1.12], + [1.99, 1.99], + [1.999999, 2], + ], + 'h:mm:ss.sss': [ + [null, null], + [0, 0], + [0.4994, 0.499], + [0.4995, 0.5], + [0.9991, 0.999], + [0.9995, 1], + [1.123, 1.123], + [1.999, 1.999], + [1.999999, 2], + ], +} + +describe('Duration field type utilities', () => { + describe('parse format: d h', () => { + for (const [input, expected] of VALID_DURATION_VALUES['d h']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'd h')).toBe(expected) + }) + } + }) + describe('parse format: d h:mm', () => { + for (const [input, expected] of VALID_DURATION_VALUES['d h:mm']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'd h:mm')).toBe(expected) + }) + } + }) + describe('parse format: d h:mm:ss', () => { + for (const [input, expected] of VALID_DURATION_VALUES['d h:mm:ss']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'd h:mm:ss')).toBe(expected) + }) + } + }) + describe('parse format: h:mm', () => { + for (const [input, expected] of VALID_DURATION_VALUES['h:mm']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'h:mm')).toBe(expected) + }) + } + }) + describe('parse format: h:mm:ss', () => { + for (const [input, expected] of VALID_DURATION_VALUES['h:mm:ss']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'h:mm:ss')).toBe(expected) + }) + } + }) + describe('parse format: h:mm:ss.s', () => { + for (const [input, expected] of VALID_DURATION_VALUES['h:mm:ss.s']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'h:mm:ss.s')).toBe(expected) + }) + } + }) + describe('parse format: h:mm:ss.ss', () => { + for (const [input, expected] of VALID_DURATION_VALUES['h:mm:ss.ss']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'h:mm:ss.ss')).toBe(expected) + }) + } + }) + describe('parse format: h:mm:ss.sss', () => { + for (const [input, expected] of VALID_DURATION_VALUES['h:mm:ss.sss']) { + test(`"${input}" should be parsed to ${expected}`, () => { + expect(parseDurationValue(input, 'h:mm:ss.sss')).toBe(expected) + }) + } + }) + describe('parse invalid values', () => { + for (const input of INVALID_DURATION_VALUES) { + test(`"${input}" should be parsed to null`, () => { + expect(parseDurationValue(input, 'h:mm:ss.sss')).toBe(null) + }) + } + }) + + describe('format format: d h', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['d h']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'd h')).toBe(expected) + }) + } + }) + + describe('format format: d h:mm', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['d h:mm']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'd h:mm')).toBe(expected) + }) + } + }) + + describe('format format: d h:mm:ss', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['d h:mm:ss']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'd h:mm:ss')).toBe(expected) + }) + } + }) + + describe('format format: h:mm', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['h:mm']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'h:mm')).toBe(expected) + }) + } + }) + + describe('format format: h:mm:ss', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['h:mm:ss']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'h:mm:ss')).toBe(expected) + }) + } + }) + + describe('format format: h:mm:ss.s', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['h:mm:ss.s']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'h:mm:ss.s')).toBe(expected) + }) + } + }) + + describe('format format: h:mm:ss.ss', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['h:mm:ss.ss']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'h:mm:ss.ss')).toBe(expected) + }) + } + }) + + describe('format format: h:mm:ss.sss', () => { + for (const [input, expected] of DURATION_FORMATTED_VALUES['h:mm:ss.sss']) { + test(`"${input}" should be formatted to ${expected}`, () => { + expect(formatDurationValue(input, 'h:mm:ss.sss')).toBe(expected) + }) + } + }) + + describe('round value: d h', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['d h']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'd h')).toBe(expected) + }) + } + }) + + describe('round value: d h:mm', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['d h:mm']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'd h:mm')).toBe(expected) + }) + } + }) + + describe('round value: d h:mm:ss', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['d h:mm:ss']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'd h:mm:ss')).toBe(expected) + }) + } + }) + + describe('round value: h:mm', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['h:mm']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'h:mm')).toBe(expected) + }) + } + }) + + describe('round value: h:mm:ss', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['h:mm:ss']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'h:mm:ss')).toBe(expected) + }) + } + }) + + describe('round value: h:mm:ss.s', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['h:mm:ss.s']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'h:mm:ss.s')).toBe(expected) + }) + } + }) + + describe('round value: h:mm:ss.ss', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['h:mm:ss.ss']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'h:mm:ss.ss')).toBe(expected) + }) + } + }) + + describe('round value: h:mm:ss.sss', () => { + for (const [input, expected] of DURATION_ROUNDING_VALUES['h:mm:ss.sss']) { + test(`"${input}" should be rounded to ${expected}`, () => { + expect(roundDurationValueToFormat(input, 'h:mm:ss.sss')).toBe(expected) + }) + } + }) +}) diff --git a/web-frontend/test/unit/database/utils/row.spec.js b/web-frontend/test/unit/database/utils/row.spec.js index b83890c29..39dd721ee 100644 --- a/web-frontend/test/unit/database/utils/row.spec.js +++ b/web-frontend/test/unit/database/utils/row.spec.js @@ -5,7 +5,7 @@ import { } from '@baserow/modules/database/utils/row' import { TestApp } from '@baserow/test/helpers/testApp' -describe('Row untilities', () => { +describe('Row utilities', () => { let testApp = null let store = null