archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-15 01:28:30 +00:00
Code Issues Projects Releases Wiki Activity

Fix divide function sometimes resulting in errors and fix up and make available round and int functions

Browse source
This commit is contained in:
Nigel Gott 2022-08-29 17:11:38 +00:00
parent 9dfb3c2330
commit 298faf1760
7 changed files with 222 additions and 56 deletions
backend
src/baserow/contrib/database/formula/ast
function_defs.py
tests/baserow/contrib/database/formula
test_baserow_formula_results.py
changelog.md
web-frontend
locales
en.json
modules/database
formula
functions.js
plugin.js
test/unit/database/formula
formulaFunctions.spec.js

156
backend/src/baserow/contrib/database/formula/ast/function_defs.py
View file

@ -51,8 +51,10 @@ from baserow.contrib.database.formula.ast.function import (
aggregate_wrapper,
)
from baserow.contrib.database.formula.ast.tree import (
BaserowDecimalLiteral,
BaserowExpression,
BaserowFunctionCall,
BaserowIntegerLiteral,
)
from baserow.contrib.database.formula.expression_generator.django_expressions import (
AndExpr,
@ -121,6 +123,7 @@ def register_formula_functions(registry):
registry.register(BaserowLeast())
registry.register(BaserowRound())
registry.register(BaserowInt())
registry.register(BaserowTrunc())
# Boolean functions
registry.register(BaserowIf())
registry.register(BaserowEqual())
@ -428,18 +431,39 @@ class BaserowRound(TwoArgumentBaserowFunction):
arg1: BaserowExpression[BaserowFormulaNumberType],
arg2: BaserowExpression[BaserowFormulaNumberType],
) -> BaserowExpression[BaserowFormulaType]:
if isinstance(arg2, BaserowIntegerLiteral):
guessed_number_decimal_places = arg2.literal
elif isinstance(arg2, BaserowDecimalLiteral):
guessed_number_decimal_places = int(arg2.literal)
else:
guessed_number_decimal_places = NUMBER_MAX_DECIMAL_PLACES
return func_call.with_valid_type(
BaserowFormulaNumberType(
number_decimal_places=BaserowFormulaNumberType.MAX_DIGITS
number_decimal_places=min(
max(guessed_number_decimal_places, 0), NUMBER_MAX_DECIMAL_PLACES
)
)
)
def to_django_expression(self, arg1: Expression, arg2: Expression) -> Expression:
return Func(arg1, arg2, function="round", output_field=arg1.output_field)
return handle_arg_being_nan(
arg_to_check_if_nan=arg2,
when_nan=Value(Decimal("NaN")),
when_not_nan=(
Func(
arg1,
# The round function requires an integer input.
trunc_numeric_to_int(arg2),
function="round",
output_field=arg1.output_field,
)
),
)
class BaserowInt(OneArgumentBaserowFunction):
type = "int"
class BaserowTrunc(OneArgumentBaserowFunction):
type = "trunc"
arg_type = [BaserowFormulaNumberType]
def type_function(
@ -452,7 +476,60 @@ class BaserowInt(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Cast(arg, output_field=fields.IntegerField())
# If we get given a NaN trunc will crash, instead just return NaN if given NaN.
return handle_arg_being_nan(
arg_to_check_if_nan=arg,
when_nan=Cast(
Value("NaN"),
output_field=int_like_numeric_output_field(),
),
when_not_nan=Func(
arg, function="trunc", output_field=int_like_numeric_output_field()
),
)
def int_like_numeric_output_field() -> fields.DecimalField:
return fields.DecimalField(
max_digits=BaserowFormulaNumberType.MAX_DIGITS, decimal_places=0
)
class BaserowInt(BaserowTrunc):
"""
Kept for backwards compatability as was introduced in v3 of formula language but
renamed to trunc in v4.
"""
type = "int"
def trunc_numeric_to_int(expr: Expression) -> Expression:
return Cast(
Func(expr, function="trunc", output_field=expr.output_field),
output_field=fields.IntegerField(),
)
def handle_arg_being_nan(
arg_to_check_if_nan: Expression,
when_nan: Expression,
when_not_nan: Expression,
) -> Expression:
return Case(
When(
condition=(
EqualsExpr(
arg_to_check_if_nan,
Value(Decimal("Nan")),
output_field=fields.BooleanField(),
)
),
then=when_nan,
),
default=when_not_nan,
output_field=when_not_nan.output_field,
)
class BaserowDivide(TwoArgumentBaserowFunction):
@ -478,6 +555,10 @@ class BaserowDivide(TwoArgumentBaserowFunction):
# Prevent divide by zero's by swapping 0 for NaN causing the entire expression
# to evaluate to NaN. The front-end then treats NaN values as a per cell error
# to display to the user.
max_dp_output = fields.DecimalField(
max_digits=BaserowFormulaNumberType.MAX_DIGITS,
decimal_places=NUMBER_MAX_DECIMAL_PLACES,
)
return ExpressionWrapper(
arg1
/ Case(
@ -486,8 +567,9 @@ class BaserowDivide(TwoArgumentBaserowFunction):
then=Value(Decimal("NaN")),
),
default=arg2,
output_field=max_dp_output,
),
output_field=fields.DecimalField(decimal_places=NUMBER_MAX_DECIMAL_PLACES),
output_field=max_dp_output,
)
@ -580,7 +662,7 @@ class BaserowIf(ThreeArgumentBaserowFunction):
return Case(
When(condition=arg1, then=arg2),
default=arg3,
output_field=arg1.output_field,
output_field=arg2.output_field,
)
@ -594,14 +676,14 @@ class BaserowToNumber(OneArgumentBaserowFunction):
arg: BaserowExpression[BaserowFormulaValidType],
) -> BaserowExpression[BaserowFormulaType]:
return func_call.with_valid_type(
BaserowFormulaNumberType(number_decimal_places=5)
BaserowFormulaNumberType(number_decimal_places=NUMBER_MAX_DECIMAL_PLACES)
)
def to_django_expression(self, arg: Expression) -> Expression:
return Func(
arg,
function="try_cast_to_numeric",
output_field=fields.DecimalField(decimal_places=0),
output_field=int_like_numeric_output_field(),
)
@ -805,7 +887,7 @@ class BaserowDay(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Extract(arg, "day", output_field=fields.DecimalField(decimal_places=0))
return Extract(arg, "day", output_field=int_like_numeric_output_field())
class BaserowMonth(OneArgumentBaserowFunction):
@ -822,7 +904,7 @@ class BaserowMonth(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Extract(arg, "month", output_field=fields.DecimalField(decimal_places=0))
return Extract(arg, "month", output_field=int_like_numeric_output_field())
class BaserowDateDiff(ThreeArgumentBaserowFunction):
@ -851,7 +933,7 @@ class BaserowDateDiff(ThreeArgumentBaserowFunction):
arg2,
arg3,
function="date_diff",
output_field=fields.DecimalField(decimal_places=0),
output_field=int_like_numeric_output_field(),
)
@ -943,7 +1025,7 @@ class BaserowSearch(TwoArgumentBaserowFunction):
)
def to_django_expression(self, arg1: Expression, arg2: Expression) -> Expression:
return StrIndex(arg1, arg2, output_field=fields.DecimalField(decimal_places=0))
return StrIndex(arg1, arg2, output_field=int_like_numeric_output_field())
class BaserowContains(TwoArgumentBaserowFunction):
@ -988,9 +1070,7 @@ class BaserowRowId(ZeroArgumentBaserowFunction):
) -> WrappedExpressionWithMetadata:
if model_instance is None:
return WrappedExpressionWithMetadata(
ExpressionWrapper(
F("id"), output_field=fields.DecimalField(decimal_places=0)
)
ExpressionWrapper(F("id"), output_field=int_like_numeric_output_field())
)
else:
# noinspection PyUnresolvedReferences
@ -1017,7 +1097,7 @@ class BaserowLength(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Length(arg, output_field=fields.DecimalField(decimal_places=0))
return Length(arg, output_field=int_like_numeric_output_field())
class BaserowReverse(OneArgumentBaserowFunction):
@ -1171,7 +1251,7 @@ class BaserowCount(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Count(arg, output_field=fields.DecimalField(decimal_places=0))
return Count(arg, output_field=int_like_numeric_output_field())
class BaserowFilter(TwoArgumentBaserowFunction):
@ -1466,12 +1546,20 @@ class BaserowLeft(TwoArgumentBaserowFunction):
arg1: BaserowExpression[BaserowFormulaValidType],
arg2: BaserowExpression[BaserowFormulaNumberType],
) -> BaserowExpression[BaserowFormulaType]:
return func_call.with_args([arg1, BaserowInt()(arg2)]).with_valid_type(
arg1.expression_type
)
return func_call.with_valid_type(arg1.expression_type)
def to_django_expression(self, arg1: Expression, arg2: Expression) -> Expression:
return Left(arg1, arg2, output_field=fields.TextField())
return handle_arg_being_nan(
arg_to_check_if_nan=arg2,
when_nan=Value(None),
when_not_nan=(
Left(
arg1,
trunc_numeric_to_int(arg2),
output_field=fields.TextField(),
)
),
)
class BaserowRight(TwoArgumentBaserowFunction):
@ -1485,12 +1573,20 @@ class BaserowRight(TwoArgumentBaserowFunction):
arg1: BaserowExpression[BaserowFormulaValidType],
arg2: BaserowExpression[BaserowFormulaNumberType],
) -> BaserowExpression[BaserowFormulaType]:
return func_call.with_args([arg1, BaserowInt()(arg2)]).with_valid_type(
arg1.expression_type
)
return func_call.with_valid_type(arg1.expression_type)
def to_django_expression(self, arg1: Expression, arg2: Expression) -> Expression:
return Right(arg1, arg2, output_field=fields.TextField())
return handle_arg_being_nan(
arg_to_check_if_nan=arg2,
when_nan=Value(None),
when_not_nan=(
Right(
arg1,
trunc_numeric_to_int(arg2),
output_field=fields.TextField(),
)
),
)
class BaserowRegexReplace(ThreeArgumentBaserowFunction):
@ -1554,7 +1650,7 @@ class BaserowYear(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Extract(arg, "year", output_field=fields.DecimalField(decimal_places=0))
return Extract(arg, "year", output_field=int_like_numeric_output_field())
class BaserowSecond(OneArgumentBaserowFunction):
@ -1571,9 +1667,7 @@ class BaserowSecond(OneArgumentBaserowFunction):
)
def to_django_expression(self, arg: Expression) -> Expression:
return Extract(
arg, "second", output_field=fields.DecimalField(decimal_places=0)
)
return Extract(arg, "second", output_field=int_like_numeric_output_field())
class BaserowBcToNull(OneArgumentBaserowFunction):
@ -1590,7 +1684,7 @@ class BaserowBcToNull(OneArgumentBaserowFunction):
def to_django_expression(self, arg: Expression) -> Expression:
expr_to_get_year = Extract(
arg, "year", output_field=fields.DecimalField(decimal_places=0)
arg, "year", output_field=int_like_numeric_output_field()
)
return Case(
When(

75
backend/tests/baserow/contrib/database/formula/test_baserow_formula_results.py
View file

@ -1,14 +1,12 @@
import datetime
import sys
import traceback
from datetime import timedelta
from decimal import Decimal
from re import search
from typing import Any, List, Optional
from django.conf import settings
from django.urls import reverse
from django.utils.duration import duration_string
import pytest
from rest_framework.status import HTTP_200_OK
@ -38,6 +36,7 @@ from baserow.contrib.database.formula.types.formula_type import (
)
from baserow.contrib.database.formula.types.type_checker import MustBeManyExprChecker
from baserow.contrib.database.rows.handler import RowHandler
from baserow.core.trash.handler import TrashHandler
VALID_FORMULA_TESTS = [
("'test'", "test"),
@ -48,7 +47,7 @@ VALID_FORMULA_TESTS = [
("CONCAT('test', ' ', 'works')", "test works"),
("CONCAT('test', ' ', UPPER('works'))", "test WORKS"),
(
"UPPER(" * 100 + "'test'" + ")" * 100,
"UPPER(" * 50 + "'test'" + ")" * 50,
"TEST",
),
(
@ -81,9 +80,9 @@ VALID_FORMULA_TESTS = [
"9" * 100 + "+1",
"NaN",
),
("tonumber('1')", "1.00000"),
("tonumber('1')", "1.0000000000"),
("tonumber('a')", "NaN"),
("tonumber('-12.12345')", "-12.12345"),
("tonumber('-12.12345')", "-12.1234500000"),
("1.2 * 2", "2.4"),
("isblank(1)", False),
("isblank('')", True),
@ -130,7 +129,7 @@ VALID_FORMULA_TESTS = [
("or(false, true)", True),
("or(true, true)", True),
("'a' + 'b'", "ab"),
("date_interval('1 year')", duration_string(timedelta(days=365))),
("date_interval('1 year')", "1 year"),
("date_interval('1 year') > date_interval('1 day')", True),
("date_interval('1 invalid')", None),
("todate('20200101', 'YYYYMMDD') + date_interval('1 year')", "2021-01-01"),
@ -146,14 +145,8 @@ VALID_FORMULA_TESTS = [
")",
"6",
),
(
"todate('20200101', 'YYYYMMDD') - todate('20210101', 'YYYYMMDD')",
duration_string(-timedelta(days=366)),
),
(
"date_interval('1 year') - date_interval('1 day')",
duration_string(timedelta(days=364)),
),
("todate('20200101', 'YYYYMMDD') - todate('20210101', 'YYYYMMDD')", "-366 days"),
("date_interval('1 year') - date_interval('1 day')", "1 year -1 days"),
("replace('test test', 'test', 'a')", "a a"),
("search('test test', 'test')", "1"),
("search('a', 'test')", "0"),
@ -172,13 +165,38 @@ VALID_FORMULA_TESTS = [
("contains('a', 'x')", False),
("left('a', 2)", "a"),
("left('abc', 2)", "ab"),
("left('abcde', -2)", "abc"),
("left('abcde', 2)", "ab"),
("left('abc', 2/0)", None),
("right('a', 2)", "a"),
("right('abc', 2)", "bc"),
("right('abcde', -2)", "cde"),
("right('abcde', 2)", "de"),
("right('abc', 2/0)", None),
("when_empty(1, 2)", "1"),
("round(1.12345, 0)", "1"),
("round(1.12345, 4)", "1.1234"),
("round(1.12345, 100)", "1.12345"),
("round(1.12345, 4)", "1.1235"),
("round(1.12345, 100)", "1.1234500000"),
("round(1234.5678, -2)", "1200"),
("round(1234.5678, -2.99999)", "1200"),
("round(1234.5678, -2.00001)", "1200"),
("round(1234.5678, 1/0)", "NaN"),
("round(1234.5678, tonumber('invalid'))", "NaN"),
("round(1/0, 1/0)", "NaN"),
("round(1/0, 2)", "NaN"),
("round(tonumber('invalid'), 2)", "NaN"),
("trunc(1.1234)", "1"),
("trunc(1.56)", "1"),
("trunc(-1.56)", "-1"),
("trunc(1/0)", "NaN"),
("trunc(tonumber('invalid'))", "NaN"),
("int(1.1234)", "1"),
("int(1.56)", "1"),
("int(-1.56)", "-1"),
("int(1/0)", "NaN"),
("int(tonumber('invalid'))", "NaN"),
("1/2/4", "0.1250000000"),
("divide(1, if(true,1,1))", "1.0000000000"),
]
@ -605,6 +623,31 @@ def construct_some_literal_args(formula_func):
return fake_args
@pytest.mark.django_db
def test_valid_formulas(data_fixture, api_client):
user, token = data_fixture.create_user_and_token()
table = data_fixture.create_database_table(user=user)
table.get_model().objects.create()
for test_formula, expected_value in VALID_FORMULA_TESTS:
formula_field = FieldHandler().create_field(
user, table, "formula", formula=test_formula, name="test formula"
)
response = api_client.get(
reverse("api:database:rows:list", kwargs={"table_id": table.id}),
{},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
response_json = response.json()
assert response_json["count"] == 1
actual_value = response_json["results"][0][formula_field.db_column]
assert actual_value == expected_value, (
f"Expected the formula: {test_formula} to be {expected_value} but instead "
f"it was {actual_value}"
)
TrashHandler.permanently_delete(formula_field)
@pytest.mark.parametrize("test_input,error,detail", INVALID_FORMULA_TESTS)
@pytest.mark.django_db
def test_invalid_formulas(test_input, error, detail, data_fixture, api_client):

2
changelog.md
View file

@ -103,6 +103,8 @@ For example:
* Fix backspace and delete keys breaking after selecting a formula text cell. [#1085](https://gitlab.com/bramw/baserow/-/issues/1085)
* Fixed problem when new webhooks would be sent twice with both old and new payload.
* Fixed problem causing kanban view duplication to fail silently. [#1109](https://gitlab.com/bramw/baserow/-/issues/1109)
* Display round and trunc functions in the formula edit modal, rename int to trunc and make these functions handle weird inputs better. [#1095](https://gitlab.com/bramw/baserow/-/issues/1095)
* Fix some rare errors when combining the if and divide formula functions. [#1086](https://gitlab.com/bramw/baserow/-/issues/1086)
### Breaking Changes

2
web-frontend/locales/en.json
View file

@ -346,7 +346,7 @@
"sumDescription": "Sums all of the values and returns the result.",
"filterDescription": "Filters down an expression involving a lookup/link field reference or a lookup function call.",
"roundDescription": "Returns first argument rounded to the number of digits specified by the second argument.",
"intDescription": "Returns only the first argument converted into an integer by truncating any decimal places."
"truncDescription": "Returns only the first argument converted into an integer by truncating any decimal places."
},
"functionnalGridViewFieldLinkRow": {
"unnamed": "unnamed row {value}"

37
web-frontend/modules/database/formula/functions.js
View file

@ -977,7 +977,11 @@ export class BaserowLeft extends BaserowFunctionDefinition {
}
getExamples() {
return ['left("abcd", 2) = "ab"']
return [
'left("abcde", 2) = "ab"',
'left("abcde", -2) = "abc"',
'when_empty(left("abcd", 1/0), "error") = "error"',
]
}
getFormulaType() {
@ -1000,7 +1004,11 @@ export class BaserowRight extends BaserowFunctionDefinition {
}
getExamples() {
return ['right("abcd", 2) = "cd"']
return [
'right("abcde", 2) = "de"',
'right("abcde", -2) = "cde"',
'when_empty(right("abcd", 1/0), "error") = "error"',
]
}
getFormulaType() {
@ -1091,7 +1099,13 @@ export class BaserowRound extends BaserowFunctionDefinition {
}
getExamples() {
return ['round(1.12345,2) = 1.12']
return [
'round(1.12345,2) = 1.12',
'round(1234.5678, -2) = 1200',
'round(1234.11111, 2.999) = 1234.11',
'round(1234.11111, 1/0) = NaN',
'round(1234.11111, tonumber("invalid number")) = NaN',
]
}
getFormulaType() {
@ -1099,22 +1113,29 @@ export class BaserowRound extends BaserowFunctionDefinition {
}
}
export class BaserowInt extends BaserowFunctionDefinition {
export class BaserowTrunc extends BaserowFunctionDefinition {
static getType() {
return 'int'
return 'trunc'
}
getDescription() {
const { i18n } = this.app
return i18n.t('formulaFunctions.intDescription')
return i18n.t('formulaFunctions.truncDescription')
}
getSyntaxUsage() {
return ['round(int)']
return ['trunc(number)']
}
getExamples() {
return ['int(1.49) = 1']
return [
'trunc(1.49) = 1',
'trunc(1.51) = 1',
'trunc(-1.51) = -1',
'trunc(-1.49) = -1',
'trunc(1/0) = NaN',
'trunc(tonumber("invalid")) = NaN',
]
}
getFormulaType() {

4
web-frontend/modules/database/plugin.js
View file

@ -146,6 +146,8 @@ import {
BaserowLeft,
BaserowContains,
BaserowFilter,
BaserowTrunc,
BaserowRound,
} from '@baserow/modules/database/formula/functions'
import {
BaserowFormulaArrayType,
@ -401,6 +403,8 @@ export default (context) => {
app.$registry.register('formula_function', new BaserowAvg(context))
app.$registry.register('formula_function', new BaserowSum(context))
app.$registry.register('formula_function', new BaserowFilter(context))
app.$registry.register('formula_function', new BaserowTrunc(context))
app.$registry.register('formula_function', new BaserowRound(context))
// Formula Types
app.$registry.register('formula_type', new BaserowFormulaTextType(context))

2
web-frontend/test/unit/database/formula/formulaFunctions.spec.js
View file

@ -23,7 +23,9 @@ describe('Formula Functions Test', () => {
'contains',
'left',
'right',
'round',
'trim',
'trunc',
'regex_replace',
'multiply',
'divide',