archive/bramw_baserow
1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-07 22:35:36 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'generate-formula-with-ai' into 'develop'

Browse source 
Generate formula with AI initial

See merge request
This commit is contained in:
Bram Wiepjes 2024-05-02 14:37:53 +00:00
commit cab16de230
33 changed files with 1526 additions and 85 deletions
backend
requirements
base.inbase.txt
src/baserow
api/generative_ai
errors.py
core/locale/en/LC_MESSAGES
django.po
changelog/entries/unreleased/feature
generate_formula_using_ai.json
premium
backend
src/baserow_premium
api/fields
serializers.pyurls.pyviews.py
apps.py
fields
actions.pyhandler.pypydantic_models.py
locale/en/LC_MESSAGES
django.po
prompts
__init__.pygenerate_formula.prompt
tests/baserow_premium_tests
api/fields
test_formula_field_views.pytest_generate_formula_prompt.py
fields
test_ai_field_handler.py
web-frontend/modules/baserow_premium
components/field
AIFormulaForm.vueAIFormulaModal.vueFieldAISubForm.vueFormulaFieldAI.vue
fieldTypes.js
locales
en.json
plugin.js
services
field.js
web-frontend
locales
en.json
modules
core
components/ai
SelectAIModelForm.vue
locales
en.json
plugins
clientHandler.js
database
components
field
FieldFormulaSubForm.vue
formula
FieldFormulaInitialSubForm.vue
fieldTypes.js

1
backend/requirements/base.in
View file

@ -69,3 +69,4 @@ sentry-sdk==1.39.1
openai==1.9.0
typing_extensions==4.7.1
ollama==0.1.5
langchain==0.1.16

91
backend/requirements/base.txt
View file

@ -6,6 +6,12 @@
#
advocate==1.0.0
# via -r base.in
aiohttp==3.9.5
# via
# langchain
# langchain-community
aiosignal==1.3.1
# via aiohttp
amqp==5.1.1
# via kombu
annotated-types==0.6.0
@ -26,9 +32,13 @@ asgiref==3.6.0
# django
# opentelemetry-instrumentation-asgi
async-timeout==4.0.2
# via redis
# via
# aiohttp
# langchain
# redis
attrs==22.1.0
# via
# aiohttp
# automat
# jsonschema
# service-identity
@ -111,6 +121,10 @@ cryptography==41.0.3
# service-identity
daphne==4.0.0
# via channels
dataclasses-json==0.6.4
# via
# langchain
# langchain-community
defusedxml==0.7.1
# via pysaml2
deprecated==1.2.13
@ -168,6 +182,10 @@ elementpath==3.0.2
# via xmlschema
faker==18.3.1
# via -r base.in
frozenlist==1.4.1
# via
# aiohttp
# aiosignal
google-api-core==2.11.0
# via
# google-cloud-core
@ -189,6 +207,8 @@ googleapis-common-protos==1.58.0
# via
# google-api-core
# opentelemetry-exporter-otlp-proto-http
greenlet==3.0.3
# via sqlalchemy
gunicorn==20.1.0
# via -r base.in
h11==0.14.0
@ -214,6 +234,7 @@ idna==3.4
# hyperlink
# requests
# twisted
# yarl
importlib-metadata==6.0.1
# via opentelemetry-api
incremental==22.10.0
@ -228,24 +249,58 @@ jmespath==0.10.0
# via
# boto3
# botocore
jsonpatch==1.33
# via
# langchain
# langchain-core
jsonpointer==2.4
# via jsonpatch
jsonschema==4.16.0
# via drf-spectacular
kombu==5.2.4
# via celery
langchain==0.1.16
# via -r base.in
langchain-community==0.0.34
# via langchain
langchain-core==0.1.46
# via
# langchain
# langchain-community
# langchain-text-splitters
langchain-text-splitters==0.0.1
# via langchain
langsmith==0.1.51
# via
# langchain
# langchain-community
# langchain-core
loguru==0.6.0
# via -r base.in
markdown-it-py==3.0.0
# via rich
marshmallow==3.21.1
# via dataclasses-json
mdurl==0.1.2
# via markdown-it-py
monotonic==1.6
# via posthog
msgpack==1.0.4
# via channels-redis
multidict==6.0.5
# via
# aiohttp
# yarl
mypy-extensions==1.0.0
# via typing-inspect
ndg-httpsclient==0.5.1
# via advocate
netifaces==0.11.0
# via advocate
numpy==1.26.4
# via
# langchain
# langchain-community
oauthlib==3.2.2
# via requests-oauthlib
ollama==0.1.5
@ -352,6 +407,12 @@ opentelemetry-util-http==0.42b0
# opentelemetry-instrumentation-django
# opentelemetry-instrumentation-requests
# opentelemetry-instrumentation-wsgi
orjson==3.10.1
# via langsmith
packaging==23.2
# via
# langchain-core
# marshmallow
pillow==10.0.1
# via -r base.in
posthog==3.0.1
@ -383,7 +444,11 @@ pyasn1-modules==0.2.8
pycparser==2.21
# via cffi
pydantic==2.5.3
# via openai
# via
# langchain
# langchain-core
# langsmith
# openai
pydantic-core==2.14.6
# via pydantic
pygments==2.17.2
@ -421,6 +486,9 @@ pytz==2022.4
pyyaml==6.0
# via
# drf-spectacular
# langchain
# langchain-community
# langchain-core
# uvicorn
redis==4.5.4
# via
@ -439,6 +507,9 @@ requests==2.31.0
# azure-core
# google-api-core
# google-cloud-storage
# langchain
# langchain-community
# langsmith
# opentelemetry-exporter-otlp-proto-http
# posthog
# pysaml2
@ -473,10 +544,18 @@ sniffio==1.3.0
# anyio
# httpx
# openai
sqlalchemy==2.0.29
# via
# langchain
# langchain-community
sqlparse==0.4.4
# via django
tenacity==8.1.0
# via celery-redbeat
# via
# celery-redbeat
# langchain
# langchain-community
# langchain-core
tqdm==4.65.0
# via
# -r base.in
@ -498,8 +577,12 @@ typing-extensions==4.7.1
# prosemirror
# pydantic
# pydantic-core
# sqlalchemy
# twisted
# typing-inspect
# uvicorn
typing-inspect==0.9.0
# via dataclasses-json
tzdata==2023.3
# via
# -r base.in
@ -543,6 +626,8 @@ wrapt==1.14.1
# opentelemetry-instrumentation-redis
xmlschema==2.1.0
# via pysaml2
yarl==1.9.4
# via aiohttp
zipp==3.15.0
# via
# -r base.in

10
backend/src/baserow/api/generative_ai/errors.py
View file

@ -10,3 +10,13 @@ ERROR_MODEL_DOES_NOT_BELONG_TO_TYPE = (
HTTP_400_BAD_REQUEST,
"The requested model does not belong to the provided type.",
)
ERROR_GENERATIVE_AI_PROMPT = (
"ERROR_GENERATIVE_AI_PROMPT",
HTTP_400_BAD_REQUEST,
"Something went wrong prompting the model",
)
ERROR_OUTPUT_PARSER = (
"ERROR_OUTPUT_PARSER",
HTTP_400_BAD_REQUEST,
"The model didn't respond with the correct output. Please try again.",
)

4
backend/src/baserow/core/locale/en/LC_MESSAGES/django.po
View file

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2024-04-22 17:45+0000\n"
"POT-Creation-Date: 2024-05-01 19:55+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -515,7 +515,7 @@ msgstr ""
msgid "Account deletion cancelled - Baserow"
msgstr ""
#: src/baserow/core/user/handler.py:252
#: src/baserow/core/user/handler.py:255
#, python-format
msgid "%(name)s's workspace"
msgstr ""

7
changelog/entries/unreleased/feature/generate_formula_using_ai.json Normal file
View file

@ -0,0 +1,7 @@
{
"type": "feature",
"message": "Allow generating a Baserow formula using AI",
"issue_number": null,
"bullet_points": [],
"created_at": "2024-04-29"
}

17
premium/backend/src/baserow_premium/api/fields/serializers.py
View file

@ -12,3 +12,20 @@ class GenerateAIFieldValueViewSerializer(serializers.Serializer):
def to_internal_value(self, data):
return super().to_internal_value(data)
class GenerateFormulaWithAIRequestSerializer(serializers.Serializer):
ai_type = serializers.CharField(
help_text="The AI model type that must be used when generating the formula."
)
ai_model = serializers.CharField(
help_text="The AI model that must be used when generating the formula."
)
ai_prompt = serializers.CharField(
help_text="The human readable input used to generate the formula.",
max_length=1000,
)
class GenerateFormulaWithAIResponseSerializer(serializers.Serializer):
formula = serializers.CharField(help_text="The formula generated by the AI.")

7
premium/backend/src/baserow_premium/api/fields/urls.py
View file

@ -1,6 +1,6 @@
from django.urls import re_path
from .views import AsyncGenerateAIFieldValuesView
from .views import AsyncGenerateAIFieldValuesView, GenerateFormulaWithAIView
app_name = "baserow_premium.api.fields"
@ -10,4 +10,9 @@ urlpatterns = [
AsyncGenerateAIFieldValuesView.as_view(),
name="async_generate_ai_field_values",
),
re_path(
r"table/(?P<table_id>[0-9]+)/generate-ai-formula/$",
GenerateFormulaWithAIView.as_view(),
name="generate_ai_formula",
),
]

89
premium/backend/src/baserow_premium/api/fields/views.py
View file

@ -1,11 +1,13 @@
from django.db import transaction
from baserow_premium.fields.actions import GenerateFormulaWithAIActionType
from baserow_premium.fields.models import AIField
from baserow_premium.fields.tasks import generate_ai_values_for_rows
from baserow_premium.license.features import PREMIUM
from baserow_premium.license.handler import LicenseHandler
from drf_spectacular.openapi import OpenApiParameter, OpenApiTypes
from drf_spectacular.utils import extend_schema
from langchain_core.exceptions import OutputParserException
from rest_framework import status
from rest_framework.permissions import IsAuthenticated
from rest_framework.request import Request
@ -16,7 +18,9 @@ from baserow.api.decorators import map_exceptions, validate_body
from baserow.api.errors import ERROR_USER_NOT_IN_GROUP
from baserow.api.generative_ai.errors import (
ERROR_GENERATIVE_AI_DOES_NOT_EXIST,
ERROR_GENERATIVE_AI_PROMPT,
ERROR_MODEL_DOES_NOT_BELONG_TO_TYPE,
ERROR_OUTPUT_PARSER,
)
from baserow.api.schemas import (
CLIENT_SESSION_ID_SCHEMA_PARAMETER,
@ -25,20 +29,29 @@ from baserow.api.schemas import (
)
from baserow.contrib.database.api.fields.errors import ERROR_FIELD_DOES_NOT_EXIST
from baserow.contrib.database.api.rows.errors import ERROR_ROW_DOES_NOT_EXIST
from baserow.contrib.database.api.tables.errors import ERROR_TABLE_DOES_NOT_EXIST
from baserow.contrib.database.fields.exceptions import FieldDoesNotExist
from baserow.contrib.database.fields.handler import FieldHandler
from baserow.contrib.database.fields.operations import ListFieldsOperationType
from baserow.contrib.database.rows.exceptions import RowDoesNotExist
from baserow.contrib.database.rows.handler import RowHandler
from baserow.contrib.database.table.exceptions import TableDoesNotExist
from baserow.contrib.database.table.handler import TableHandler
from baserow.core.action.registries import action_type_registry
from baserow.core.exceptions import UserNotInWorkspace
from baserow.core.generative_ai.exceptions import (
GenerativeAIPromptError,
GenerativeAITypeDoesNotExist,
ModelDoesNotBelongToType,
)
from baserow.core.generative_ai.registries import generative_ai_model_type_registry
from baserow.core.handler import CoreHandler
from .serializers import GenerateAIFieldValueViewSerializer
from .serializers import (
GenerateAIFieldValueViewSerializer,
GenerateFormulaWithAIRequestSerializer,
GenerateFormulaWithAIResponseSerializer,
)
class AsyncGenerateAIFieldValuesView(APIView):
@ -130,3 +143,77 @@ class AsyncGenerateAIFieldValuesView(APIView):
generate_ai_values_for_rows.delay(request.user.id, ai_field.id, req_row_ids)
return Response(status=status.HTTP_202_ACCEPTED)
class GenerateFormulaWithAIView(APIView):
permission_classes = (IsAuthenticated,)
@extend_schema(
parameters=[
OpenApiParameter(
name="table_id",
location=OpenApiParameter.PATH,
type=OpenApiTypes.INT,
description="The table to generate the formula for.",
),
],
tags=["Database table fields"],
operation_id="generate_formula_with_ai",
description=(
"This endpoint generates a Baserow formula for the table related to the "
"provided id, based on the human readable input provided in the request "
"body."
"\nThis is a **premium** feature."
),
request=GenerateFormulaWithAIRequestSerializer,
responses={
200: GenerateFormulaWithAIResponseSerializer,
400: get_error_schema(
[
"ERROR_GENERATIVE_AI_DOES_NOT_EXIST",
"ERROR_MODEL_DOES_NOT_BELONG_TO_TYPE",
"ERROR_OUTPUT_PARSER",
"ERROR_GENERATIVE_AI_PROMPT",
"ERROR_USER_NOT_IN_GROUP",
]
),
404: get_error_schema(
[
"ERROR_TABLE_DOES_NOT_EXIST",
]
),
},
)
@transaction.atomic
@map_exceptions(
{
UserNotInWorkspace: ERROR_USER_NOT_IN_GROUP,
GenerativeAITypeDoesNotExist: ERROR_GENERATIVE_AI_DOES_NOT_EXIST,
ModelDoesNotBelongToType: ERROR_MODEL_DOES_NOT_BELONG_TO_TYPE,
TableDoesNotExist: ERROR_TABLE_DOES_NOT_EXIST,
GenerativeAIPromptError: ERROR_GENERATIVE_AI_PROMPT,
OutputParserException: ERROR_OUTPUT_PARSER,
}
)
@validate_body(GenerateFormulaWithAIRequestSerializer)
def post(self, request: Request, table_id: int, data: dict) -> Response:
table = TableHandler().get_table(table_id)
workspace = table.database.workspace
LicenseHandler.raise_if_user_doesnt_have_feature(
PREMIUM, request.user, workspace
)
CoreHandler().check_permissions(
request.user,
ListFieldsOperationType.type,
workspace=table.database.workspace,
context=table,
allow_if_template=True,
)
formula = action_type_registry.get(GenerateFormulaWithAIActionType.type).do(
request.user, table, data["ai_type"], data["ai_model"], data["ai_prompt"]
)
return Response({"formula": formula}, status=status.HTTP_200_OK)

3
premium/backend/src/baserow_premium/apps.py
View file

@ -22,6 +22,7 @@ class BaserowPremiumConfig(AppConfig):
field_type_registry,
)
from .fields.actions import GenerateFormulaWithAIActionType
from .fields.field_converters import AIFieldConverter
from .fields.field_types import AIFieldType
@ -40,6 +41,8 @@ class BaserowPremiumConfig(AppConfig):
from baserow.core.action.registries import action_type_registry
from baserow.core.registries import plugin_registry
action_type_registry.register(GenerateFormulaWithAIActionType())
from .export.exporter_types import JSONTableExporter, XMLTableExporter
from .plugins import PremiumPlugin
from .views.decorator_types import (

67
premium/backend/src/baserow_premium/fields/actions.py Normal file
View file

@ -0,0 +1,67 @@
import dataclasses
from django.contrib.auth.models import AbstractUser
from django.utils.translation import gettext_lazy as _
from baserow_premium.fields.handler import AIFieldHandler
from baserow.contrib.database.action.scopes import (
TABLE_ACTION_CONTEXT,
TableActionScopeType,
)
from baserow.contrib.database.table.models import Table
from baserow.core.action.registries import ActionType, ActionTypeDescription
class GenerateFormulaWithAIActionType(ActionType):
type = "generate_formula_with_ai"
description = ActionTypeDescription(
_("Generate Formula With AI"),
_('Generate formula with AI using "%(ai_type)s" and "%(ai_model)s"'),
TABLE_ACTION_CONTEXT,
)
analytics_params = ["table_id", "database_id", "ai_type", "ai_model"]
@dataclasses.dataclass
class Params:
table_id: int
table_name: str
database_id: int
database_name: str
ai_type: str
ai_model: str
@classmethod
def do(
cls,
user: AbstractUser,
table: Table,
ai_type: str,
ai_model: str,
ai_prompt: str,
):
formula = AIFieldHandler.generate_formula_with_ai(
table, ai_type, ai_model, ai_prompt
)
database = table.database
workspace = database.workspace
cls.register_action(
user,
cls.Params(
table.id,
table.name,
database.id,
database.name,
ai_type,
ai_model,
),
cls.scope(workspace.id),
workspace,
)
return formula
@classmethod
def scope(cls, table_id: int):
return TableActionScopeType.value(table_id)

63
premium/backend/src/baserow_premium/fields/handler.py Normal file
View file

@ -0,0 +1,63 @@
import json
from baserow_premium.prompts import get_generate_formula_prompt
from langchain_core.output_parsers import JsonOutputParser
from langchain_core.prompts import PromptTemplate
from baserow.contrib.database.fields.registries import field_type_registry
from baserow.contrib.database.table.models import Table
from baserow.core.db import specific_iterator
from baserow.core.generative_ai.exceptions import ModelDoesNotBelongToType
from baserow.core.generative_ai.registries import generative_ai_model_type_registry
from .pydantic_models import BaserowFormulaModel
class AIFieldHandler:
@classmethod
def generate_formula_with_ai(
cls, table: Table, ai_type: str, ai_model: str, ai_prompt: str
) -> str:
"""
Generate a formula using the provided AI type, model and prompt.
:param table: The table where to generate the formula for.
:param ai_type: The generate AI type that must be used.
:param ai_model: The model related to the AI type that must be used.
:param ai_prompt: The prompt that must be executed.
:raises ModelDoesNotBelongToType: if the provided model doesn't belong to the
type
:return: The generated model.
"""
generative_ai_model_type = generative_ai_model_type_registry.get(ai_type)
ai_models = generative_ai_model_type.get_enabled_models(
table.database.workspace
)
if ai_model not in ai_models:
raise ModelDoesNotBelongToType(model_name=ai_model)
table_schema = []
for field in specific_iterator(table.field_set.all()):
field_type = field_type_registry.get_by_model(field)
table_schema.append(field_type.export_serialized(field))
table_schema_json = json.dumps(table_schema, indent=4)
output_parser = JsonOutputParser(pydantic_object=BaserowFormulaModel)
format_instructions = output_parser.get_format_instructions()
prompt = PromptTemplate(
template=get_generate_formula_prompt() + "\n{format_instructions}",
input_variables=["table_schema_json", "user_prompt"],
partial_variables={"format_instructions": format_instructions},
)
message = prompt.format(
table_schema_json=table_schema_json, user_prompt=ai_prompt
)
response = generative_ai_model_type.prompt(
ai_model, message, workspace=table.database.workspace
)
response_json = output_parser.parse(response)
return response_json["formula"]

5
premium/backend/src/baserow_premium/fields/pydantic_models.py Normal file
View file

@ -0,0 +1,5 @@
from langchain_core.pydantic_v1 import BaseModel, Field
class BaserowFormulaModel(BaseModel):
formula: str = Field(description="The generated Baserow formula")

11
premium/backend/src/baserow_premium/locale/en/LC_MESSAGES/django.po
View file

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2024-04-22 17:45+0000\n"
"POT-Creation-Date: 2024-05-01 19:55+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -18,6 +18,15 @@ msgstr ""
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#: src/baserow_premium/fields/actions.py:19
msgid "Generate Formula With AI"
msgstr ""
#: src/baserow_premium/fields/actions.py:20
#, python-format
msgid "Generate formula with AI using \"%(ai_type)s\" and \"%(ai_model)s\""
msgstr ""
#: src/baserow_premium/row_comments/actions.py:23
msgid "Create row comment"
msgstr ""

7
premium/backend/src/baserow_premium/prompts/__init__.py Normal file
View file

@ -0,0 +1,7 @@
from functools import cache
from importlib.resources import read_text
@cache
def get_generate_formula_prompt():
return read_text("baserow_premium.prompts", "generate_formula.prompt")

228
premium/backend/src/baserow_premium/prompts/generate_formula.prompt Normal file
View file

@ -0,0 +1,228 @@
You're a Baserow formula generator, and will only respond with a Baserow formula. Below you will find the documentation of the Baserow formula language.
URL functions
The markdown table below contains the URL related formula functions.
```
| Functions | Details | Syntax | Examples |
| --------- | ------- | ------ | -------- |
| button | Creates a button using the URI (first argument) and label (second argument). | button(text, text) | button('http://your-text-here.com', 'your-label') |
| get_link_label | Gets the label from a formula using the link or button functions. | get_link_label(button) | get_link_label(field('formula button field')) = 'your-label' |
| get_link_url | Gets the url from a formula using the link or button functions. | get_link_url(link) | get_link_url(field('formula link field')) = 'http://your-text-here.com' |
| link | Creates a hyperlink using the URI provided in the first argument. | link(text) | link('http://your-text-here.com') |
```
Formula functions
The markdown tables below contains the formula related functions.
```
|Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| variance sample | Calculates the sample variance of the values and returns the result. The sample variance should be used when the provided values are only for a sample or subset of values for an underlying population. | variance_sample(numbers from lookup() or field()) | variance_sample(lookup("link field", "number field")) variance_sample(field("lookup field")) variance_sample(field("link field with number primary field")) |
| variance pop | Calculates the population variance of the values and returns the result. The population variance should be used when the provided values contain a value for every single piece of data in the population. | variance_pop(numbers from lookup() or field()) | variance_pop(lookup("link field", "number field")) variance_pop(field("lookup field")) variance_pop(field("link field with number primary field")) |
| sum | Sums all of the values and returns the result. | sum(numbers from lookup() or field()) | sum(lookup("link field", "number field")) sum(field("lookup field")) sum(field("link field with number primary field")) |
| stddev sample | Calculates the sample standard deviation of the values and returns the result. The sample deviation should be used when the provided values are only for a sample or subset of values for an underlying population. | stddev_sample(numbers from lookup() or field()) | stddev_sample(lookup("link field", "number field")) stddev_sample(field("lookup field")) stddev_sample(field("link field with number primary field")) |
| stddev pop | Calculates the population standard deviation of the values and returns the result. The population standard deviation should be used when the provided values contain a value for every single piece of data in the population. | stddev_pop(numbers from lookup() or field()) | stddev_pop(lookup("link field", "number field")) stddev_pop(field("lookup field")) stddev_pop(field("link field with number primary field")) |
| min | Returns the smallest number from all the looked up values provided. | min(numbers from a lookup() or field()) | min(lookup("link field", "number field")) min(field("lookup field")) min(field("link field with text primary field")) |
| max | Returns the largest number from all the looked up values provided. | max(numbers from a lookup() or field()) | max(lookup("link field", "number field")) max(field("lookup field")) max(field("link field with text primary field")) |
| join | Concats all of the values from the first input together using the values from the second input. | join(text from lookup() or field(), text) | join(lookup("link field", "number field"), "_") join(field("lookup field"), field("different lookup field")) join(field("link field with text primary field"), ",") |
| filter | Filters down an expression involving a lookup/link field reference or a lookup function call. | filter(an expression involving lookup() or field(a link/lookup field), boolean) | sum(filter(lookup("link field", "number field"), lookup("link field", "number field") > 10)) filter(field("lookup field"), contains(field("lookup field"), "a")) filter(field("link field") + "a", length(field("link field")) > 10") |
| every | Returns true if every one of the provided looked up values is true, false otherwise. | every(boolean values from a lookup() or field()) | every(field("my lookup") = "test") |
| count | Returns the number of items in its first argument. | count(array) | count(field('my link row field')) |
| avg | Averages all of the values and returns the result. | avg(numbers from lookup() or field()) | avg(lookup("link field", "number field")) avg(field("lookup field")) avg(field("link field with number primary field")) |
| any | Returns true if any one of the provided looked up values is true, false if they are all false. | any(boolean values from a lookup() or field()) | any(field("my lookup") = "test") |
```
```
|Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| when empty | If the first input is calculated to be empty the second input will be returned instead, otherwise if the first input is not empty the first will be returned. | when_empty(any, same type as the first) | when_empty(field("a"), "default") |
| row id | Returns the rows unique identifying number. | row_id() | concat("Row ", row_id()) |
| minus `-` | Returns its two arguments subtracted. | number - number minus(number, number) date - date date - date_interval date_interval - date_interval | 3-1 = 2 |
| lookup | Looks up the values from a field in another table for rows in a link row field. The first argument should be the name of a link row field in the current table and the second should be the name of a field in the linked table. | lookup('a link row field name', 'field name in other the table') | lookup('link row field', 'first name') = lookup('link row field', 'last name') |
| field | Returns the field named by the single text argument. | field('a field name') | field('my text field') = 'flag' |
| add `+` | Returns its two arguments added together. | number + number text + text date + date_interval date_interval + date_interval date_interval + date add(number, number) | 1+1 = 2 'a' + 'b' = 'ab' |
| date interval | Returns the date interval corresponding to the provided argument. | date_interval(text) | date_interval('1 year') date_interval('2 seconds') |
```
Date and time functions
Build more powerful formulas around dates in Baserow. The `today()` and `now()` functions update every 10 minutes.
The `today()` function is useful for calculating intervals or when you need to have the current date displayed on a table. The `now()` function is useful when you need to display the current date and time on your table or calculate a value based on the current date and time, and have that value updated each time you open your database.
Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| year | Returns the number of years in the provided date. | year(date) | year(field("my date")) |
| now | Returns the current date and time in utc. | now() | now() > todate("2021-12-12 13:00:00", "YYYY-MM-DD HH24:MI:SS") |
| todate | Returns the first argument converted into a date given a date format string as the second argument. | todate(text, text) | todate('20210101', 'YYYYMMDD') |
| todate_tz | Returns the first argument converted into a date given a date format string as the second argument and the [timezone][5] provided as third argument. | todate_tz(text, text, text) | now() > todate("2021-12-12 13:00:00", "YYYY-MM-DD HH24:MI:SS") |
| second | Returns the number of seconds in the provided date. | second(date) | second(field("dates")) == 2 |
| month | Returns the number of months in the provided date. | month(date) | month(todate("2021-12-12", "YYYY-MM-DD")) = 12 |
| today | Returns the current date in utc. | today() | today() > todate("2021-12-12", "YYYY-MM-DD") |
| day | Returns the day of the month as a number between 1 to 31 from the argument. | day(date) | day(todate('20210101', 'YYYYMMDD')) = 1 |
| datetime_format | Converts the date to text given a way of formatting the date. | datetime_format(date, text) | datetime_format(field('date field'), 'YYYY') |
| date_diff | Given a date unit to measure in as the first argument ('year', 'month', 'week', 'day', 'hour', 'minute', 'seconds') calculates and returns the number of units from the second argument to the third. | date_diff(text, date, date) | date_diff('yy', todate('2000-01-01', 'YYYY-MM-DD'), todate('2020-01-01', 'YYYY-MM-DD')) = 20 |
| datetime_format_tz|
Boolean functions
The markdown table below contains the boolean functions.
```
|Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| or | Returns the logical or of the first and second argument, so if either are true then the result is true, otherwise it is false. | or(boolean, boolean) | or(true, false) = true and(true, true) = true or(field('first test'), field('second test')) |
| not_equal `!=` | Returns if its two arguments have different values. | any != any not_equal(any, any) | 1!=2 'a' != 'b’ |
| not | Returns false if the argument is true and true if the argument is false. | not(boolean) | not(true) = false not(10=2) = true |
| less_than_or_equal `<=` | Returns true if the first argument less than or equal to the second, otherwise false. | any <= any | 1 <= 1 = true if(field('a') <= field('b'), 'a smaller', 'b is greater than or equal') |
| less_than `<` | Returns true if the first argument less than the second, otherwise false. | any < any | 2 < 1 = false if(field('a') < field('b'), 'a is smaller', 'b is bigger or equal') |
| isblank | Returns true if the argument is empty or blank, false otherwise. | isblank(any) | isblank('10') |
| if | If the first argument is true then returns the second argument, otherwise returns the third. | if(bool, any, any) | if(field('text field') = 'on', 'it is on', 'it is off') |
| greater_than_or_equal `>=` | Returns true if the first argument is greater than or equal to the second, otherwise false. | any >= any | 1 >= 1 = true if(field('a') >= field('b'), 'a is bigger or equal', 'b is smaller') |
| greater_than `>` | Returns true if the first argument greater than the second, otherwise false. | any > any | 1 > 2 = false if(field('a') > field('b'), 'a is bigger', 'b is bigger or equal') |
| equal `=` | Returns if its two arguments have the same value. | any = any equal(any, any) | 1=1 'a' = 'a' |
| and | Returns the logical and of the first and second argument, so if they are bothtrue then the result is true, otherwise it is false. | and(boolean, boolean) | and(true, false) = false and(true, true) = true and(field('first test'), field('second test')) |
| is_null | Returns true if the argument is null, false otherwise | is_null(any) | is_null('10') |
| is_image | Returns if the single file returned from the index function is an image or not. | is_image(a file) | is_image(index(field("File field"), 0)) |
```
Number functions
The markdown table below contains the number functions.
```
| Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| tonumber | Converts the input to a number if possible. | tonumber(text) | tonumber('10') = 10 |
| sqrt | Returns the square root of the argument provided. | sqrt(number) | sqrt(9) = 3 |
| least | Returns the smallest of the two inputs. | least(number, number) | least(1,2) = 1 |
| greatest | Returns the greatest value of the two inputs. | greatest(number, number) | greatest(1,2) = 2 |
| divide `/` | Returns its two arguments divided, the first divided by the second. | number / number divide(number, number) | 10/2 = 5 |
| abs | Returns the absolute value for the argument number provided. | abs(number) | abs(1.49) = 1.49 |
| ceil | Returns the smallest integer that is greater than or equal the argument number provided. | ceil(number) | ceil(1.49) = 2 |
| even | Returns true if the argument provided is an even number, false otherwise. | even(number) | even(2) = true |
| exp | Returns the result of the constant e ≈ 2.718 raised to the argument number provided. | exp(number) | exp(1.000) = 2.718 |
| floor | Returns the largest integer that is less than or equal the argument number provided. | floor(number) | floor(1.49) = 1 |
| is_nan | Returns true if the argument is 'NaN', returns false otherwise. | is_nan(number) | is_nan(1 / 0) = true |
| ln | Natural logarithm function: returns the exponent to which the constant e ≈ 2.718 must be raised to produce the argument. | ln(number) | ln(2.718) = 1.000 |
| log | Logarithm function: returns the exponent to which the first argument must be raised to produce the second argument. | log(number, number) | log(3, 9) = 2 |
| mod | Returns the remainder of the division between the first argument and the second argument. | mod(number, number) | mod(5, 2) = 1 |
| multiply `*` | Returns its two arguments multiplied together. | multiply(number, number) | 2*5 = 10 |
| odd | Returns true if the argument provided is an odd number, false otherwise. | odd(number) | odd(2) = false |
| power | Returns the result of the first argument raised to the second argument exponent. | power(number, number) | power(3, 2) = 9 |
| round | Returns first argument rounded to the number of digits specified by the second argument. | round(number, number) | round(1.12345,2) = 1.12 |
| sign | Returns 1 if the argument is a positive number, -1 if the argument is a negative one, 0 otherwise. | sign(number) | sign(2.1234) = 1 |
| trunc | Returns only the first argument converted into an integer by truncating any decimal places. | trunc(number) | trunc(1.49) = 1 |
| when nan | Returns the first argument if it's not 'NaN'. Returns the second argument if the first argument is 'NaN' | when_nan(number, fallback) | when_nan(1 / 0, 4) = 4 |
| get_file_size | Returns the file size from a single file returned from the index function. | get_file_size(a file) | get_file_size(index(field("File field"), 0))
| get_image_width | Returns the image width from a single file returned from the index function. | get_image_width(a file) | get_image_width(index(field("File field"), 0)) |
| get_image_height | Returns the image height from a single file returned from the index function. | get_image_height(a file) | get_image_height(index(field("File field"), 0)) |
| get_file_count | Creates a button using the URI (first argument) and label (second argument). | get_file_count(a file field) | get_file_count(field("File field")) |
```
Text functions
The markdown table below contains the text related functions.
```
|Functions | Details | Syntax | Examples |
| --- | --- | --- | --- |
| upper | Returns its argument in upper case. | upper(text) | upper('a') = 'A' |
| trim | Removes all whitespace from the left and right sides of the input. | trim(text) | trim(" abc ") = "abc" |
| totext | Converts the input to text. | totext(any) | totext(10) = '10' |
| t | Returns the arguments value if it is text, but otherwise ''. | t(any) | t(10) |
| search | Returns a positive integer starting from 1 for the first occurrence of the second argument inside the first, or 0 if no occurrence is found. | search(text, text) | search("test a b c test", "test") = 1 search("none", "test") = 0 |
| right | Extracts the right most characters from the first input, stops when it has extracted the number of characters specified by the second input. | right(text, number) | right("abcd", 2) = "cd" |
| reverse | Returns the reversed text of the provided first argument. | reverse(text) | reverse("abc") = "cba" |
| replace | Replaces all instances of the second argument in the first argument with the third argument. | replace(text, text, text) | replace("test a b c test", "test", "1") = "1 a b c 1" |
| regex_replace | Replaces any text in the first input which matches the regex specified by the second input with the text in the third input. | regex_replace(text, regex text, replacement text) | regex_replace("abc", "a", "1") = "1bc" |
| lower | Returns its argument in lower case. | lower(text) | lower('A') = 'a' |
| length | Returns the number of characters in the first argument provided. | length(text) | length("abc") = 3 |
| left | Extracts the left most characters from the first input, stops when it has extracted the number of characters specified by the second input. | left(text, number) | left("abcd", 2) = "ab" |
| contains | Returns true if the first piece of text contains at least once the second. | contains(text,text) | contains("test", "e") = true |
| concat | Returns its arguments joined together as a single piece of text. | concat(any, any, ...) | concat('A', 1, 1=2) = 'A1false' |
| encode_uri | Returns a encoded URI string from the argument provided. | encode_uri(text) | encode_uri('http://example.com/wiki/Señor') = 'http://example.com/wiki/Se%c3%b1or' |
| encode_uri_component | Returns a encoded URI string component from the argument provided. | encode_uri_component(text) | encode_uri_component('Hello World') = 'Hello%20World' |
| split_part | Extracts a segment from a delimited string based on a delimiter and index (numeric indicator indicating which element from string should be returned) | split_part(text, delimiter, position) | split_part('John, Jane, Michael', ', ', 2) = 'Jane' |
| has_option | Returns true if the first argument is a multiple select field or a lookup to a single select field and the second argument is one of the options. | has_option(multiple select, text); has_option(lookup(link row, single select), text) | has_option(field('multiple select'), 'option_a'); has_option(lookup(field('link row'), field('single select')), 'option_a') |
| get_file_visible_name | Returns the visible file name from a single file returned from the index function. | get_file_visible_name(a file) | get_file_visible_name(index(field("File field"), 0)) |
| get_file_mime_type | Returns the file mime type from a single file returned from the index function. | get_file_mime_type(a file) | get_file_mime_type(index(field("File field"), 0)) |
```
Boolean functions not working well with fields as arguments
Formula functions, for example, isblank(), or when_empty work with simple values like text, number, or date fields. Computed fields like Link-to-table, look-up, and rollup fields can contain multiple items which makes them arrays or lists.
To create formulas to make a Boolean test on data in field C, taking data from field A if it’s TRUE, otherwise taking data from field B if it’s FALSE, you need to convert any array to text using the join() function. For example: `if(isblank(join(field('Organization'),'')), field('Notes'), field('Name'))`.
Using join() to convert the list to text, handles the empty scenario correctly. This formula checks if the Organization field (a link-to-table field) has a value. If it’s true, it shows the content of the Name field; otherwise, it displays the content of the Notes field.
What a Baserow Formula Field is
A Baserow Formula field lets you create a field whose contents are calculated based on a Baserow Formula you’ve provided. A Baserow Formula is simply some text written in a particular way such that Baserow can understand it, for example the text 1+1 is a Baserow formula which will calculate the result 2 for every row.
A Simple Formula Example
Imagine you have a table with a normal text field called text field with 3 rows containing the text one,two and three respectively. If you then create a formula field with the formula concat('Number', field('text field')) the resulting table would look like:
```
|text field|formula field|
|----------|-------------|
|one|Number one|
|two|Number two|
|three|Number three|
```
Breaking down a simple formula
Let’s split apart the formula concat('Number', field('text field')) to understand what is going on:
* `concat`: Concat is one of many formula functions you can use. It will join together all the inputs you give to it into one single piece of text.
* `(`: To give inputs to a formula function you first have to write an opening parenthesis indicating the inputs will follow.
* `Number`: This is the first input we are giving to concat and it is literally just the text Number. When writing literal pieces of text in a formula you need to surround them with quotes.
* `,`: As we are giving multiple inputs to concat we need to separate each input with a comma.
* `field('text field')`: This is the second and final input we are giving to concat. We could keep on adding however many inputs as we wanted however as long as each was separated by a comma. This second input is a reference to the field in the same table with the name text field. For each cell in the formula field this reference will be replaced by whatever the value in the text field field is for that row.
* `)`: Finally, we need to tell Baserow we’ve finished giving inputs to the concat function, we do this with a matching closing parenthesis.
What is a formula function?
A function in a formula takes a number of inputs depending on the type of the function. It does some calculation using those inputs and produces an output. Functions also sometimes only take specific types of inputs. For example the datetime_format only accepts two inputs, the first must be a date (either a field reference to a date field Or a sub formula which calculates a date) and the second must be some text.
All the available functions for you to use are shown in the expanded formula edit box which appears when you click on the formula whilst editing a formula field.
Using numbers in formulas
Formulas can be used to do numerical calculations. The standard maths operators exist like +,-,* and /. You can use whole numbers or decimal numbers directly in your formula like so (field('number field') + 10.005)/10
Conditional calculations
If you need to do a calculation conditionally then the if function and comparison operators will let you do this. For example the following formula calculates whether a date field is the first day of a month, IF(day(field('some date')) = 1, true, false).
You can compare fields and sub-formulas using the >, >= <=, <, = and != operators.
Using Dates
Use the todate function to create a constant date inside a formula like so: todate('2020-01-01 10:20:30', 'YYYY-MM-DD HH:MI:SS'). The first argument is the date you want in text form and the second is the format of the date text.
Using Date intervals
Subtracting two dates returns the difference in time between the two dates: field('date a') - field('date b'). The date_interval function lets you create intervals inside the formula to work with.
Need to calculate a new date based on a date/time interval? Use the date_interval function like so: field('my date column') - date_interval('1 year')
This is the end of the formula documentation and explanation
--------------------------------------
In the JSON below, you will fine the fields of the table where the formula is created. When referencing a field using the `field` function, you're only allowed to reference these fields, the ones that are in the table. Field names can't be made up. Below an array of the fields in the table in JSON format, where each item represents a field with some additional options.
```
{table_schema_json}
```
You're a Baserow formula generator, and you're only responding with the correct formula. The formula you're generating can only contain function and operators available to the Baserow formula, not any other formula language. It can only reference fields in the JSON described above, not other fields.
Generate a Baserow formula based on the following input: "{user_prompt}".

344
premium/backend/tests/baserow_premium_tests/api/fields/test_formula_field_views.py Normal file
View file

@ -0,0 +1,344 @@
from unittest.mock import patch
from django.shortcuts import reverse
from django.test.utils import override_settings
import pytest
from rest_framework.status import (
HTTP_200_OK,
HTTP_400_BAD_REQUEST,
HTTP_402_PAYMENT_REQUIRED,
HTTP_404_NOT_FOUND,
)
from baserow.core.generative_ai.registries import generative_ai_model_type_registry
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_without_license(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=False,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test",
"ai_model": "test",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_402_PAYMENT_REQUIRED
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_table_does_not_exist(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": 0},
),
{
"ai_type": "test",
"ai_model": "test",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_404_NOT_FOUND
assert response.json()["error"] == "ERROR_TABLE_DOES_NOT_EXIST"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_user_not_in_workspace(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
user_2, token_2 = premium_data_fixture.create_user_and_token(
email="test2@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test_generative_ai",
"ai_model": "model_1",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token_2}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json()["error"] == "ERROR_USER_NOT_IN_GROUP"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_generative_ai_does_not_exist(
premium_data_fixture, api_client
):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "does_not_exist",
"ai_model": "model_1",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json()["error"] == "ERROR_GENERATIVE_AI_DOES_NOT_EXIST"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_model_does_not_belong_to_type(
premium_data_fixture, api_client
):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test_generative_ai",
"ai_model": "does_not_exist",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json()["error"] == "ERROR_MODEL_DOES_NOT_BELONG_TO_TYPE"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_with_error(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test_generative_ai_prompt_error",
"ai_model": "test_1",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json()["error"] == "ERROR_GENERATIVE_AI_PROMPT"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_output_parser_error(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
text_field = premium_data_fixture.create_text_field(table=table)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test_generative_ai",
"ai_model": "test_1",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json()["error"] == "ERROR_OUTPUT_PARSER"
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula_invalid_request(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "",
"ai_model": "",
"ai_prompt": "",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
assert response.status_code == HTTP_400_BAD_REQUEST
assert response.json() == {
"error": "ERROR_REQUEST_BODY_VALIDATION",
"detail": {
"ai_type": [{"error": "This field may not be blank.", "code": "blank"}],
"ai_model": [{"error": "This field may not be blank.", "code": "blank"}],
"ai_prompt": [{"error": "This field may not be blank.", "code": "blank"}],
},
}
@pytest.mark.django_db
@pytest.mark.field_ai
@override_settings(DEBUG=True)
def test_generate_formula(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user, token = premium_data_fixture.create_user_and_token(
email="test@test.nl",
password="password",
first_name="Test1",
has_active_premium_license=True,
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
text_field = premium_data_fixture.create_text_field(table=table)
generative_ai_instance = generative_ai_model_type_registry.get("test_generative_ai")
with patch.object(
generative_ai_instance, "prompt", return_value='{"formula": "field()"}'
) as mock:
response = api_client.post(
reverse(
"api:premium:fields:generate_ai_formula",
kwargs={"table_id": table.id},
),
{
"ai_type": "test_generative_ai",
"ai_model": "test_1",
"ai_prompt": "Generate a formula with all field types",
},
format="json",
HTTP_AUTHORIZATION=f"JWT {token}",
)
prompt = mock.call_args[0][1]
assert "You're a Baserow formula generator," in prompt
assert f'"name": "{text_field.name}"' in prompt
assert "Generate a formula with all field types" in prompt
assert response.status_code == HTTP_200_OK
assert response.json() == {"formula": "field()"}

29
premium/backend/tests/baserow_premium_tests/api/fields/test_generate_formula_prompt.py Normal file
View file

@ -0,0 +1,29 @@
from baserow_premium.prompts import get_generate_formula_prompt
from baserow.contrib.database.formula.registries import formula_function_registry
def test_if_prompt_contains_all_formula_functions():
prompt = get_generate_formula_prompt()
# These functions are for internal usage, and are not in the web-frontend
# documentation.
formula_exceptions = [
"tovarchar",
"error_to_nan",
"bc_to_null",
"error_to_null",
"array_agg",
"array_agg_unnesting",
"multiple_select_options_agg",
"get_single_select_value",
"multiple_select_count",
"string_agg_multiple_select_values",
"string_agg_array_of_multiple_select_values",
"jsonb_extract_path_text",
"array_agg_no_nesting",
]
for function in formula_function_registry.registry.keys():
if function not in formula_exceptions and function not in prompt:
assert False, f"{function} is not present in generate_formula.prompt"

122
premium/backend/tests/baserow_premium_tests/fields/test_ai_field_handler.py Normal file
View file

@ -0,0 +1,122 @@
from unittest.mock import patch
import pytest
from baserow_premium.fields.handler import AIFieldHandler
from langchain_core.exceptions import OutputParserException
from baserow.core.generative_ai.exceptions import (
GenerativeAITypeDoesNotExist,
ModelDoesNotBelongToType,
)
from baserow.core.generative_ai.registries import generative_ai_model_type_registry
@pytest.mark.django_db
@pytest.mark.field_ai
def test_generate_formula_generative_ai_does_not_exist(
premium_data_fixture, api_client
):
premium_data_fixture.register_fake_generate_ai_type()
user = premium_data_fixture.create_user(
email="test@test.nl",
password="password",
first_name="Test1",
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
with pytest.raises(GenerativeAITypeDoesNotExist):
AIFieldHandler.generate_formula_with_ai(
table,
"does_not_exist",
"model_1",
"Generate a formula with all field types",
)
@pytest.mark.django_db
@pytest.mark.field_ai
def test_generate_formula_model_does_not_belong_to_type(
premium_data_fixture, api_client
):
premium_data_fixture.register_fake_generate_ai_type()
user = premium_data_fixture.create_user(
email="test@test.nl",
password="password",
first_name="Test1",
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
with pytest.raises(ModelDoesNotBelongToType):
AIFieldHandler.generate_formula_with_ai(
table,
"test_generative_ai",
"does_not_exist",
"Generate a formula with all field types",
)
@pytest.mark.django_db
@pytest.mark.field_ai
def test_generate_formula_output_parser_error(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user = premium_data_fixture.create_user(
email="test@test.nl",
password="password",
first_name="Test1",
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
with pytest.raises(OutputParserException):
AIFieldHandler.generate_formula_with_ai(
table,
"test_generative_ai",
"test_1",
"Generate a formula with all field types",
)
@pytest.mark.django_db
@pytest.mark.field_ai
def test_generate_formula(premium_data_fixture, api_client):
premium_data_fixture.register_fake_generate_ai_type()
user = premium_data_fixture.create_user(
email="test@test.nl",
password="password",
first_name="Test1",
)
database = premium_data_fixture.create_database_application(
user=user, name="database"
)
table = premium_data_fixture.create_database_table(name="table", database=database)
text_field = premium_data_fixture.create_text_field(table=table)
generative_ai_instance = generative_ai_model_type_registry.get("test_generative_ai")
with patch.object(
generative_ai_instance, "prompt", return_value='{"formula": "field()"}'
) as mock:
formula = AIFieldHandler.generate_formula_with_ai(
table,
"test_generative_ai",
"test_1",
"Generate a formula with all field types",
)
prompt = mock.call_args[0][1]
assert "You're a Baserow formula generator," in prompt
assert f'"name": "{text_field.name}"' in prompt
assert "Generate a formula with all field types" in prompt
assert formula == "field()"

84
premium/web-frontend/modules/baserow_premium/components/field/AIFormulaForm.vue Normal file
View file

@ -0,0 +1,84 @@
<template>
<form @submit.prevent="submit">
<SelectAIModelForm
:database="database"
:default-values="defaultValues"
></SelectAIModelForm>
<div class="control">
<label class="control__label control__label--small">{{
$t('aiFormulaModal.label')
}}</label>
<div class="control__description">
{{ $t('aiFormulaModal.labelDescription') }}
</div>
<div class="control__elements">
<div>
<textarea
v-model="values.ai_prompt"
type="text"
class="input field-long-text"
:class="{
'input--error':
$v.values.ai_prompt.$dirty && $v.values.ai_prompt.$error,
}"
@input="$v.values.ai_prompt.$touch()"
></textarea>
</div>
<div
v-if="$v.values.ai_prompt.$error && !$v.values.ai_prompt.maxLength"
class="error"
>
{{
$t('error.maxLength', {
max: $v.values.ai_prompt.$params.maxLength.max,
})
}}
</div>
<div
v-if="$v.values.ai_prompt.$error && !$v.values.ai_prompt.required"
class="error"
>
{{ $t('error.requiredField') }}
</div>
</div>
</div>
<div class="align-right">
<slot></slot>
</div>
</form>
</template>
<script>
import SelectAIModelForm from '@baserow/modules/core/components/ai/SelectAIModelForm'
import { required, maxLength } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
export default {
name: 'AIFormulaForm',
components: { SelectAIModelForm },
mixins: [form],
props: {
database: {
type: Object,
required: true,
},
},
data() {
return {
allowedValues: ['ai_prompt'],
values: {
ai_prompt: '',
},
}
},
validations: {
values: {
ai_prompt: {
required,
maxLength: maxLength(1000),
},
},
},
}
</script>

97
premium/web-frontend/modules/baserow_premium/components/field/AIFormulaModal.vue Normal file
View file

@ -0,0 +1,97 @@
<template>
<Modal ref="modal" small>
<template #content>
<div class="box__title">
<h2 class="row_modal__title">{{ $t('aiFormulaModal.title') }}</h2>
</div>
<Error :error="error"></Error>
<template v-if="hasModels">
<p>
{{ $t('aiFormulaModal.description') }}
</p>
<AIFormulaForm :database="database" @submitted="submit">
<Button type="primary" :loading="loading" :disabled="loading">{{
$t('aiFormulaModal.generate')
}}</Button>
</AIFormulaForm>
</template>
<p v-else>
{{ $t('aiFormulaModal.noModels') }}
</p>
</template>
</Modal>
</template>
<script>
import { required } from 'vuelidate/lib/validators'
import modal from '@baserow/modules/core/mixins/modal'
import error from '@baserow/modules/core/mixins/error'
import FieldService from '@baserow_premium/services/field'
import AIFormulaForm from '@baserow_premium/components/field/AIFormulaForm.vue'
export default {
name: 'AIFormulaModal',
components: { AIFormulaForm },
mixins: [modal, error],
props: {
database: {
type: Object,
required: true,
},
table: {
type: Object,
required: true,
},
},
data() {
return {
loading: false,
allowedValues: ['ai_prompt'],
values: {
ai_prompt: '',
},
}
},
computed: {
// Return the reactive object that can be updated in runtime.
workspace() {
return this.$store.getters['workspace/get'](this.database.workspace.id)
},
hasModels() {
return Object.values(this.workspace.generative_ai_models_enabled).some(
(models) => models.length > 0
)
},
},
methods: {
async submit(values) {
if (this.loading) {
return
}
this.loading = true
try {
const { data } = await FieldService(this.$client).generateAIFormula(
this.table.id,
values.ai_generative_ai_type,
values.ai_generative_ai_model,
values.ai_prompt
)
this.$emit('formula', data.formula)
this.hide()
} catch (error) {
this.handleError(error)
} finally {
this.loading = false
}
},
},
validations: {
values: {
ai_prompt: { required },
},
},
}
</script>

79
premium/web-frontend/modules/baserow_premium/components/field/FieldAISubForm.vue
View file

@ -1,57 +1,9 @@
<template>
<div v-if="!isDeactivated">
<div class="control">
<label class="control__label control__label--small">{{
$t('fieldAISubForm.AIType')
}}</label>
<div class="control__elements">
<Dropdown
v-model="values.ai_generative_ai_type"
class="dropdown--floating"
:class="{
'dropdown--error': $v.values.ai_generative_ai_type.$error,
}"
:fixed-items="true"
:show-search="false"
small
@hide="$v.values.ai_generative_ai_type.$touch()"
@change="$refs.aiModel.select(aIModelsPerType[0])"
>
<DropdownItem
v-for="aIType in aITypes"
:key="aIType"
:name="aIType"
:value="aIType"
/>
</Dropdown>
</div>
</div>
<div class="control">
<label class="control__label control__label--small">
{{ $t('fieldAISubForm.AIModel') }}
</label>
<div class="control__elements">
<Dropdown
ref="aiModel"
v-model="values.ai_generative_ai_model"
class="dropdown--floating"
:class="{
'dropdown--error': $v.values.ai_generative_ai_model.$error,
}"
:fixed-items="true"
:show-search="false"
small
@hide="$v.values.ai_generative_ai_model.$touch()"
>
<DropdownItem
v-for="aIType in aIModelsPerType"
:key="aIType"
:name="aIType"
:value="aIType"
/>
</Dropdown>
</div>
</div>
<SelectAIModelForm
:default-values="defaultValues"
:database="database"
></SelectAIModelForm>
<div class="control">
<label class="control__label control__label--small">
{{ $t('fieldAISubForm.prompt') }}
@ -87,21 +39,16 @@ import { required } from 'vuelidate/lib/validators'
import form from '@baserow/modules/core/mixins/form'
import fieldSubForm from '@baserow/modules/database/mixins/fieldSubForm'
import FormulaInputField from '@baserow/modules/core/components/formula/FormulaInputField'
import SelectAIModelForm from '@baserow/modules/core/components/ai/SelectAIModelForm'
export default {
name: 'FieldAISubForm',
components: { FormulaInputField },
components: { SelectAIModelForm, FormulaInputField },
mixins: [form, fieldSubForm],
data() {
return {
allowedValues: [
'ai_generative_ai_type',
'ai_generative_ai_model',
'ai_prompt',
],
allowedValues: ['ai_prompt'],
values: {
ai_generative_ai_type: null,
ai_generative_ai_model: null,
ai_prompt: '',
},
}
@ -126,16 +73,6 @@ export default {
dataProviders() {
return [this.$registry.get('databaseDataProvider', 'fields')]
},
aITypes() {
return Object.keys(this.workspace.generative_ai_models_enabled || {})
},
aIModelsPerType() {
return (
this.workspace.generative_ai_models_enabled[
this.values.ai_generative_ai_type
] || []
)
},
isDeactivated() {
return this.$registry
.get('field', this.fieldType)
@ -144,8 +81,6 @@ export default {
},
validations: {
values: {
ai_generative_ai_type: { required },
ai_generative_ai_model: { required },
ai_prompt: { required },
},
},

55
premium/web-frontend/modules/baserow_premium/components/field/FormulaFieldAI.vue Normal file
View file

@ -0,0 +1,55 @@
<template>
<div class="margin-top-1">
<a v-if="hasPremium" @click="$refs.aiModal.show()">
<i class="iconoir-magic-wand"></i>
{{ $t('formulaFieldAI.generateWithAI') }}
</a>
<a v-else @click="$refs.premiumModal.show()">
<i class="iconoir-lock"></i>
{{ $t('formulaFieldAI.generateWithAI') }}
</a>
<AIFormulaModal
ref="aiModal"
:database="database"
:table="table"
@formula="$emit('update-formula', $event)"
></AIFormulaModal>
<PremiumModal
ref="premiumModal"
:workspace="workspace"
:name="$t('formulaFieldAI.featureName')"
></PremiumModal>
</div>
</template>
<script>
import AIFormulaModal from '@baserow_premium/components/field/AIFormulaModal'
import PremiumFeatures from '@baserow_premium/features'
import PremiumModal from '@baserow_premium/components/PremiumModal.vue'
export default {
name: 'FormulaFieldAI',
components: { PremiumModal, AIFormulaModal },
props: {
database: {
type: Object,
required: true,
},
table: {
type: Object,
required: true,
},
},
computed: {
hasPremium() {
return this.$hasFeature(
PremiumFeatures.PREMIUM,
this.database.workspace_id
)
},
workspace() {
return this.$store.getters['workspace/get'](this.database.workspace.id)
},
},
}
</script>

12
premium/web-frontend/modules/baserow_premium/fieldTypes.js
View file

@ -1,4 +1,7 @@
import { FieldType } from '@baserow/modules/database/fieldTypes'
import {
FieldType,
FormulaFieldType,
} from '@baserow/modules/database/fieldTypes'
import RowHistoryFieldText from '@baserow/modules/database/components/row/RowHistoryFieldText'
import RowCardFieldText from '@baserow/modules/database/components/card/RowCardFieldText'
import { collatedStringCompare } from '@baserow/modules/core/utils/string'
@ -11,6 +14,7 @@ import GridViewFieldAI from '@baserow_premium/components/views/grid/fields/GridV
import FunctionalGridViewFieldAI from '@baserow_premium/components/views/grid/fields/FunctionalGridViewFieldAI'
import RowEditFieldAI from '@baserow_premium/components/row/RowEditFieldAI'
import FieldAISubForm from '@baserow_premium/components/field/FieldAISubForm'
import FormulaFieldAI from '@baserow_premium/components/field/FormulaFieldAI'
import GridViewFieldAIGenerateValuesContextItem from '@baserow_premium/components/views/grid/fields/GridViewFieldAIGenerateValuesContextItem'
import PremiumModal from '@baserow_premium/components/PremiumModal'
import PremiumFeatures from '@baserow_premium/features'
@ -109,3 +113,9 @@ export class AIFieldType extends FieldType {
return PremiumModal
}
}
export class PremiumFormulaFieldType extends FormulaFieldType {
getAdditionalFormInputComponents() {
return [FormulaFieldAI]
}
}

14
premium/web-frontend/modules/baserow_premium/locales/en.json
View file

@ -364,13 +364,23 @@
"regenerate": "Re-Generate"
},
"fieldAISubForm": {
"AIType": "AI Type",
"AIModel": "AI Model",
"prompt": "Prompt",
"premiumFeature": "The AI field is a premium feature"
},
"rowEditFieldAI": {
"generate": "Generate",
"createRowBefore": "The AI value can be generated after the row has been created."
},
"aiFormulaModal": {
"title": "Generate formula using AI",
"description": "Note that the generated formulas might not always work as expected. We're constructing a prompt for the model, and we load the output in the formula input. It works best with a high parameter model like gpt-4-turbo-preview.",
"label": "Prompt",
"labelDescription": "Describe the formula you would like to generate",
"generate": "Generate",
"noModels": "Your Baserow instance and workspace doesn't have any AI models configured. Click on the three dots next to your workspace, then on settings to configure them."
},
"formulaFieldAI": {
"generateWithAI": "Generate using AI",
"featureName": "Generate formula using AI"
}
}

6
premium/web-frontend/modules/baserow_premium/plugin.js
View file

@ -43,7 +43,10 @@ import {
RowCommentNotificationType,
} from '@baserow_premium/notificationTypes'
import { CommentsRowModalSidebarType } from '@baserow_premium/rowModalSidebarTypes'
import { AIFieldType } from '@baserow_premium/fieldTypes'
import {
AIFieldType,
PremiumFormulaFieldType,
} from '@baserow_premium/fieldTypes'
export default (context) => {
const { store, app, isDev } = context
@ -92,6 +95,7 @@ export default (context) => {
app.$registry.register('exporter', new JSONTableExporter(context))
app.$registry.register('exporter', new XMLTableExporter(context))
app.$registry.register('field', new AIFieldType(context))
app.$registry.register('field', new PremiumFormulaFieldType(context))
app.$registry.register('view', new KanbanViewType(context))
app.$registry.register('view', new CalendarViewType(context))

10
premium/web-frontend/modules/baserow_premium/services/field.js
View file

@ -6,5 +6,15 @@ export default (client) => {
{ row_ids: rowIds }
)
},
generateAIFormula(tableId, aiType, aiModel, prompt) {
return client.post(
`/database/fields/table/${tableId}/generate-ai-formula/`,
{
ai_type: aiType,
ai_model: aiModel,
ai_prompt: prompt,
}
)
},
}
}

6
web-frontend/locales/en.json
View file

@ -325,7 +325,11 @@
"generativeAIDoesNotExistTitle": "Generative AI does not exist",
"generativeAIDoesNotExistDescription": "The generative AI model does not exist.",
"modelDoesNotBelongToTypeTitle": "The selected model does not belong to the AI Type",
"modelDoesNotBelongToTypeDescription": "The selected model does not belong to the selected AI type."
"modelDoesNotBelongToTypeDescription": "The selected model does not belong to the selected AI type.",
"outputParserTitle": "Wrong output",
"outputParserDescription": "The model responded with an incorrect output. Please try again.",
"generateAIPromptTitle": "Prompt error",
"generateAIPromptDescription": "Something was wrong with the constructed prompt."
},
"importerType": {
"csv": "Import a CSV file",

108
web-frontend/modules/core/components/ai/SelectAIModelForm.vue Normal file
View file

@ -0,0 +1,108 @@
<template>
<div>
<div class="control">
<label class="control__label control__label--small">{{
$t('selectAIModelForm.AIType')
}}</label>
<div class="control__elements">
<Dropdown
v-model="values.ai_generative_ai_type"
class="dropdown--floating"
:class="{
'dropdown--error': $v.values.ai_generative_ai_type.$error,
}"
:fixed-items="true"
:show-search="false"
small
@hide="$v.values.ai_generative_ai_type.$touch()"
@change="$refs.aiModel.select(aIModelsPerType[0])"
>
<DropdownItem
v-for="aIType in aITypes"
:key="aIType"
:name="aIType"
:value="aIType"
/>
</Dropdown>
</div>
</div>
<div class="control">
<label class="control__label control__label--small">
{{ $t('selectAIModelForm.AIModel') }}
</label>
<div class="control__elements">
<Dropdown
ref="aiModel"
v-model="values.ai_generative_ai_model"
class="dropdown--floating"
:class="{
'dropdown--error': $v.values.ai_generative_ai_model.$error,
}"
:fixed-items="true"
:show-search="false"
small
@hide="$v.values.ai_generative_ai_model.$touch()"
>
<DropdownItem
v-for="aIType in aIModelsPerType"
:key="aIType"
:name="aIType"
:value="aIType"
/>
</Dropdown>
</div>
</div>
</div>
</template>
<script>
import { mapGetters } from 'vuex'
import { required } from 'vuelidate/lib/validators'
import modal from '@baserow/modules/core/mixins/modal'
import form from '@baserow/modules/core/mixins/form'
export default {
name: 'SelectAIModelForm',
mixins: [form, modal],
props: {
database: {
type: Object,
required: true,
},
},
data() {
return {
allowedValues: ['ai_generative_ai_type', 'ai_generative_ai_model'],
values: {
ai_generative_ai_type: null,
ai_generative_ai_model: null,
},
}
},
computed: {
...mapGetters({
settings: 'settings/get',
}),
// Return the reactive object that can be updated in runtime.
workspace() {
return this.$store.getters['workspace/get'](this.database.workspace.id)
},
aITypes() {
return Object.keys(this.workspace.generative_ai_models_enabled || {})
},
aIModelsPerType() {
return (
this.workspace.generative_ai_models_enabled[
this.values.ai_generative_ai_type
] || []
)
},
},
validations: {
values: {
ai_generative_ai_type: { required },
ai_generative_ai_model: { required },
},
},
}
</script>

4
web-frontend/modules/core/locales/en.json
View file

@ -657,5 +657,9 @@
"unorderedList": "Bullet list",
"code": "Code",
"taskList": "Task list"
},
"selectAIModelForm": {
"AIType": "AI Type",
"AIModel": "AI Model"
}
}

8
web-frontend/modules/core/plugins/clientHandler.js
View file

@ -134,6 +134,14 @@ export class ClientErrorMap {
app.i18n.t('clientHandler.disabledPasswordProviderTitle'),
app.i18n.t('clientHandler.disabledPasswordProviderMessage')
),
ERROR_OUTPUT_PARSER: new ResponseErrorMessage(
app.i18n.t('clientHandler.outputParserTitle'),
app.i18n.t('clientHandler.outputParserDescription')
),
ERROR_GENERATIVE_AI_PROMPT: new ResponseErrorMessage(
app.i18n.t('clientHandler.generateAIPromptTitle'),
app.i18n.t('clientHandler.generateAIPromptDescription')
),
// TODO: Move to enterprise module if possible
ERROR_CANNOT_DISABLE_ALL_AUTH_PROVIDERS: new ResponseErrorMessage(
app.i18n.t('clientHandler.cannotDisableAllAuthProvidersTitle'),

1
web-frontend/modules/database/components/field/FieldFormulaSubForm.vue
View file

@ -16,6 +16,7 @@
$refs.advancedFormulaEditContext.openContext($event)
"
@refresh-formula-type="refreshFormulaType"
@update-formula="values.formula = $event"
>
</FieldFormulaInitialSubForm>
<FormulaAdvancedEditContext

14
web-frontend/modules/database/components/formula/FieldFormulaInitialSubForm.vue
View file

@ -16,6 +16,14 @@
@click="$emit('open-advanced-context', $refs.formulaInput)"
@input="$emit('open-advanced-context', $refs.formulaInput)"
/>
<component
:is="component"
v-for="(component, index) in additionalInputComponents"
:key="index"
:database="database"
:table="table"
@update-formula="$emit('update-formula', $event)"
></component>
<div v-if="loading" class="loading"></div>
<template v-else>
<div v-if="error" class="error formula-field__error">
@ -53,6 +61,7 @@ import form from '@baserow/modules/core/mixins/form'
import fieldSubForm from '@baserow/modules/database/mixins/fieldSubForm'
import FormulaTypeSubForms from '@baserow/modules/database/components/formula/FormulaTypeSubForms'
import { FormulaFieldType } from '@baserow/modules/database/fieldTypes'
export default {
name: 'FieldFormulaInitialSubForm',
@ -94,6 +103,11 @@ export default {
showTypeFormattingOptions() {
return !(this.loading || this.formulaTypeRefreshNeeded || this.error)
},
additionalInputComponents() {
return this.$registry
.get('field', FormulaFieldType.getType())
.getAdditionalFormInputComponents()
},
},
}
</script>

8
web-frontend/modules/database/fieldTypes.js
View file

@ -3591,6 +3591,14 @@ export class FormulaFieldType extends FieldType {
return FieldFormulaSubForm
}
/**
* Can optionally return additional components that are rendered directly below
* the field form formula input.
*/
getAdditionalFormInputComponents() {
return []
}
getIsReadOnly() {
return true
}