Merge branch '3453-implement-distribution-aggregation-in-the-local-baserow-aggregate-rows-service' into 'develop'

Draft: Resolve "Implement distribution aggregation in the Local Baserow aggregate rows service."

Closes #3453

See merge request baserow/baserow!3157

backend/src/baserow/contrib/database/apps.py

@@ -243,6 +243,7 @@ class DatabaseConfig(AppConfig):
             CheckedFieldAggregationType,
             CheckedPercentageFieldAggregationType,
             CountFieldAggregationType,
+            DistributionFieldAggregationType,
             EarliestDateFieldAggregationType,
             EmptyCountFieldAggregationType,
             EmptyPercentageFieldAggregationType,
@@ -279,6 +280,7 @@ class DatabaseConfig(AppConfig):
         field_aggregation_registry.register(StdDevFieldAggregationType())
         field_aggregation_registry.register(VarianceFieldAggregationType())
         field_aggregation_registry.register(MedianFieldAggregationType())
+        field_aggregation_registry.register(DistributionFieldAggregationType())
 
         from .fields.field_converters import (
             AutonumberFieldConverter,

backend/src/baserow/contrib/database/fields/field_aggregations.py

@@ -35,6 +35,7 @@ from baserow.contrib.database.formula.types.formula_types import (
 from baserow.contrib.database.views.view_aggregations import (
     AverageViewAggregationType,
     CountViewAggregationType,
+    DistributionViewAggregationType,
     EmptyCountViewAggregationType,
     MaxViewAggregationType,
     MedianViewAggregationType,
@@ -341,3 +342,14 @@ class MedianFieldAggregationType(FieldAggregationType):
     type = "median"
     raw_type = MedianViewAggregationType
     compatible_field_types = raw_type.compatible_field_types
+
+
+class DistributionFieldAggregationType(FieldAggregationType):
+    """
+    Compute the distribution of values
+    """
+
+    type = "distribution"
+    result_type = "array"
+    raw_type = DistributionViewAggregationType
+    compatible_field_types = raw_type.compatible_field_types

backend/src/baserow/contrib/database/fields/registries.py

@@ -52,7 +52,10 @@ from baserow.contrib.database.views.exceptions import (
     AggregationTypeAlreadyRegistered,
     AggregationTypeDoesNotExist,
 )
-from baserow.contrib.database.views.utils import AnnotatedAggregation
+from baserow.contrib.database.views.utils import (
+    AnnotatedAggregation,
+    DistributionAggregation,
+)
 from baserow.core.registries import ImportExportConfig
 from baserow.core.registry import (
     APIUrlsInstanceMixin,
@@ -2171,7 +2174,7 @@ class FieldConverter(Instance):
         """
 
         raise NotImplementedError(
-            "Each field converter must have an alter_field " "method."
+            "Each field converter must have an alter_field method."
         )
 
 
@@ -2235,6 +2238,7 @@ class FieldAggregationType(Instance):
             raise IncompatibleField()
 
         aggregation_dict = self._get_aggregation_dict(queryset, model_field, field)
+        distribution_dict = self._get_distribution_dict(queryset, model_field, field)
 
         # Check if the returned aggregations contain a `AnnotatedAggregation`,
         # and if so, apply the annotations and only keep the actual aggregation in
@@ -2246,8 +2250,14 @@ class FieldAggregationType(Instance):
                 aggregation_dict[key] = value.aggregation
 
         results = queryset.aggregate(**aggregation_dict)
+        results.update(distribution_dict)
+
+        raw_aggregation_result = results.get(
+            f"{field.db_column}_raw", results.get(field.db_column)
+        )
         return self._compute_final_aggregation(
-            results[f"{field.db_column}_raw"], results.get("total", None)
+            raw_aggregation_result,
+            results.get("total", None),
         )
 
     def field_is_compatible(self, field: "Field") -> bool:
@@ -2259,8 +2269,6 @@ class FieldAggregationType(Instance):
         :return: True if the field is compatible, False otherwise.
         """
 
-        from baserow.contrib.database.fields.registries import field_type_registry
-
         field_type = field_type_registry.get_by_model(field.specific_class)
 
         return any(
@@ -2280,6 +2288,36 @@ class FieldAggregationType(Instance):
 
         return self.raw_type().get_aggregation(field.db_column, model_field, field)
 
+    def _get_distribution_dict(
+        self, queryset: QuerySet, model_field: DjangoField, field: Field
+    ) -> dict[str, any]:
+        """
+        Returns a dictionary defining the distributions for the queryset.aggregate
+        call.
+
+        :param queryset: The queryset to select only the rows that should
+            be aggregated.
+        :param model_field: The Django model field of the field that
+            the aggregation is for.
+        :param field: The field that the aggregation is for.
+        :return:
+        """
+
+        aggregation = self._get_raw_aggregation(model_field, field.specific)
+        if not isinstance(aggregation, DistributionAggregation):
+            return {}
+
+        formatted = []
+        raw_calculation = aggregation.calculate(queryset.all())
+        for result in raw_calculation:
+            formatted.append(
+                {
+                    "value": result[0],
+                    "count": result[1],
+                }
+            )
+        return {field.db_column: formatted}
+
     def _get_aggregation_dict(
         self,
         queryset: QuerySet,
@@ -2288,7 +2326,7 @@ class FieldAggregationType(Instance):
         include_agg_type=False,
     ) -> dict:
         """
-        Returns a dictinary defining the aggregation for the queryset.aggregate
+        Returns a dictionary defining the aggregation for the queryset.aggregate
         call.
 
         :param queryset: The queryset to select only the rows that should
@@ -2301,6 +2339,9 @@ class FieldAggregationType(Instance):
         aggregation = self._get_raw_aggregation(model_field, field.specific)
         key = f"{field.db_column}_{self.type}" if include_agg_type else field.db_column
         aggregation_dict = {f"{key}_raw": aggregation}
+        if isinstance(aggregation, DistributionAggregation):
+            return {}
+
         # Check if the returned aggregations contain a `AnnotatedAggregation`,
         # and if so, apply the annotations and only keep the actual aggregation in
         # the dict. This is needed because some aggregations require annotated values
@@ -2314,7 +2355,9 @@ class FieldAggregationType(Instance):
 
         return aggregation_dict
 
-    def _compute_final_aggregation(self, raw_aggregation_result, total_count: int):
+    def _compute_final_aggregation(
+        self, raw_aggregation_result, total_count: Optional[int] = None
+    ):
         """
         For field aggregation types that require 'with_total' the number of all
         rows to compute the final number this method will be called to compute

backend/src/baserow/contrib/integrations/local_baserow/service_types.py

@@ -64,9 +64,6 @@ from baserow.contrib.database.views.exceptions import (
 )
 from baserow.contrib.database.views.models import DEFAULT_SORT_TYPE_KEY
 from baserow.contrib.database.views.service import ViewService
-from baserow.contrib.database.views.view_aggregations import (
-    DistributionViewAggregationType,
-)
 from baserow.contrib.integrations.local_baserow.api.serializers import (
     LocalBaserowTableServiceFieldMappingSerializer,
 )
@@ -1255,10 +1252,6 @@ class LocalBaserowAggregateRowsUserServiceType(
     dispatch_type = DispatchTypes.DISPATCH_DATA_SOURCE
     serializer_mixins = LocalBaserowTableServiceFilterableMixin.mixin_serializer_mixins
 
-    # Local Baserow aggregate rows does not currently support the distribution
-    # aggregation type, this will be resolved in a future release.
-    unsupported_aggregation_types = [DistributionViewAggregationType.type]
-
     def get_schema_name(self, service: LocalBaserowAggregateRows) -> str:
         """
         The Local Baserow aggregation schema name added to the `title` in
@@ -1291,24 +1284,43 @@ class LocalBaserowAggregateRowsUserServiceType(
         if not service.field or not service.aggregation_type:
             return None
 
+        result_key = "results" if self.returns_list(service) else "result"
+
         # The `result` must be an allowed field, otherwise we have no schema.
-        if allowed_fields is not None and "result" not in allowed_fields:
+        if allowed_fields is not None and result_key not in allowed_fields:
             return {}
 
         # Pluck out the aggregation type which this service uses. We'll use its
         # `result_type` to inform the schema what the expected `result` format is.
         aggregation_type = field_aggregation_registry.get(service.aggregation_type)
 
-        return {
-            "title": self.get_schema_name(service),
-            "type": "object",
-            "properties": {
-                "result": {
+        schema = {"title": self.get_schema_name(service)}
+
+        if aggregation_type.result_type == "array":
+            schema["type"] = "array"
+            schema["items"] = {
+                "type": "object",
+                "properties": {
+                    "value": {
+                        "title": f"{service.field.name} value",
+                        "type": "string",
+                    },
+                    "count": {
+                        "title": f"{service.field.name} distribution",
+                        "type": "number",
+                    },
+                },
+            }
+        else:
+            schema["type"] = "object"
+            schema["properties"] = {
+                f"{result_key}": {
                     "title": f"{service.field.name} result",
                     "type": aggregation_type.result_type,
                 }
-            },
-        }
+            }
+
+        return schema
 
     def get_context_data(
         self,
@@ -1377,6 +1389,10 @@ class LocalBaserowAggregateRowsUserServiceType(
         field_id: int
         aggregation_type: str
 
+    def returns_list(self, service: LocalBaserowAggregateRows) -> bool:
+        aggregation_type = field_aggregation_registry.get(service.aggregation_type)
+        return aggregation_type.result_type == "array"
+
     def prepare_values(
         self,
         values: Dict[str, Any],
@@ -1403,13 +1419,6 @@ class LocalBaserowAggregateRowsUserServiceType(
             "aggregation_type", getattr(instance, "aggregation_type", "")
         )
 
-        if aggregation_type in self.unsupported_aggregation_types:
-            raise DRFValidationError(
-                detail=f"The {aggregation_type} aggregation type "
-                "is not currently supported.",
-                code="unsupported_aggregation_type",
-            )
-
         if "table" in values:
             # Reset the field if the table has changed
             if (
@@ -1563,9 +1572,12 @@ class LocalBaserowAggregateRowsUserServiceType(
         :return: Aggregations.
         """
 
-        only_field_names = self.get_used_field_names(service, dispatch_context)
-        if only_field_names and "result" not in only_field_names:
-            return {"data": {"result": None}}
+        result_key = "results" if self.returns_list(service) else "result"
+
+        # TODO: resolve
+        # only_field_names = self.get_used_field_names(service, dispatch_context)
+        # if only_field_names and result_key not in only_field_names:
+        #     return {"data": {result_key: []}}
 
         try:
             table = resolved_values["table"]
@@ -1583,7 +1595,7 @@ class LocalBaserowAggregateRowsUserServiceType(
             result = agg_type.aggregate(queryset, model_field, field)
 
             return {
-                "data": {"result": result},
+                "data": {result_key: result},
                 "baserow_table_model": model,
             }
         except DjangoFieldDoesNotExist as ex:
@@ -1615,10 +1627,8 @@ class LocalBaserowAggregateRowsUserServiceType(
         Returns the usual properties for this service type.
         """
 
-        if path[0] == "result":
-            return ["result"]
-
-        return []
+        # TODO: confirm this is right
+        return ["result", "value", "count"]
 
 
 class LocalBaserowGetRowUserServiceType(

backend/tests/baserow/contrib/integrations/local_baserow/service_types/test_aggregate_rows_service_type.py

@@ -587,29 +587,3 @@ def test_local_baserow_aggregate_rows_dispatch_data_field_type_not_compatible_an
         exc.value.args[0] == f"The field with ID {field.id} is not compatible "
         f"with the aggregation type {service.aggregation_type}"
     )
-
-
-@pytest.mark.django_db
-def test_create_local_baserow_aggregate_rows_service_with_unsupported_aggregation_type(
-    data_fixture,
-):
-    user = data_fixture.create_user()
-    page = data_fixture.create_builder_page(user=user)
-    dashboard = page.builder
-    table = data_fixture.create_database_table(user=user)
-    field = data_fixture.create_number_field(table=table)
-    view = data_fixture.create_grid_view(user=user, table=table)
-    integration = data_fixture.create_local_baserow_integration(
-        application=dashboard, user=user
-    )
-    service = data_fixture.create_local_baserow_aggregate_rows_service(
-        table=table, field=field, integration=integration
-    )
-    service_type = service.get_type()
-    unsupported_agg_type = service_type.unsupported_aggregation_types[0]
-
-    with pytest.raises(
-        ValidationError,
-        match=f"The {unsupported_agg_type} aggregation type is not currently supported.",
-    ):
-        service_type.prepare_values({"aggregation_type": unsupported_agg_type}, user)

web-frontend/modules/builder/components/dataSource/DataSourceDropdown.vue

@@ -86,7 +86,7 @@ export default {
         return dataSource.name
       }
       const service = this.$registry.get('service', dataSource.type)
-      const suffix = service.returnsList
+      const suffix = service.returnsList({ service: dataSource })
         ? this.$t('integrationsCommon.multipleRows')
         : this.$t('integrationsCommon.singleRow')
       return `${dataSource.name} (${suffix})`

web-frontend/modules/builder/components/elements/components/CollectionElementHeader.vue

@@ -1,7 +1,7 @@
 <template>
   <component
     :is="serviceType.adhocHeaderComponent"
-    v-if="dataSource"
+    v-if="dataSource && elementType.adhocRefinementsSupported(dataSource)"
     class="collection-element__header margin-bottom-1"
     :sortable-properties="
       elementType.adhocSortableProperties(element, dataSource)

web-frontend/modules/builder/components/elements/components/forms/general/RecordSelectorElementForm.vue

@@ -190,15 +190,17 @@ export default {
       if (this.localDataSources === null) {
         return null
       }
-      return this.localDataSources.filter(
-        (dataSource) =>
-          this.$registry.get('service', dataSource.type).returnsList
+      return this.localDataSources.filter((dataSource) =>
+        this.$registry
+          .get('service', dataSource.type)
+          .returnsList({ service: dataSource })
       )
     },
     listSharedDataSources() {
-      return this.sharedDataSources.filter(
-        (dataSource) =>
-          this.$registry.get('service', dataSource.type).returnsList
+      return this.sharedDataSources.filter((dataSource) =>
+        this.$registry
+          .get('service', dataSource.type)
+          .returnsList({ service: dataSource })
       )
     },
   },

web-frontend/modules/builder/dataProviderTypes.js

@@ -117,7 +117,7 @@ export class DataSourceDataProviderType extends DataProviderType {
 
     const serviceType = this.app.$registry.get('service', dataSource.type)
 
-    if (serviceType.returnsList) {
+    if (serviceType.returnsList({ service: dataSource })) {
       return dataSourceContents[dataSource.id]?.results
     } else {
       return dataSourceContents[dataSource.id]

web-frontend/modules/builder/elementTypeMixins.js

@@ -1,4 +1,5 @@
 import { ELEMENT_EVENTS, SHARE_TYPES } from '@baserow/modules/builder/enums'
+import { LocalBaserowAggregateRowsServiceType } from '@baserow/modules/integrations/serviceTypes'
 
 export const ContainerElementTypeMixin = (Base) =>
   class extends Base {
@@ -54,6 +55,18 @@ export const CollectionElementTypeMixin = (Base) =>
   class extends Base {
     isCollectionElement = true
 
+    /**
+     * Response for returning whether this collection element, using this dataSource,
+     * supports adhoc filtering (e.g. filtering, sorting, searching). Normally they
+     * will, but if the collection element is used by an aggregation returning a list
+     * of records, then it will not.
+     * @param dataSource
+     */
+    adhocRefinementsSupported(dataSource) {
+      const serviceType = this.app.$registry.get('service', dataSource.type)
+      return serviceType.adhocRefinementsSupported
+    }
+
     /**
      * A helper function responsible for returning this collection element's
      * schema properties.
@@ -326,7 +339,10 @@ export const CollectionElementTypeMixin = (Base) =>
       const serviceType = this.app.$registry.get('service', dataSource.type)
 
       // If the data source type doesn't return a list, we should have a schema_property
-      if (!serviceType.returnsList && !parentWithDataSource.schema_property) {
+      if (
+        !serviceType.returnsList({ service: dataSource }) &&
+        !parentWithDataSource.schema_property
+      ) {
         return true
       }
 

web-frontend/modules/builder/mixins/collectionElementForm.js

@@ -49,7 +49,13 @@ export default {
      * @returns {boolean} - Whether the property options are available.
      */
     propertyOptionsAvailable() {
-      return this.selectedDataSource && this.selectedDataSourceReturnsList
+      const { element } = this.applicationContext
+      const elementType = this.$registry.get('element', element.type)
+      return (
+        this.selectedDataSource &&
+        this.selectedDataSourceReturnsList &&
+        elementType.adhocRefinementsSupported(this.selectedDataSource)
+      )
     },
     /**
      * In collection element forms, the ability to view paging options
@@ -150,7 +156,9 @@ export default {
       return this.$registry.get('service', this.selectedDataSource.type)
     },
     selectedDataSourceReturnsList() {
-      return this.selectedDataSourceType?.returnsList
+      return this.selectedDataSourceType?.returnsList({
+        service: this.selectedDataSource,
+      })
     },
     maxItemPerPage() {
       if (!this.selectedDataSourceType) {

web-frontend/modules/builder/store/elementContent.js

@@ -179,7 +179,10 @@ const actions = {
 
     // We have a data source, but if it doesn't return a list,
     // it needs to have a `schema_property` to work correctly.
-    if (!serviceType.returnsList && element.schema_property === null) {
+    if (
+      !serviceType.returnsList({ service: dataSource }) &&
+      element.schema_property === null
+    ) {
       // If we previously had a list data source, we might have content,
       // so rather than leave the content *until a schema property is set*,
       // clear it.
@@ -246,7 +249,7 @@ const actions = {
           })
         }
 
-        if (serviceType.returnsList) {
+        if (serviceType.returnsList({ service: dataSource })) {
           // The service type returns a list of results, we'll set the content
           // using the results key and set the range for future paging.
           commit('SET_CONTENT', {

web-frontend/modules/core/serviceTypes.js

@@ -29,9 +29,11 @@ export class ServiceType extends Registerable {
   }
 
   /**
-   * Whether the service returns a collection of records.
+   * Whether the service returns a collection of records. Most of the time this
+   * will simply return true or false, but a service can be provided to determine
+   * if a specific service type returns a list of records.
    */
-  get returnsList() {
+  returnsList({ service }) {
     return false
   }
 

web-frontend/modules/dashboard/components/data_source/AggregateRowsDataSourceForm.vue

@@ -112,9 +112,6 @@
             :key="viewAggregation.getType()"
             :name="viewAggregation.getName()"
             :value="viewAggregation.getType()"
-            :disabled="
-              unsupportedAggregationTypes.includes(viewAggregation.getType())
-            "
           >
           </DropdownItem>
         </Dropdown>
@@ -227,10 +224,6 @@ export default {
     aggregationTypeNames() {
       return this.viewAggregationTypes.map((aggType) => aggType.getType())
     },
-    unsupportedAggregationTypes() {
-      return this.$registry.get('service', 'local_baserow_aggregate_rows')
-        .unsupportedAggregationTypes
-    },
   },
   watch: {
     dataSource: {

web-frontend/modules/integrations/localBaserow/components/services/LocalBaserowAggregateRowsForm.vue

@@ -45,9 +45,6 @@
               :key="viewAggregation.getType()"
               :name="viewAggregation.getName()"
               :value="viewAggregation.getType()"
-              :disabled="
-                unsupportedAggregationTypes.includes(viewAggregation.getType())
-              "
             >
             </DropdownItem>
           </Dropdown>
@@ -130,10 +127,6 @@ export default {
     }
   },
   computed: {
-    unsupportedAggregationTypes() {
-      return this.$registry.get('service', 'local_baserow_aggregate_rows')
-        .unsupportedAggregationTypes
-    },
     viewAggregationTypes() {
       const selectedField = this.tableFields.find(
         (field) => field.id === this.values.field_id

web-frontend/modules/integrations/serviceTypes.js

@@ -8,6 +8,11 @@ import LocalBaserowAdhocHeader from '@baserow/modules/integrations/localBaserow/
 import { DistributionViewAggregationType } from '@baserow/modules/database/viewAggregationTypes'
 
 export class LocalBaserowTableServiceType extends ServiceType {
+  // Determines whether collection elements with data sources using this
+  // service are allowed to perform adhoc refinements (filtering, sorting, searching).
+  // By default, they cannot, only the list rows service type can.
+  adhocRefinementsSupported = false
+
   get integrationType() {
     return this.app.$registry.get(
       'integration',
@@ -112,6 +117,8 @@ export class LocalBaserowGetRowServiceType extends LocalBaserowTableServiceType
 }
 
 export class LocalBaserowListRowsServiceType extends LocalBaserowTableServiceType {
+  adhocRefinementsSupported = true
+
   static getType() {
     return 'local_baserow_list_rows'
   }
@@ -131,7 +138,7 @@ export class LocalBaserowListRowsServiceType extends LocalBaserowTableServiceTyp
     return LocalBaserowAdhocHeader
   }
 
-  get returnsList() {
+  returnsList({ service }) {
     return true
   }
 
@@ -234,6 +241,15 @@ export class LocalBaserowAggregateRowsServiceType extends LocalBaserowTableServi
     return 'local_baserow_aggregate_rows'
   }
 
+  returnsList({ service }) {
+    // TODO: store this in the registry
+    return service.aggregation_type === 'distribution'
+  }
+
+  get maxResultLimit() {
+    return 100
+  }
+
   get name() {
     return this.app.i18n.t('serviceType.localBaserowAggregateRows')
   }
@@ -242,12 +258,19 @@ export class LocalBaserowAggregateRowsServiceType extends LocalBaserowTableServi
     return LocalBaserowAggregateRowsForm
   }
 
-  /**
-   * Local Baserow aggregate rows does not currently support the distribution
-   * aggregation type, this will be resolved in a future release.
-   */
-  get unsupportedAggregationTypes() {
-    return [DistributionViewAggregationType.getType()]
+  getDefaultCollectionFields(service) {
+    return Object.keys(service.schema.items.properties)
+      .filter((field) => field !== 'id')
+      .map((field) => {
+        const outputType = 'text'
+        const valueFormula = `get('current_record.${field}')`
+        return {
+          name: service.schema.items.properties[field].title,
+          type: outputType,
+          value: valueFormula,
+          id: uuid(),
+        }
+      })
   }
 
   getResult(service, data) {