Merge branch '320-attachment-filename-filter' into 'develop'

Showing how a filename contains filter could work without larger changes

Closes #320

See merge request bramw/baserow!180

backend/src/baserow/contrib/database/config.py

@@ -71,11 +71,13 @@ class DatabaseConfig(AppConfig):
             EqualViewFilterType, NotEqualViewFilterType, EmptyViewFilterType,
             NotEmptyViewFilterType, DateEqualViewFilterType, DateNotEqualViewFilterType,
             HigherThanViewFilterType, LowerThanViewFilterType, ContainsViewFilterType,
-            ContainsNotViewFilterType, BooleanViewFilterType,
+            FilenameContainsViewFilterType, ContainsNotViewFilterType,
-            SingleSelectEqualViewFilterType, SingleSelectNotEqualViewFilterType
+            BooleanViewFilterType, SingleSelectEqualViewFilterType,
+            SingleSelectNotEqualViewFilterType
         )
         view_filter_type_registry.register(EqualViewFilterType())
         view_filter_type_registry.register(NotEqualViewFilterType())
+        view_filter_type_registry.register(FilenameContainsViewFilterType())
         view_filter_type_registry.register(ContainsViewFilterType())
         view_filter_type_registry.register(ContainsNotViewFilterType())
         view_filter_type_registry.register(HigherThanViewFilterType())

backend/src/baserow/contrib/database/table/models.py

@@ -201,6 +201,13 @@ class TableModelQuerySet(models.QuerySet):
                     model_field
                 )
 
+                view_filter_annotation = view_filter_type.get_annotation(
+                    field_name,
+                    value
+                )
+                if view_filter_annotation:
+                    self = self.annotate(**view_filter_annotation)
+
                 # Depending on filter type we are going to combine the Q either as
                 # AND or as OR.
                 if filter_type == FILTER_TYPE_AND:

backend/src/baserow/contrib/database/views/handler.py

@@ -254,6 +254,13 @@ class ViewHandler:
                 model_field
             )
 
+            view_filter_annotation = view_filter_type.get_annotation(
+                field_name,
+                view_filter.value
+            )
+            if view_filter_annotation:
+                queryset = queryset.annotate(**view_filter_annotation)
+
             # Depending on filter type we are going to combine the Q either as AND or
             # as OR.
             if view.filter_type == FILTER_TYPE_AND:

backend/src/baserow/contrib/database/views/registries.py

@@ -120,6 +120,25 @@ class ViewFilterType(Instance):
 
         raise NotImplementedError('Each must have his own get_filter method.')
 
+    def get_annotation(self, field_name, value):
+        """
+        Optional method allowing this ViewFilterType to annotate the queryset prior to
+        the application of any Q filters returned by ViewFilterType.get_filter.
+
+        Should return a dictionary which can be unpacked into an annotate call or None
+        if you do not wish any annotation to be applied by your filter.
+
+        :param field_name: The name of the field that needs to be filtered.
+        :type field_name: str
+        :param value: The value that the field must be compared to.
+        :type value: str
+        :return: The dict object that will be unpacked into an annotate call or None if
+            no annotation needs to be done.
+        :rtype: None or dict
+        """
+
+        return None
+
 
 class ViewFilterTypeRegistry(Registry):
     """

backend/src/baserow/contrib/database/views/view_filters.py

@@ -6,6 +6,7 @@ from dateutil import parser
 from dateutil.parser import ParserError
 
 from django.db.models import Q, IntegerField, BooleanField
+from django.db.models.expressions import RawSQL
 from django.db.models.fields.related import ManyToManyField, ForeignKey
 from django.contrib.postgres.fields import JSONField
 
@@ -62,6 +63,58 @@ class NotEqualViewFilterType(NotViewFilterTypeMixin, EqualViewFilterType):
     type = 'not_equal'
 
 
+class FilenameContainsViewFilterType(ViewFilterType):
+    """
+    The filename contains filter checks if the filename's visible name contains the
+    provided filter value. It is only compatible with fields.JSONField which contain
+    a list of File JSON Objects.
+    """
+
+    type = 'filename_contains'
+    compatible_field_types = [
+        FileFieldType.type
+    ]
+
+    def get_annotation(self, field_name, value):
+        value = value.strip()
+
+        # If an empty value has been provided we do not want to filter at all.
+        if value == '':
+            return None
+
+        # It is not possible to use Django's ORM to query for if one item in a JSONB
+        # list has has a key which contains a specified value.
+        #
+        # The closest thing the Django ORM provides is:
+        #   queryset.filter(your_json_field__contains=[{"key":"value"}])
+        # However this is an exact match, so in the above example [{"key":"value_etc"}]
+        # would not match the filter.
+        #
+        # Instead we have to resort to RawSQL to use various built in PostgreSQL JSON
+        # Array manipulation functions to be able to 'iterate' over a JSONB list
+        # performing `like` on individual keys in said list.
+        num_files_with_name_like_value = f"""
+            EXISTS(
+                SELECT attached_files ->> 'visible_name'
+                FROM JSONB_ARRAY_ELEMENTS("{field_name}") as attached_files
+                WHERE UPPER(attached_files ->> 'visible_name') LIKE UPPER(%s)
+            )
+        """
+        query = RawSQL(num_files_with_name_like_value, params=[f"%{value}%"],
+                       output_field=BooleanField())
+        return {f"{field_name}_matches_visible_names": query}
+
+    def get_filter(self, field_name, value, model_field):
+        value = value.strip()
+
+        # If an empty value has been provided we do not want to filter at all.
+        if value == '':
+            return Q()
+
+        # Check if the model_field has a file which matches the provided filter value.
+        return Q(**{f'{field_name}_matches_visible_names': True})
+
+
 class ContainsViewFilterType(ViewFilterType):
     """
     The contains filter checks if the field value contains the provided filter value.

backend/tests/baserow/contrib/database/view/test_view_filters.py

@@ -1329,3 +1329,67 @@ def test_not_empty_filter_type(data_fixture):
     filter.field = single_select_field
     filter.save()
     assert handler.apply_filters(grid_view, model.objects.all()).get().id == row_2.id
+
+
+@pytest.mark.django_db
+def test_filename_contains_filter_type(data_fixture):
+    user = data_fixture.create_user()
+    table = data_fixture.create_database_table(user=user)
+    grid_view = data_fixture.create_grid_view(table=table)
+    file_field = data_fixture.create_file_field(table=table)
+
+    handler = ViewHandler()
+    model = table.get_model()
+
+    row = model.objects.create(**{
+        f'field_{file_field.id}': [{'visible_name': 'test_file.png'}],
+    })
+    row_with_multiple_files = model.objects.create(**{
+        f'field_{file_field.id}': [
+            {'visible_name': 'test.doc'},
+            {'visible_name': 'test.txt'}
+        ],
+    })
+    row_with_no_files = model.objects.create(**{
+        f'field_{file_field.id}': [],
+    })
+
+    filter = data_fixture.create_view_filter(
+        view=grid_view,
+        field=file_field,
+        type='filename_contains',
+        value='test_file.png'
+    )
+    ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
+    assert len(ids) == 1
+    assert row.id in ids
+
+    filter.value = '.jpg'
+    filter.save()
+    ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
+    assert len(ids) == 0
+
+    filter.value = '.png'
+    filter.save()
+    ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
+    assert len(ids) == 1
+    assert row.id in ids
+
+    filter.value = 'test.'
+    filter.save()
+    ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
+    assert len(ids) == 1
+    assert row_with_multiple_files.id in ids
+
+    filter.value = ''
+    filter.save()
+    ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
+    assert len(ids) == 3
+    assert row.id in ids
+    assert row_with_multiple_files.id in ids
+    assert row_with_no_files.id in ids
+
+    results = model.objects.all().filter_by_fields_object(filter_object={
+        f'filter__field_{file_field.id}__filename_contains': ['.png'],
+    }, filter_type='AND')
+    assert len(results) == 1

changelog.md

@@ -24,6 +24,7 @@
 * Made it possible for the admin to disable new signups.
 * Reduced the amount of queries when using the link row field.
 * Respect the date format when converting to a date field.
+* Added a field type filename contains filter.
 
 ## Released (2021-02-04)
 

web-frontend/modules/database/fieldTypes.js

@@ -327,6 +327,10 @@ export class LongTextFieldType extends FieldType {
     return RowEditFieldLongText
   }
 
+  getEmptyValue(field) {
+    return ''
+  }
+
   getSort(name, order) {
     return (a, b) => {
       const stringA = a[name] === null ? '' : '' + a[name]

web-frontend/modules/database/plugin.js

@@ -18,6 +18,7 @@ import {
   DateEqualViewFilterType,
   DateNotEqualViewFilterType,
   ContainsViewFilterType,
+  FilenameContainsViewFilterType,
   ContainsNotViewFilterType,
   HigherThanViewFilterType,
   LowerThanViewFilterType,
@@ -53,6 +54,7 @@ export default ({ store, app }) => {
   app.$registry.register('viewFilter', new DateEqualViewFilterType())
   app.$registry.register('viewFilter', new DateNotEqualViewFilterType())
   app.$registry.register('viewFilter', new ContainsViewFilterType())
+  app.$registry.register('viewFilter', new FilenameContainsViewFilterType())
   app.$registry.register('viewFilter', new ContainsNotViewFilterType())
   app.$registry.register('viewFilter', new HigherThanViewFilterType())
   app.$registry.register('viewFilter', new LowerThanViewFilterType())

web-frontend/modules/database/viewFilters.js

@@ -140,6 +140,45 @@ export class NotEqualViewFilterType extends ViewFilterType {
   }
 }
 
+export class FilenameContainsViewFilterType extends ViewFilterType {
+  static getType() {
+    return 'filename_contains'
+  }
+
+  getName() {
+    return 'filename contains'
+  }
+
+  getInputComponent() {
+    return ViewFilterTypeText
+  }
+
+  getCompatibleFieldTypes() {
+    return ['file']
+  }
+
+  matches(rowValue, filterValue) {
+    filterValue = filterValue.toString().toLowerCase().trim()
+
+    if (filterValue === '') {
+      return true
+    }
+
+    for (let i = 0; i < rowValue.length; i++) {
+      const visibleName = rowValue[i].visible_name
+        .toString()
+        .toLowerCase()
+        .trim()
+
+      if (visibleName.includes(filterValue)) {
+        return true
+      }
+    }
+
+    return false
+  }
+}
+
 export class ContainsViewFilterType extends ViewFilterType {
   static getType() {
     return 'contains'