1
0
Fork 0
mirror of https://gitlab.com/bramw/baserow.git synced 2025-04-10 07:37:30 +00:00

Proof of concept implementation showing how a filename contains filter could work given that file metadata is stored as a JSONField list.

This commit is contained in:
Nigel Gott 2021-02-26 12:40:05 +00:00 committed by Bram Wiepjes
parent 4534369709
commit 8889a6fb2f
10 changed files with 200 additions and 2 deletions
backend
src/baserow/contrib/database
tests/baserow/contrib/database/view
changelog.md
web-frontend/modules/database

View file

@ -71,11 +71,13 @@ class DatabaseConfig(AppConfig):
EqualViewFilterType, NotEqualViewFilterType, EmptyViewFilterType,
NotEmptyViewFilterType, DateEqualViewFilterType, DateNotEqualViewFilterType,
HigherThanViewFilterType, LowerThanViewFilterType, ContainsViewFilterType,
ContainsNotViewFilterType, BooleanViewFilterType,
SingleSelectEqualViewFilterType, SingleSelectNotEqualViewFilterType
FilenameContainsViewFilterType, ContainsNotViewFilterType,
BooleanViewFilterType, SingleSelectEqualViewFilterType,
SingleSelectNotEqualViewFilterType
)
view_filter_type_registry.register(EqualViewFilterType())
view_filter_type_registry.register(NotEqualViewFilterType())
view_filter_type_registry.register(FilenameContainsViewFilterType())
view_filter_type_registry.register(ContainsViewFilterType())
view_filter_type_registry.register(ContainsNotViewFilterType())
view_filter_type_registry.register(HigherThanViewFilterType())

View file

@ -201,6 +201,13 @@ class TableModelQuerySet(models.QuerySet):
model_field
)
view_filter_annotation = view_filter_type.get_annotation(
field_name,
value
)
if view_filter_annotation:
self = self.annotate(**view_filter_annotation)
# Depending on filter type we are going to combine the Q either as
# AND or as OR.
if filter_type == FILTER_TYPE_AND:

View file

@ -254,6 +254,13 @@ class ViewHandler:
model_field
)
view_filter_annotation = view_filter_type.get_annotation(
field_name,
view_filter.value
)
if view_filter_annotation:
queryset = queryset.annotate(**view_filter_annotation)
# Depending on filter type we are going to combine the Q either as AND or
# as OR.
if view.filter_type == FILTER_TYPE_AND:

View file

@ -120,6 +120,25 @@ class ViewFilterType(Instance):
raise NotImplementedError('Each must have his own get_filter method.')
def get_annotation(self, field_name, value):
"""
Optional method allowing this ViewFilterType to annotate the queryset prior to
the application of any Q filters returned by ViewFilterType.get_filter.
Should return a dictionary which can be unpacked into an annotate call or None
if you do not wish any annotation to be applied by your filter.
:param field_name: The name of the field that needs to be filtered.
:type field_name: str
:param value: The value that the field must be compared to.
:type value: str
:return: The dict object that will be unpacked into an annotate call or None if
no annotation needs to be done.
:rtype: None or dict
"""
return None
class ViewFilterTypeRegistry(Registry):
"""

View file

@ -6,6 +6,7 @@ from dateutil import parser
from dateutil.parser import ParserError
from django.db.models import Q, IntegerField, BooleanField
from django.db.models.expressions import RawSQL
from django.db.models.fields.related import ManyToManyField, ForeignKey
from django.contrib.postgres.fields import JSONField
@ -62,6 +63,58 @@ class NotEqualViewFilterType(NotViewFilterTypeMixin, EqualViewFilterType):
type = 'not_equal'
class FilenameContainsViewFilterType(ViewFilterType):
"""
The filename contains filter checks if the filename's visible name contains the
provided filter value. It is only compatible with fields.JSONField which contain
a list of File JSON Objects.
"""
type = 'filename_contains'
compatible_field_types = [
FileFieldType.type
]
def get_annotation(self, field_name, value):
value = value.strip()
# If an empty value has been provided we do not want to filter at all.
if value == '':
return None
# It is not possible to use Django's ORM to query for if one item in a JSONB
# list has has a key which contains a specified value.
#
# The closest thing the Django ORM provides is:
# queryset.filter(your_json_field__contains=[{"key":"value"}])
# However this is an exact match, so in the above example [{"key":"value_etc"}]
# would not match the filter.
#
# Instead we have to resort to RawSQL to use various built in PostgreSQL JSON
# Array manipulation functions to be able to 'iterate' over a JSONB list
# performing `like` on individual keys in said list.
num_files_with_name_like_value = f"""
EXISTS(
SELECT attached_files ->> 'visible_name'
FROM JSONB_ARRAY_ELEMENTS("{field_name}") as attached_files
WHERE UPPER(attached_files ->> 'visible_name') LIKE UPPER(%s)
)
"""
query = RawSQL(num_files_with_name_like_value, params=[f"%{value}%"],
output_field=BooleanField())
return {f"{field_name}_matches_visible_names": query}
def get_filter(self, field_name, value, model_field):
value = value.strip()
# If an empty value has been provided we do not want to filter at all.
if value == '':
return Q()
# Check if the model_field has a file which matches the provided filter value.
return Q(**{f'{field_name}_matches_visible_names': True})
class ContainsViewFilterType(ViewFilterType):
"""
The contains filter checks if the field value contains the provided filter value.

View file

@ -1329,3 +1329,67 @@ def test_not_empty_filter_type(data_fixture):
filter.field = single_select_field
filter.save()
assert handler.apply_filters(grid_view, model.objects.all()).get().id == row_2.id
@pytest.mark.django_db
def test_filename_contains_filter_type(data_fixture):
user = data_fixture.create_user()
table = data_fixture.create_database_table(user=user)
grid_view = data_fixture.create_grid_view(table=table)
file_field = data_fixture.create_file_field(table=table)
handler = ViewHandler()
model = table.get_model()
row = model.objects.create(**{
f'field_{file_field.id}': [{'visible_name': 'test_file.png'}],
})
row_with_multiple_files = model.objects.create(**{
f'field_{file_field.id}': [
{'visible_name': 'test.doc'},
{'visible_name': 'test.txt'}
],
})
row_with_no_files = model.objects.create(**{
f'field_{file_field.id}': [],
})
filter = data_fixture.create_view_filter(
view=grid_view,
field=file_field,
type='filename_contains',
value='test_file.png'
)
ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
assert len(ids) == 1
assert row.id in ids
filter.value = '.jpg'
filter.save()
ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
assert len(ids) == 0
filter.value = '.png'
filter.save()
ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
assert len(ids) == 1
assert row.id in ids
filter.value = 'test.'
filter.save()
ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
assert len(ids) == 1
assert row_with_multiple_files.id in ids
filter.value = ''
filter.save()
ids = [r.id for r in handler.apply_filters(grid_view, model.objects.all()).all()]
assert len(ids) == 3
assert row.id in ids
assert row_with_multiple_files.id in ids
assert row_with_no_files.id in ids
results = model.objects.all().filter_by_fields_object(filter_object={
f'filter__field_{file_field.id}__filename_contains': ['.png'],
}, filter_type='AND')
assert len(results) == 1

View file

@ -24,6 +24,7 @@
* Made it possible for the admin to disable new signups.
* Reduced the amount of queries when using the link row field.
* Respect the date format when converting to a date field.
* Added a field type filename contains filter.
## Released (2021-02-04)

View file

@ -327,6 +327,10 @@ export class LongTextFieldType extends FieldType {
return RowEditFieldLongText
}
getEmptyValue(field) {
return ''
}
getSort(name, order) {
return (a, b) => {
const stringA = a[name] === null ? '' : '' + a[name]

View file

@ -18,6 +18,7 @@ import {
DateEqualViewFilterType,
DateNotEqualViewFilterType,
ContainsViewFilterType,
FilenameContainsViewFilterType,
ContainsNotViewFilterType,
HigherThanViewFilterType,
LowerThanViewFilterType,
@ -53,6 +54,7 @@ export default ({ store, app }) => {
app.$registry.register('viewFilter', new DateEqualViewFilterType())
app.$registry.register('viewFilter', new DateNotEqualViewFilterType())
app.$registry.register('viewFilter', new ContainsViewFilterType())
app.$registry.register('viewFilter', new FilenameContainsViewFilterType())
app.$registry.register('viewFilter', new ContainsNotViewFilterType())
app.$registry.register('viewFilter', new HigherThanViewFilterType())
app.$registry.register('viewFilter', new LowerThanViewFilterType())

View file

@ -140,6 +140,45 @@ export class NotEqualViewFilterType extends ViewFilterType {
}
}
export class FilenameContainsViewFilterType extends ViewFilterType {
static getType() {
return 'filename_contains'
}
getName() {
return 'filename contains'
}
getInputComponent() {
return ViewFilterTypeText
}
getCompatibleFieldTypes() {
return ['file']
}
matches(rowValue, filterValue) {
filterValue = filterValue.toString().toLowerCase().trim()
if (filterValue === '') {
return true
}
for (let i = 0; i < rowValue.length; i++) {
const visibleName = rowValue[i].visible_name
.toString()
.toLowerCase()
.trim()
if (visibleName.includes(filterValue)) {
return true
}
}
return false
}
}
export class ContainsViewFilterType extends ViewFilterType {
static getType() {
return 'contains'