support commas in data_attrs query param

.github/workflows/oas-check.yml

@@ -12,13 +12,8 @@ on:
 jobs:
   open-api-workflow-check-oas:
     uses: maykinmedia/open-api-workflows/.github/workflows/oas-check.yml@v1
-    strategy:
-      matrix:
-        version:
-          - v2
     with:
-      schema-path: 'src/objects/api/${{ matrix.version }}/openapi.yaml'
-      schema-options: "--api-version ${{ matrix.version }}"
+      schema-path: 'src/objects/api/v2/openapi.yaml'
       python-version: '3.11'
       django-settings-module: 'objects.conf.ci'
       apt-packages: 'libgdal-dev gdal-bin'

src/objects/api/v2/filters.py

@@ -1,18 +1,58 @@
 from datetime import date as date_
 
 from django import forms
+from django.db.models import QuerySet
 from django.utils.translation import gettext_lazy as _
 
 from django_filters import filters
 from rest_framework import serializers
 from vng_api_common.filtersets import FilterSet
 
 from objects.core.models import ObjectRecord, ObjectType
-from objects.utils.filters import ObjectTypeFilter
+from objects.utils.filters import ManyCharFilter, ObjectTypeFilter
 
 from ..constants import Operators
 from ..utils import display_choice_values_for_help_text, string_to_value
-from ..validators import validate_data_attrs
+from ..validators import validate_data_attr, validate_data_attrs
+
+DATA_ATTR_VALUE_HELP_TEXT = f"""A valid parameter value has the form `key__operator__value`.
+`key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+Valid operator values are:
+{display_choice_values_for_help_text(Operators)}
+
+`value` may not contain double underscore or comma characters.
+`key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+"""
+
+
+def filter_data_attr_value_part(value_part: str, queryset: QuerySet) -> QuerySet:
+    """
+    filter one value part for data_attr and data_attrs filters
+    """
+    variable, operator, str_value = value_part.rsplit("__", 2)
+    real_value = string_to_value(str_value)
+
+    if operator == "exact":
+        #  for exact operator try to filter on string and numeric values
+        in_vals = [str_value]
+        if real_value != str_value:
+            in_vals.append(real_value)
+        queryset = queryset.filter(**{f"data__{variable}__in": in_vals})
+    elif operator == "icontains":
+        # icontains treats everything like strings
+        queryset = queryset.filter(**{f"data__{variable}__icontains": str_value})
+    elif operator == "in":
+        # in must be a list
+        values = str_value.split("|")
+        queryset = queryset.filter(**{f"data__{variable}__in": values})
+
+    else:
+        # gt, gte, lt, lte operators
+        queryset = queryset.filter(**{f"data__{variable}__{operator}": real_value})
+    return queryset
 
 
 class ObjectRecordFilterForm(forms.Form):
@@ -62,63 +102,68 @@ class ObjectRecordFilterSet(FilterSet):
         method="filter_data_attrs",
         validators=[validate_data_attrs],
         help_text=_(
-            """Only include objects that have attributes with certain values.
+            """**DEPRECATED: Use 'data_attr' instead**.
+Only include objects that have attributes with certain values.
 Data filtering expressions are comma-separated and are structured as follows:
-A valid parameter value has the form `key__operator__value`.
-`key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
-Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
-
-Valid operator values are:
-%(operator_choices)s
 
-`value` may not contain double underscore or comma characters.
-`key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+%(value_part_help_text)s
 
 Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
 should be used. If `height` is nested inside `dimensions` attribute, query should look like
 `data_attrs=dimensions__height__exact__100`
+
+`value` may not contain comma, since commas are used as separator between filtering expressions.
+If you want to use commas in `value` you can use `data_attr` query parameter.
+"""
+        )
+        % {"value_part_help_text": DATA_ATTR_VALUE_HELP_TEXT},
+    )
+
+    data_attr = ManyCharFilter(
+        method="filter_data_attr",
+        validators=[validate_data_attr],
+        help_text=_(
+            """Only include objects that have attributes with certain values.
+
+%(value_part_help_text)s
+
+Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+should be used. If `height` is nested inside `dimensions` attribute, query should look like
+`data_attr=dimensions__height__exact__100`
+
+This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+* `value` may contain commas
+* only one filtering expression is allowed
+
+If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
 """
         )
-        % {"operator_choices": display_choice_values_for_help_text(Operators)},
+        % {"value_part_help_text": DATA_ATTR_VALUE_HELP_TEXT},
     )
+
     data_icontains = filters.CharFilter(
         method="filter_data_icontains",
         help_text=_("Search in all `data` values of string properties."),
     )
 
     class Meta:
         model = ObjectRecord
-        fields = ("type", "data_attrs", "date", "registrationDate")
+        fields = ("type", "data_attrs", "data_attr", "date", "registrationDate")
         form = ObjectRecordFilterForm
 
     def filter_data_attrs(self, queryset, name, value: str):
         parts = value.split(",")
 
         for value_part in parts:
-            variable, operator, str_value = value_part.rsplit("__", 2)
-            real_value = string_to_value(str_value)
-
-            if operator == "exact":
-                #  for exact operator try to filter on string and numeric values
-                in_vals = [str_value]
-                if real_value != value:
-                    in_vals.append(real_value)
-                queryset = queryset.filter(**{f"data__{variable}__in": in_vals})
-            elif operator == "icontains":
-                # icontains treats everything like strings
-                queryset = queryset.filter(
-                    **{f"data__{variable}__icontains": str_value}
-                )
-            elif operator == "in":
-                # in must be a list
-                values = str_value.split("|")
-                queryset = queryset.filter(**{f"data__{variable}__in": values})
-
-            else:
-                # gt, gte, lt, lte operators
-                queryset = queryset.filter(
-                    **{f"data__{variable}__{operator}": real_value}
-                )
+            queryset = filter_data_attr_value_part(value_part, queryset)
+
+        return queryset
+
+    def filter_data_attr(self, queryset, name, value: list):
+        for value_part in value:
+            queryset = filter_data_attr_value_part(value_part, queryset)
 
         return queryset
 

src/objects/api/v2/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: Objects API
-  version: 2.4.3 (v2)
+  version: 2.4.3
   description: |
     An API to manage Objects.
 
@@ -88,13 +88,51 @@ paths:
         description: 'The desired ''Coordinate Reference System'' (CRS) of the response
           data. According to the GeoJSON spec, WGS84 is the default (EPSG: 4326 is
           the same as WGS84).'
+      - in: query
+        name: data_attr
+        schema:
+          type: string
+        description: |
+          Only include objects that have attributes with certain values.
+
+          A valid parameter value has the form `key__operator__value`.
+          `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+          Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+          Valid operator values are:
+          * `exact` - equal to
+          * `gt` - greater than
+          * `gte` - greater than or equal to
+          * `lt` - lower than
+          * `lte` - lower than or equal to
+          * `icontains` - case-insensitive partial match
+          * `in` - in a list of values separated by `|`
+
+          `value` may not contain double underscore or comma characters.
+          `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+
+
+          Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+          should be used. If `height` is nested inside `dimensions` attribute, query should look like
+          `data_attr=dimensions__height__exact__100`
+
+          This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+          * `value` may contain commas
+          * only one filtering expression is allowed
+
+          If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+          Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
       - in: query
         name: data_attrs
         schema:
           type: string
         description: |
+          **DEPRECATED: Use 'data_attr' instead**.
           Only include objects that have attributes with certain values.
           Data filtering expressions are comma-separated and are structured as follows:
+
           A valid parameter value has the form `key__operator__value`.
           `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
           Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
@@ -111,9 +149,14 @@ paths:
           `value` may not contain double underscore or comma characters.
           `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
 
+
+
           Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
           should be used. If `height` is nested inside `dimensions` attribute, query should look like
           `data_attrs=dimensions__height__exact__100`
+
+          `value` may not contain comma, since commas are used as separator between filtering expressions.
+          If you want to use commas in `value` you can use `data_attr` query parameter.
       - in: query
         name: data_icontains
         schema:
@@ -620,8 +663,10 @@ paths:
                   data_attrs:
                     type: string
                     description: |
+                      **DEPRECATED: Use 'data_attr' instead**.
                       Only include objects that have attributes with certain values.
                       Data filtering expressions are comma-separated and are structured as follows:
+
                       A valid parameter value has the form `key__operator__value`.
                       `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
                       Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
@@ -638,9 +683,48 @@ paths:
                       `value` may not contain double underscore or comma characters.
                       `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
 
+
+
                       Example: in order to display only objects with `height` equal to 100, query `data_attrs=height__exact__100`
                       should be used. If `height` is nested inside `dimensions` attribute, query should look like
                       `data_attrs=dimensions__height__exact__100`
+
+                      `value` may not contain comma, since commas are used as separator between filtering expressions.
+                      If you want to use commas in `value` you can use `data_attr` query parameter.
+                  data_attr:
+                    type: string
+                    description: |
+                      Only include objects that have attributes with certain values.
+
+                      A valid parameter value has the form `key__operator__value`.
+                      `key` is the attribute name, `operator` is the comparison operator to be used and `value` is the attribute value.
+                      Note: Values can be string, numeric, or dates (ISO format; YYYY-MM-DD).
+
+                      Valid operator values are:
+                      * `exact` - equal to
+                      * `gt` - greater than
+                      * `gte` - greater than or equal to
+                      * `lt` - lower than
+                      * `lte` - lower than or equal to
+                      * `icontains` - case-insensitive partial match
+                      * `in` - in a list of values separated by `|`
+
+                      `value` may not contain double underscore or comma characters.
+                      `key` may not contain comma characters and includes double underscore only if it indicates nested attributes.
+
+
+
+                      Example: in order to display only objects with `height` equal to 100, query `data_attr=height__exact__100`
+                      should be used. If `height` is nested inside `dimensions` attribute, query should look like
+                      `data_attr=dimensions__height__exact__100`
+
+                      This filter is very similar to the old `data_attrs` filter, but it has two differences:
+
+                      * `value` may contain commas
+                      * only one filtering expression is allowed
+
+                      If you want to use several filtering expressions, just use this `data_attr` several times in the query string.
+                      Example: `data_attr=height__exact__100&data_attr=naam__icontains__boom`
                   date:
                     type: string
                     format: date

src/objects/api/validators.py

@@ -68,32 +68,54 @@ def __call__(self, new_value, serializer_field):
             raise serializers.ValidationError(self.message, code=self.code)
 
 
+def validate_data_attr_value_part(value_part: str, code: str):
+    try:
+        variable, operator, val = value_part.rsplit("__", 2)
+    except ValueError:
+        message = _(
+            "Filter expression '%(value_part)s' doesn't have the shape 'key__operator__value'"
+        ) % {"value_part": value_part}
+        raise serializers.ValidationError(message, code=code)
+
+    if operator not in Operators.values:
+        message = _("Comparison operator `%(operator)s` is unknown") % {
+            "operator": operator
+        }
+        raise serializers.ValidationError(message, code=code)
+
+    if operator not in (
+        Operators.exact,
+        Operators.icontains,
+        Operators.in_list,
+    ) and isinstance(string_to_value(val), str):
+        message = _(
+            "Operator `%(operator)s` supports only dates and/or numeric values"
+        ) % {"operator": operator}
+        raise serializers.ValidationError(message, code=code)
+
+
 def validate_data_attrs(value: str):
+    # todo remove when 'data_attrs' filter is removed
     code = "invalid-data-attrs-query"
     parts = value.split(",")
 
     for value_part in parts:
-        try:
-            variable, operator, val = value_part.rsplit("__", 2)
-        except ValueError as exc:
-            raise serializers.ValidationError(exc.args[0], code=code) from exc
-
-        if operator not in Operators.values:
-            message = _("Comparison operator `%(operator)s` is unknown") % {
-                "operator": operator
-            }
-            raise serializers.ValidationError(message, code=code)
+        validate_data_attr_value_part(value_part, code)
 
-        if operator not in (
-            Operators.exact,
-            Operators.icontains,
-            Operators.in_list,
-        ) and isinstance(string_to_value(val), str):
+
+def validate_data_attr(value: list):
+    code = "invalid-data-attr-query"
+
+    for value_part in value:
+        # check that comma can be only in the value part
+        if "," in value_part.rsplit("__", 1)[0]:
             message = _(
-                "Operator `%(operator)s` supports only dates and/or numeric values"
-            ) % {"operator": operator}
+                "Filter expression '%(value_part)s' doesn't have the shape 'key__operator__value'"
+            ) % {"value_part": value_part}
             raise serializers.ValidationError(message, code=code)
 
+        validate_data_attr_value_part(value_part, code)
+
 
 class GeometryValidator:
     code = "geometry-not-allowed"