Merge pull request adobe#42 from tamagoko/csv-annototion-support

adding FromCSVToList annotation

.github/workflows/build.yaml

@@ -10,10 +10,9 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-        - 3.7
-        - 3.8
         - 3.9
         - '3.10'
+        - 3.11
     steps:
     - uses: actions/checkout@v2
       with:

.pylint-license-header

@@ -1,5 +1,5 @@
 """
-Copyright 2021 Adobe
+Copyright 20\d\d Adobe
 All Rights Reserved.
 
 NOTICE: Adobe permits you to use, modify, and distribute this file in accordance

.pylintrc

@@ -64,85 +64,14 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
-        raw-checker-failed,
+disable=raw-checker-failed,
         bad-inline-option,
         locally-disabled,
         file-ignored,
         suppressed-message,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating,
-        deprecated-operator-function,
-        deprecated-urllib-function,
-        xreadlines-attribute,
-        deprecated-sys-function,
-        exception-escape,
-        comprehension-escape,
         logging-fstring-interpolation,
         missing-module-docstring,
         missing-class-docstring,
@@ -593,5 +522,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=buildins.BaseException,
+                       buildins.Exception

.python-version

@@ -1 +1 @@
-3.7.2
+3.11.0

README.rst

@@ -250,6 +250,35 @@ if you desire better validation for list, set, or dict fields, this must most li
 Additionally, lists, sets, and dicts will ignore null values from the database. Therefore you must provide default
 values for these fields when used or else validation will fail.
 
+Added annotations when using DbMapResultModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+When using the ``DbMapResultModel`` mapper, there are some additional annotations that may be used to help with
+mapping. These annotations are not required, but may be helpful in some cases.
+
+* FromCSVToList - this annotation will convert a comma separated string into a list. This is useful when you have
+  a column containing a csv or a query that uses ``group_concat`` to combine multiple rows into a single row. This
+  annotation may be used on any field that is a list. For example:
+
+  .. code-block:: python
+
+    from dysql.pydantic_mappers import DbMapResultModel, FromCSVToList
+
+    class CsvModel(DbMapResultModel):
+        id: int
+        name: str
+        # This annotation will convert the string into a list of ints
+        list_from_string_int: FromCSVToList[List[int]]
+        # This annotation will convert the string into a list of strings
+        list_from_string: FromCSVToList[List[str]]
+        # This annotation will convert the string into a list of ints or None if the string is null or empty
+        list_from_string_int_nullable: FromCSVToList[List[int] | None]
+        # This annotation will convert the string into a list of strings or None if the string is null or empty
+        list_from_string_nullable: FromCSVToList[List[str] | None]
+
+        # if using python <= 3.9, you can use typing.Union instead of the pipe operator
+        # list_from_string_nullable: FromCSVToList[Union[List[str],None]]
+
+
 @sqlquery
 ~~~~~~~~~
 This is for making SQL ``select`` calls. An optional mapper may be specified to

dysql/annotations.py

@@ -0,0 +1,31 @@
+"""
+Copyright 2023 Adobe
+All Rights Reserved.
+
+NOTICE: Adobe permits you to use, modify, and distribute this file in accordance
+with the terms of the Adobe license agreement accompanying it.
+"""
+
+from typing import TypeVar, Annotated
+
+from pydantic import BeforeValidator
+
+# pylint: disable=invalid-name
+T = TypeVar('T')
+
+
+def _transform_csv(value: str) -> T:
+    if not value:
+        return None
+
+    if isinstance(value, str):
+        return list(map(str.strip, value.split(',')))
+
+    if isinstance(value, list):
+        return value
+    # if we don't have a string or type T we aren't going to be able to transform it
+    return [value]
+
+
+# Annotation that helps transform a CSV string into a list of type T
+FromCSVToList = Annotated[T, BeforeValidator(_transform_csv)]

dysql/databases.py

@@ -185,6 +185,7 @@ def current_database(self) -> Database:
         """
         The current database instance, retrieved using contextvars (if python 3.7+) or the default database.
         """
+        # pylint: disable=unnecessary-dunder-call
         return self.__getitem__(_get_current_database())
 
 

dysql/pydantic_mappers.py

@@ -138,7 +138,7 @@ def map_record(self, record: sqlalchemy.engine.Row) -> None:
             self.__dict__.update(current_dict)
         else:
             # Init takes care of validation and assigning values to each field with conversions in place, etc
-            self.__init__(**current_dict)
+            self.__init__(**current_dict)  # pylint: disable=unnecessary-dunder-call
 
     def raw(self) -> dict:
         return self.model_dump()

dysql/test/test_annotations.py

@@ -0,0 +1,61 @@
+"""
+Copyright 2023 Adobe
+All Rights Reserved.
+
+NOTICE: Adobe permits you to use, modify, and distribute this file in accordance
+with the terms of the Adobe license agreement accompanying it.
+"""
+from typing import List, Union
+
+import pytest
+from pydantic import BaseModel, ValidationError
+
+from dysql.annotations import FromCSVToList
+
+
+class StrCSVModel(BaseModel):
+    values: FromCSVToList[List[str]]
+
+
+class IntCSVModel(BaseModel):
+    values: FromCSVToList[List[int]]
+
+
+class NullableStrCSVModel(BaseModel):
+    values: FromCSVToList[Union[List[str], None]]
+
+
+class NullableIntCSVModel(BaseModel):
+    values: FromCSVToList[Union[List[int], None]]
+
+
+@pytest.mark.parametrize('cls, values, expected', [
+    (StrCSVModel, '1,2,3', ['1', '2', '3']),
+    (StrCSVModel, 'a,b', ['a', 'b']),
+    (StrCSVModel, 'a', ['a']),
+    (NullableStrCSVModel, '', None),
+    (NullableStrCSVModel, None, None),
+    (StrCSVModel, ['a', 'b'], ['a', 'b']),
+    (StrCSVModel, ['a', 'b', 'c'], ['a', 'b', 'c']),
+    (IntCSVModel, '1,2,3', [1, 2, 3]),
+    (IntCSVModel, '1', [1]),
+    (NullableIntCSVModel, '', None),
+    (NullableIntCSVModel, None, None),
+    (IntCSVModel, ['1', '2', '3'], [1, 2, 3]),
+    (IntCSVModel, ['1', '2', '3', 4, 5], [1, 2, 3, 4, 5])
+])
+def test_from_csv_to_list(cls, values, expected):
+    assert expected == cls(values=values).values
+
+
+@pytest.mark.parametrize('cls, values', [
+    (StrCSVModel, ''),
+    (StrCSVModel, None),
+    (IntCSVModel, 'a,b,c'),
+    (IntCSVModel, ''),
+    (IntCSVModel, None),
+    (IntCSVModel, ['a', 'b', 'c']),
+])
+def test_from_csv_to_list_invalid(cls, values):
+    with pytest.raises(ValidationError):
+        cls(values=values)

setup.py

@@ -2,10 +2,12 @@
 import os
 import subprocess
 import types
+from datetime import datetime
+
 from setuptools import setup, find_packages
 
 
-BASE_VERSION = '2.0'
+BASE_VERSION = '3.0'
 SOURCE_DIR = os.path.dirname(
     os.path.abspath(__file__)
 )
@@ -50,6 +52,7 @@ def get_version():
             if os.path.exists(DYSQL_DIR):
                 with open(HEADER_FILE, 'r', encoding='utf8') as fobj:
                     header = fobj.read()
+                header = header.replace('20\d\d', datetime.now().strftime('%Y'))
                 with open(VERSION_FILE, 'w', encoding='utf8') as fobj:
                     fobj.write(f"{header}\n__version__ = '{new_version}'\n")
             return new_version
@@ -76,13 +79,12 @@ def get_version():
     platforms=['Any'],
     packages=find_packages(exclude=('*test*',)),
     zip_safe=False,
-    install_requires=(
+    install_requires=[
         # SQLAlchemy 2+ is not yet submitted
         'sqlalchemy<2',
-    ),
-    extras_require={
-        'pydantic': ['pydantic>2'],
-    },
+        # now using features only found in pydantic 2+
+        'pydantic>=2',
+    ],
     classifiers=[
         'Development Status :: 4 - Beta',
         'License :: OSI Approved :: MIT License',

test_requirements.txt

@@ -1,6 +1,6 @@
 pytest==6.2.4
 pytest-randomly==3.10.1
 pytest-cov==2.12.1
-pylint==2.10.2
+pylint>2.10.2
 pylintfileheader==0.3.0
 pycodestyle==2.8.0