diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index df7ff18..dc222e8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,10 +10,9 @@ jobs: fail-fast: false matrix: python-version: - - 3.7 - - 3.8 - 3.9 - '3.10' + - 3.11 steps: - uses: actions/checkout@v2 with: diff --git a/.pylint-license-header b/.pylint-license-header index 6158a96..b6311f6 100644 --- a/.pylint-license-header +++ b/.pylint-license-header @@ -1,5 +1,5 @@ """ -Copyright 2021 Adobe +Copyright 20\d\d Adobe All Rights Reserved. NOTICE: Adobe permits you to use, modify, and distribute this file in accordance diff --git a/.pylintrc b/.pylintrc index 6928958..35582a9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -64,17 +64,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -82,67 +72,6 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, logging-fstring-interpolation, missing-module-docstring, missing-class-docstring, @@ -593,5 +522,5 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=buildins.BaseException, + buildins.Exception diff --git a/.python-version b/.python-version index 0b2eb36..afad818 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.7.2 +3.11.0 diff --git a/README.rst b/README.rst index 049b94c..21939e0 100644 --- a/README.rst +++ b/README.rst @@ -250,6 +250,35 @@ if you desire better validation for list, set, or dict fields, this must most li Additionally, lists, sets, and dicts will ignore null values from the database. Therefore you must provide default values for these fields when used or else validation will fail. +Added annotations when using DbMapResultModel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When using the ``DbMapResultModel`` mapper, there are some additional annotations that may be used to help with +mapping. These annotations are not required, but may be helpful in some cases. + +* FromCSVToList - this annotation will convert a comma separated string into a list. This is useful when you have + a column containing a csv or a query that uses ``group_concat`` to combine multiple rows into a single row. This + annotation may be used on any field that is a list. For example: + + .. code-block:: python + + from dysql.pydantic_mappers import DbMapResultModel, FromCSVToList + + class CsvModel(DbMapResultModel): + id: int + name: str + # This annotation will convert the string into a list of ints + list_from_string_int: FromCSVToList[List[int]] + # This annotation will convert the string into a list of strings + list_from_string: FromCSVToList[List[str]] + # This annotation will convert the string into a list of ints or None if the string is null or empty + list_from_string_int_nullable: FromCSVToList[List[int] | None] + # This annotation will convert the string into a list of strings or None if the string is null or empty + list_from_string_nullable: FromCSVToList[List[str] | None] + + # if using python <= 3.9, you can use typing.Union instead of the pipe operator + # list_from_string_nullable: FromCSVToList[Union[List[str],None]] + + @sqlquery ~~~~~~~~~ This is for making SQL ``select`` calls. An optional mapper may be specified to diff --git a/dysql/annotations.py b/dysql/annotations.py new file mode 100644 index 0000000..5b5e66f --- /dev/null +++ b/dysql/annotations.py @@ -0,0 +1,31 @@ +""" +Copyright 2023 Adobe +All Rights Reserved. + +NOTICE: Adobe permits you to use, modify, and distribute this file in accordance +with the terms of the Adobe license agreement accompanying it. +""" + +from typing import TypeVar, Annotated + +from pydantic import BeforeValidator + +# pylint: disable=invalid-name +T = TypeVar('T') + + +def _transform_csv(value: str) -> T: + if not value: + return None + + if isinstance(value, str): + return list(map(str.strip, value.split(','))) + + if isinstance(value, list): + return value + # if we don't have a string or type T we aren't going to be able to transform it + return [value] + + +# Annotation that helps transform a CSV string into a list of type T +FromCSVToList = Annotated[T, BeforeValidator(_transform_csv)] diff --git a/dysql/databases.py b/dysql/databases.py index ee36bb7..7276855 100644 --- a/dysql/databases.py +++ b/dysql/databases.py @@ -185,6 +185,7 @@ def current_database(self) -> Database: """ The current database instance, retrieved using contextvars (if python 3.7+) or the default database. """ + # pylint: disable=unnecessary-dunder-call return self.__getitem__(_get_current_database()) diff --git a/dysql/pydantic_mappers.py b/dysql/pydantic_mappers.py index 2268992..c6e89e1 100644 --- a/dysql/pydantic_mappers.py +++ b/dysql/pydantic_mappers.py @@ -138,7 +138,7 @@ def map_record(self, record: sqlalchemy.engine.Row) -> None: self.__dict__.update(current_dict) else: # Init takes care of validation and assigning values to each field with conversions in place, etc - self.__init__(**current_dict) + self.__init__(**current_dict) # pylint: disable=unnecessary-dunder-call def raw(self) -> dict: return self.model_dump() diff --git a/dysql/test/test_annotations.py b/dysql/test/test_annotations.py new file mode 100644 index 0000000..03a75a1 --- /dev/null +++ b/dysql/test/test_annotations.py @@ -0,0 +1,61 @@ +""" +Copyright 2023 Adobe +All Rights Reserved. + +NOTICE: Adobe permits you to use, modify, and distribute this file in accordance +with the terms of the Adobe license agreement accompanying it. +""" +from typing import List, Union + +import pytest +from pydantic import BaseModel, ValidationError + +from dysql.annotations import FromCSVToList + + +class StrCSVModel(BaseModel): + values: FromCSVToList[List[str]] + + +class IntCSVModel(BaseModel): + values: FromCSVToList[List[int]] + + +class NullableStrCSVModel(BaseModel): + values: FromCSVToList[Union[List[str], None]] + + +class NullableIntCSVModel(BaseModel): + values: FromCSVToList[Union[List[int], None]] + + +@pytest.mark.parametrize('cls, values, expected', [ + (StrCSVModel, '1,2,3', ['1', '2', '3']), + (StrCSVModel, 'a,b', ['a', 'b']), + (StrCSVModel, 'a', ['a']), + (NullableStrCSVModel, '', None), + (NullableStrCSVModel, None, None), + (StrCSVModel, ['a', 'b'], ['a', 'b']), + (StrCSVModel, ['a', 'b', 'c'], ['a', 'b', 'c']), + (IntCSVModel, '1,2,3', [1, 2, 3]), + (IntCSVModel, '1', [1]), + (NullableIntCSVModel, '', None), + (NullableIntCSVModel, None, None), + (IntCSVModel, ['1', '2', '3'], [1, 2, 3]), + (IntCSVModel, ['1', '2', '3', 4, 5], [1, 2, 3, 4, 5]) +]) +def test_from_csv_to_list(cls, values, expected): + assert expected == cls(values=values).values + + +@pytest.mark.parametrize('cls, values', [ + (StrCSVModel, ''), + (StrCSVModel, None), + (IntCSVModel, 'a,b,c'), + (IntCSVModel, ''), + (IntCSVModel, None), + (IntCSVModel, ['a', 'b', 'c']), +]) +def test_from_csv_to_list_invalid(cls, values): + with pytest.raises(ValidationError): + cls(values=values) diff --git a/setup.py b/setup.py index 7bec9f4..faccb97 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,12 @@ import os import subprocess import types +from datetime import datetime + from setuptools import setup, find_packages -BASE_VERSION = '2.0' +BASE_VERSION = '3.0' SOURCE_DIR = os.path.dirname( os.path.abspath(__file__) ) @@ -50,6 +52,7 @@ def get_version(): if os.path.exists(DYSQL_DIR): with open(HEADER_FILE, 'r', encoding='utf8') as fobj: header = fobj.read() + header = header.replace('20\d\d', datetime.now().strftime('%Y')) with open(VERSION_FILE, 'w', encoding='utf8') as fobj: fobj.write(f"{header}\n__version__ = '{new_version}'\n") return new_version @@ -76,13 +79,12 @@ def get_version(): platforms=['Any'], packages=find_packages(exclude=('*test*',)), zip_safe=False, - install_requires=( + install_requires=[ # SQLAlchemy 2+ is not yet submitted 'sqlalchemy<2', - ), - extras_require={ - 'pydantic': ['pydantic>2'], - }, + # now using features only found in pydantic 2+ + 'pydantic>=2', + ], classifiers=[ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: MIT License', diff --git a/test_requirements.txt b/test_requirements.txt index ec116a9..6ecbb54 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ pytest==6.2.4 pytest-randomly==3.10.1 pytest-cov==2.12.1 -pylint==2.10.2 +pylint>2.10.2 pylintfileheader==0.3.0 pycodestyle==2.8.0