From b6099b1e6d8df6a531803006f943ffb53c290087 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 10:53:42 -0600 Subject: [PATCH 1/6] adding FromCSVToList annotation adding this annotation transforms the model and returns it as a type T which is specified in the Annotated documenting the usage for FromCSVToList --- README.rst | 29 ++++++++++++++++ dysql/annotations.py | 32 ++++++++++++++++++ dysql/test/test_annotations.py | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 dysql/annotations.py create mode 100644 dysql/test/test_annotations.py diff --git a/README.rst b/README.rst index 049b94c..96bfa50 100644 --- a/README.rst +++ b/README.rst @@ -250,6 +250,35 @@ if you desire better validation for list, set, or dict fields, this must most li Additionally, lists, sets, and dicts will ignore null values from the database. Therefore you must provide default values for these fields when used or else validation will fail. +Added annotations when using DbMapResultModel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When using the ``DbMapResultModel`` mapper, there are some additional annotations that may be used to help with +mapping. These annotations are not required, but may be helpful in some cases. + +* FromCSVToList - this annotation will convert a comma separated string into a list. This is useful when you have + a column containing a csv or a query that uses ``group_concat`` to combine multiple rows into a single row. This + annotation may be used on any field that is a list. For example: + + .. code-block:: python + + from dysql.pydantic_mappers import DbMapResultModel, FromCSVToList + + class CsvModel(DbMapResultModel): + id: int + name: str + # This annotation will convert the string into a list of ints + list_from_string_int: FromCSVToList[List[int]] + # This annotation will convert the string into a list of strings + list_from_string: FromCSVToList[List[str]] + # This annotation will convert the string into a list of ints or None if the string is null or empty + list_from_string_int_nullable: FromCSVToList[List[int] | None] + # This annotation will convert the string into a list of strings or None if the string is null or empty + list_from_string_nullable: FromCSVToList[List[str] | None] + + # if using python < 3.9, you can use typing.Union instead of the pipe operator + # list_from_string_nullable: FromCSVToList[Union[List[str],None]] + + @sqlquery ~~~~~~~~~ This is for making SQL ``select`` calls. An optional mapper may be specified to diff --git a/dysql/annotations.py b/dysql/annotations.py new file mode 100644 index 0000000..1f89914 --- /dev/null +++ b/dysql/annotations.py @@ -0,0 +1,32 @@ +""" +Copyright 2021 Adobe +All Rights Reserved. + +NOTICE: Adobe permits you to use, modify, and distribute this file in accordance +with the terms of the Adobe license agreement accompanying it. +""" + +from typing import TypeVar, Annotated + +from pydantic import BeforeValidator + +# pylint: disable=invalid-name +T = TypeVar('T') + + +# we could add validators to the base model, something like @validator('csv*', pre=True)? +def _transform_csv(value: str) -> T: + if not value: + return None + + if isinstance(value, str): + return list(map(str.strip, value.split(','))) + + if isinstance(value, list): + return value + # if we don't have a string or type T we aren't going to be able to transform it + return [value] + + +# Annotation that helps transform a CSV string into a list of type T +FromCSVToList = Annotated[T, BeforeValidator(_transform_csv)] diff --git a/dysql/test/test_annotations.py b/dysql/test/test_annotations.py new file mode 100644 index 0000000..9f72438 --- /dev/null +++ b/dysql/test/test_annotations.py @@ -0,0 +1,61 @@ +""" +Copyright 2021 Adobe +All Rights Reserved. + +NOTICE: Adobe permits you to use, modify, and distribute this file in accordance +with the terms of the Adobe license agreement accompanying it. +""" +from typing import List, Union + +import pytest +from pydantic import BaseModel, ValidationError + +from dysql.annotations import FromCSVToList + + +class StrCSVModel(BaseModel): + values: FromCSVToList[List[str]] + + +class IntCSVModel(BaseModel): + values: FromCSVToList[List[int]] + + +class NullableStrCSVModel(BaseModel): + values: FromCSVToList[Union[List[str], None]] + + +class NullableIntCSVModel(BaseModel): + values: FromCSVToList[Union[List[int], None]] + + +@pytest.mark.parametrize('cls, values, expected', [ + (StrCSVModel, '1,2,3', ['1', '2', '3']), + (StrCSVModel, 'a,b', ['a', 'b']), + (StrCSVModel, 'a', ['a']), + (NullableStrCSVModel, '', None), + (NullableStrCSVModel, None, None), + (StrCSVModel, ['a', 'b'], ['a', 'b']), + (StrCSVModel, ['a', 'b', 'c'], ['a', 'b', 'c']), + (IntCSVModel, '1,2,3', [1, 2, 3]), + (IntCSVModel, '1', [1]), + (NullableIntCSVModel, '', None), + (NullableIntCSVModel, None, None), + (IntCSVModel, ['1', '2', '3'], [1, 2, 3]), + (IntCSVModel, ['1', '2', '3', 4, 5], [1, 2, 3, 4, 5]) +]) +def test_from_csv_to_list(cls, values, expected): + assert expected == cls(values=values).values + + +@pytest.mark.parametrize('cls, values', [ + (StrCSVModel, ''), + (StrCSVModel, None), + (IntCSVModel, 'a,b,c'), + (IntCSVModel, ''), + (IntCSVModel, None), + (IntCSVModel, ['a', 'b', 'c']), +]) +def test_from_csv_to_list_invalid(cls, values): + with pytest.raises(ValidationError): + cls(values=values) From c3daad0f255657469d761b4e1f2613a4e905f497 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 11:24:57 -0600 Subject: [PATCH 2/6] dropping support for python 3.7 and 3.8 --- .github/workflows/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index df7ff18..3c5845b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,8 +10,6 @@ jobs: fail-fast: false matrix: python-version: - - 3.7 - - 3.8 - 3.9 - '3.10' steps: From 884b9abe3b9101d62337999a3aa981ca71c17d0e Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 11:33:55 -0600 Subject: [PATCH 3/6] updating dependencies and major version --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 7bec9f4..b171f17 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages -BASE_VERSION = '2.0' +BASE_VERSION = '3.0' SOURCE_DIR = os.path.dirname( os.path.abspath(__file__) ) @@ -76,13 +76,12 @@ def get_version(): platforms=['Any'], packages=find_packages(exclude=('*test*',)), zip_safe=False, - install_requires=( + install_requires=[ # SQLAlchemy 2+ is not yet submitted 'sqlalchemy<2', - ), - extras_require={ - 'pydantic': ['pydantic>2'], - }, + # now using features only found in pydantic 2+ + 'pydantic>2', + ], classifiers=[ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: MIT License', From 955963cb474452c4709d9f5d5711053fcd4dd717 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 11:45:37 -0600 Subject: [PATCH 4/6] adding checks for 3.11 --- .github/workflows/build.yaml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3c5845b..dc222e8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,6 +12,7 @@ jobs: python-version: - 3.9 - '3.10' + - 3.11 steps: - uses: actions/checkout@v2 with: diff --git a/setup.py b/setup.py index b171f17..ab6ad21 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def get_version(): # SQLAlchemy 2+ is not yet submitted 'sqlalchemy<2', # now using features only found in pydantic 2+ - 'pydantic>2', + 'pydantic>=2', ], classifiers=[ 'Development Status :: 4 - Beta', From f0adb8ee624c821ca1f5a438cef2bb7fea507eaa Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 11:47:55 -0600 Subject: [PATCH 5/6] more cleanup --- .pylint-license-header | 2 +- dysql/annotations.py | 3 +-- dysql/test/test_annotations.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.pylint-license-header b/.pylint-license-header index 6158a96..318035e 100644 --- a/.pylint-license-header +++ b/.pylint-license-header @@ -1,5 +1,5 @@ """ -Copyright 2021 Adobe +Copyright 2023 Adobe All Rights Reserved. NOTICE: Adobe permits you to use, modify, and distribute this file in accordance diff --git a/dysql/annotations.py b/dysql/annotations.py index 1f89914..5b5e66f 100644 --- a/dysql/annotations.py +++ b/dysql/annotations.py @@ -1,5 +1,5 @@ """ -Copyright 2021 Adobe +Copyright 2023 Adobe All Rights Reserved. NOTICE: Adobe permits you to use, modify, and distribute this file in accordance @@ -14,7 +14,6 @@ T = TypeVar('T') -# we could add validators to the base model, something like @validator('csv*', pre=True)? def _transform_csv(value: str) -> T: if not value: return None diff --git a/dysql/test/test_annotations.py b/dysql/test/test_annotations.py index 9f72438..03a75a1 100644 --- a/dysql/test/test_annotations.py +++ b/dysql/test/test_annotations.py @@ -1,5 +1,5 @@ """ -Copyright 2021 Adobe +Copyright 2023 Adobe All Rights Reserved. NOTICE: Adobe permits you to use, modify, and distribute this file in accordance From db34db53111d7ebbb4471c41f4b1d801ac5809ad Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Fri, 22 Sep 2023 16:08:04 -0600 Subject: [PATCH 6/6] updating pylint and cleaning up errors from newer version --- .pylint-license-header | 2 +- .pylintrc | 77 ++------------------------------------- .python-version | 2 +- README.rst | 2 +- dysql/databases.py | 1 + dysql/pydantic_mappers.py | 2 +- setup.py | 3 ++ test_requirements.txt | 2 +- 8 files changed, 12 insertions(+), 79 deletions(-) diff --git a/.pylint-license-header b/.pylint-license-header index 318035e..b6311f6 100644 --- a/.pylint-license-header +++ b/.pylint-license-header @@ -1,5 +1,5 @@ """ -Copyright 2023 Adobe +Copyright 20\d\d Adobe All Rights Reserved. NOTICE: Adobe permits you to use, modify, and distribute this file in accordance diff --git a/.pylintrc b/.pylintrc index 6928958..35582a9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -64,17 +64,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -82,67 +72,6 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, logging-fstring-interpolation, missing-module-docstring, missing-class-docstring, @@ -593,5 +522,5 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=buildins.BaseException, + buildins.Exception diff --git a/.python-version b/.python-version index 0b2eb36..afad818 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.7.2 +3.11.0 diff --git a/README.rst b/README.rst index 96bfa50..21939e0 100644 --- a/README.rst +++ b/README.rst @@ -275,7 +275,7 @@ mapping. These annotations are not required, but may be helpful in some cases. # This annotation will convert the string into a list of strings or None if the string is null or empty list_from_string_nullable: FromCSVToList[List[str] | None] - # if using python < 3.9, you can use typing.Union instead of the pipe operator + # if using python <= 3.9, you can use typing.Union instead of the pipe operator # list_from_string_nullable: FromCSVToList[Union[List[str],None]] diff --git a/dysql/databases.py b/dysql/databases.py index ee36bb7..7276855 100644 --- a/dysql/databases.py +++ b/dysql/databases.py @@ -185,6 +185,7 @@ def current_database(self) -> Database: """ The current database instance, retrieved using contextvars (if python 3.7+) or the default database. """ + # pylint: disable=unnecessary-dunder-call return self.__getitem__(_get_current_database()) diff --git a/dysql/pydantic_mappers.py b/dysql/pydantic_mappers.py index 2268992..c6e89e1 100644 --- a/dysql/pydantic_mappers.py +++ b/dysql/pydantic_mappers.py @@ -138,7 +138,7 @@ def map_record(self, record: sqlalchemy.engine.Row) -> None: self.__dict__.update(current_dict) else: # Init takes care of validation and assigning values to each field with conversions in place, etc - self.__init__(**current_dict) + self.__init__(**current_dict) # pylint: disable=unnecessary-dunder-call def raw(self) -> dict: return self.model_dump() diff --git a/setup.py b/setup.py index ab6ad21..faccb97 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,8 @@ import os import subprocess import types +from datetime import datetime + from setuptools import setup, find_packages @@ -50,6 +52,7 @@ def get_version(): if os.path.exists(DYSQL_DIR): with open(HEADER_FILE, 'r', encoding='utf8') as fobj: header = fobj.read() + header = header.replace('20\d\d', datetime.now().strftime('%Y')) with open(VERSION_FILE, 'w', encoding='utf8') as fobj: fobj.write(f"{header}\n__version__ = '{new_version}'\n") return new_version diff --git a/test_requirements.txt b/test_requirements.txt index ec116a9..6ecbb54 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ pytest==6.2.4 pytest-randomly==3.10.1 pytest-cov==2.12.1 -pylint==2.10.2 +pylint>2.10.2 pylintfileheader==0.3.0 pycodestyle==2.8.0