From 9e2ebb288fdb97883c1a8082c93f565e19cd02ad Mon Sep 17 00:00:00 2001 From: ofekisr <35701650+ofekisr@users.noreply.github.com> Date: Tue, 11 Jan 2022 14:16:09 +0200 Subject: [PATCH] refactor: examples data loading for tests (#17893) * refactor: replace the way the birth_names data is generated * refactor: replace the way the birth_names data is generated * refactor structure add tests for common --- .../workflows/superset-python-unittest.yml | 2 +- .../common/example_data_generator/__init__.py | 25 ++++ .../example_data_generator/base_generator.py | 34 +++++ .../birth_names/__init__.py | 33 +++++ .../birth_names/birth_names_generator.py | 119 ++++++++++++++++++ .../birth_names_generator_factory.py | 74 +++++++++++ tests/common/example_data_generator/consts.py | 69 ++++++++++ .../string_generator.py | 42 +++++++ .../string_generator_factory.py | 46 +++++++ .../example_data_generator/tests/__init__.py | 25 ++++ .../tests/test_string_generator.py | 35 ++++++ .../fixtures/birth_names_dashboard.py | 92 +------------- 12 files changed, 507 insertions(+), 89 deletions(-) create mode 100644 tests/common/example_data_generator/__init__.py create mode 100644 tests/common/example_data_generator/base_generator.py create mode 100644 tests/common/example_data_generator/birth_names/__init__.py create mode 100644 tests/common/example_data_generator/birth_names/birth_names_generator.py create mode 100644 tests/common/example_data_generator/birth_names/birth_names_generator_factory.py create mode 100644 tests/common/example_data_generator/consts.py create mode 100644 tests/common/example_data_generator/string_generator.py create mode 100644 tests/common/example_data_generator/string_generator_factory.py create mode 100644 tests/common/example_data_generator/tests/__init__.py create mode 100644 tests/common/example_data_generator/tests/test_string_generator.py diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 18925c3ff562e..738c6138574b5 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -51,7 +51,7 @@ jobs: - name: Python unit tests if: steps.check.outcome == 'failure' run: | - pytest --durations=0 ./tests/unit_tests --cache-clear + pytest --durations=0 ./tests/common ./tests/unit_tests --cache-clear - name: Upload code coverage if: steps.check.outcome == 'failure' run: | diff --git a/tests/common/example_data_generator/__init__.py b/tests/common/example_data_generator/__init__.py new file mode 100644 index 0000000000000..dc74f5a8fa252 --- /dev/null +++ b/tests/common/example_data_generator/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/common/example_data_generator/base_generator.py b/tests/common/example_data_generator/base_generator.py new file mode 100644 index 0000000000000..ffd30e99c92d9 --- /dev/null +++ b/tests/common/example_data_generator/base_generator.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable + + +class ExampleDataGenerator(ABC): + @abstractmethod + def generate(self) -> Iterable[Dict[Any, Any]]: + ... diff --git a/tests/common/example_data_generator/birth_names/__init__.py b/tests/common/example_data_generator/birth_names/__init__.py new file mode 100644 index 0000000000000..e5379060d1f17 --- /dev/null +++ b/tests/common/example_data_generator/birth_names/__init__.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/common/example_data_generator/birth_names/birth_names_generator.py b/tests/common/example_data_generator/birth_names/birth_names_generator.py new file mode 100644 index 0000000000000..92deef3e2cde1 --- /dev/null +++ b/tests/common/example_data_generator/birth_names/birth_names_generator.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime +from random import choice, randint +from typing import Any, Dict, Iterable + +from tests.common.example_data_generator.base_generator import ExampleDataGenerator +from tests.common.example_data_generator.consts import US_STATES +from tests.common.example_data_generator.string_generator import StringGenerator + +NUM_GIRLS = "num_girls" +NUM_BOYS = "num_boys" +STATE = "state" +NUM = "num" +NAME = "name" +GENDER = "gender" +DS = "ds" +GIRL = "girl" +BOY = "boy" + +from collections import OrderedDict + +BIRTH_NAMES_COLUMNS = OrderedDict( + [ + (DS, datetime), + (GENDER, str), + (NAME, str), + (NUM, int), + (STATE, str), + (NUM_BOYS, int), + (NUM_GIRLS, int), + ] +) + + +class BirthNamesGenerator(ExampleDataGenerator): + _names_generator: StringGenerator + _start_year: int + _until_not_include_year: int + _rows_per_year: int + + def __init__( + self, + names_generator: StringGenerator, + start_year: int, + years_amount: int, + rows_per_year: int, + ) -> None: + assert start_year > -1 + assert years_amount > 0 + self._names_generator = names_generator + self._start_year = start_year + self._until_not_include_year = start_year + years_amount + self._rows_per_year = rows_per_year + + def generate(self) -> Iterable[Dict[Any, Any]]: + for year in range(self._start_year, self._until_not_include_year): + ds = self._make_year(year) + for _ in range(self._rows_per_year): + yield self.generate_row(ds) + + def _make_year(self, year: int): + return datetime(year, 1, 1, 0, 0, 0) + + def generate_row(self, dt: datetime) -> Dict[Any, Any]: + gender = choice([BOY, GIRL]) + num = randint(1, 100000) + return { + DS: dt, + GENDER: gender, + NAME: self._names_generator.generate(), + NUM: num, + STATE: choice(US_STATES), + NUM_BOYS: num if gender == BOY else 0, + NUM_GIRLS: num if gender == GIRL else 0, + } diff --git a/tests/common/example_data_generator/birth_names/birth_names_generator_factory.py b/tests/common/example_data_generator/birth_names/birth_names_generator_factory.py new file mode 100644 index 0000000000000..20abebdc56c22 --- /dev/null +++ b/tests/common/example_data_generator/birth_names/birth_names_generator_factory.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from abc import ABC, abstractmethod + +from tests.common.example_data_generator.birth_names.birth_names_generator import ( + BirthNamesGenerator, +) +from tests.common.example_data_generator.string_generator_factory import ( + StringGeneratorFactory, +) + + +class BirthNamesGeneratorFactory(ABC): + __factory: BirthNamesGeneratorFactory + + @abstractmethod + def _make(self) -> BirthNamesGenerator: + ... + + @classmethod + def make(cls) -> BirthNamesGenerator: + return cls._get_instance()._make() + + @classmethod + def set_instance(cls, factory: BirthNamesGeneratorFactory) -> None: + cls.__factory = factory + + @classmethod + def _get_instance(cls) -> BirthNamesGeneratorFactory: + if not hasattr(cls, "_BirthNamesGeneratorFactory__factory"): + cls.__factory = BirthNamesGeneratorFactoryImpl() + return cls.__factory + + +MIN_NAME_LEN = 3 +MAX_NAME_SIZE = 10 +START_YEAR = 1960 +YEARS_AMOUNT = 60 +ROW_PER_YEAR = 20 + + +class BirthNamesGeneratorFactoryImpl(BirthNamesGeneratorFactory): + def _make(self) -> BirthNamesGenerator: + string_generator = StringGeneratorFactory.make_lowercase_based( + MIN_NAME_LEN, MAX_NAME_SIZE + ) + return BirthNamesGenerator( + string_generator, START_YEAR, YEARS_AMOUNT, ROW_PER_YEAR + ) diff --git a/tests/common/example_data_generator/consts.py b/tests/common/example_data_generator/consts.py new file mode 100644 index 0000000000000..2432d60b6de49 --- /dev/null +++ b/tests/common/example_data_generator/consts.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +US_STATES = [ + "AL", + "AK", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "HI", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VT", + "VA", + "WA", + "WV", + "WI", + "WY", + "other", +] diff --git a/tests/common/example_data_generator/string_generator.py b/tests/common/example_data_generator/string_generator.py new file mode 100644 index 0000000000000..101f49c9c47b6 --- /dev/null +++ b/tests/common/example_data_generator/string_generator.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from random import choices, randint + + +class StringGenerator: + _seed_letters: str + _min_length: int + _max_length: int + + def __init__(self, seed_letters: str, min_length: int, max_length: int): + self._seed_letters = seed_letters + self._min_length = min_length + self._max_length = max_length + + def generate(self) -> str: + rv_string_length = randint(self._min_length, self._max_length) + randomized_letters = choices(self._seed_letters, k=rv_string_length) + return "".join(randomized_letters) diff --git a/tests/common/example_data_generator/string_generator_factory.py b/tests/common/example_data_generator/string_generator_factory.py new file mode 100644 index 0000000000000..caca7e2a20e42 --- /dev/null +++ b/tests/common/example_data_generator/string_generator_factory.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import string + +from tests.common.example_data_generator.string_generator import StringGenerator + + +class StringGeneratorFactory: + @classmethod + def make( + cls, seed_letters: str, min_length: int, max_length: int + ) -> StringGenerator: + cls.__validate_arguments(seed_letters, min_length, max_length) + return StringGenerator(seed_letters, min_length, max_length) + + @classmethod + def make_lowercase_based(cls, min_length: int, max_length: int) -> StringGenerator: + return cls.make(string.ascii_lowercase, min_length, max_length) + + @classmethod + def make_ascii_letters_based( + cls, min_length: int, max_length: int + ) -> StringGenerator: + return cls.make(string.ascii_letters, min_length, max_length) + + @staticmethod + def __validate_arguments( + seed_letters: str, min_length: int, max_length: int + ) -> None: + assert seed_letters, "seed_letters is empty" + assert min_length > -1, "min_length is negative" + assert max_length > min_length, "max_length is not bigger then min_length" diff --git a/tests/common/example_data_generator/tests/__init__.py b/tests/common/example_data_generator/tests/__init__.py new file mode 100644 index 0000000000000..dc74f5a8fa252 --- /dev/null +++ b/tests/common/example_data_generator/tests/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/common/example_data_generator/tests/test_string_generator.py b/tests/common/example_data_generator/tests/test_string_generator.py new file mode 100644 index 0000000000000..7088b752d5e29 --- /dev/null +++ b/tests/common/example_data_generator/tests/test_string_generator.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from unittest.mock import Mock, patch + +from tests.common.example_data_generator.string_generator import StringGenerator + + +@patch("tests.common.example_data_generator.string_generator.choices") +@patch("tests.common.example_data_generator.string_generator.randint") +def test_string_generator(randint_mock: Mock, choices_mock: Mock): + letters = "abcdets" + min_len = 3 + max_len = 5 + randomized_string_len = 4 + string_generator = StringGenerator(letters, min_len, max_len) + randint_mock.return_value = randomized_string_len + choices_mock.return_value = ["t", "e", "s", "t"] + + assert string_generator.generate() == "test" + randint_mock.assert_called_once_with(min_len, max_len) + choices_mock.assert_called_with(letters, k=randomized_string_len) diff --git a/tests/integration_tests/fixtures/birth_names_dashboard.py b/tests/integration_tests/fixtures/birth_names_dashboard.py index 56d1bc2870288..89c3c8161af92 100644 --- a/tests/integration_tests/fixtures/birth_names_dashboard.py +++ b/tests/integration_tests/fixtures/birth_names_dashboard.py @@ -14,10 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json -import string -from datetime import date, datetime -from random import choice, getrandbits, randint, random, uniform from typing import Any, Dict, List, Optional import pandas as pd @@ -31,6 +27,9 @@ from superset.models.dashboard import Dashboard from superset.models.slice import Slice from superset.utils.core import get_example_database, get_example_default_schema +from tests.common.example_data_generator.birth_names.birth_names_generator_factory import ( + BirthNamesGeneratorFactory, +) from tests.integration_tests.dashboard_utils import create_table_metadata from tests.integration_tests.test_app import app @@ -148,87 +147,4 @@ def _get_dataframe(database: Database) -> DataFrame: def _get_birth_names_data() -> List[Dict[Any, Any]]: - data = [] - names = generate_names() - for year in range(1960, 2020): - ds = datetime(year, 1, 1, 0, 0, 0) - for _ in range(20): - gender = "boy" if choice([True, False]) else "girl" - num = randint(1, 100000) - data.append( - { - "ds": ds, - "gender": gender, - "name": choice(names), - "num": num, - "state": choice(us_states), - "num_boys": num if gender == "boy" else 0, - "num_girls": num if gender == "girl" else 0, - } - ) - - return data - - -def generate_names() -> List[str]: - names = [] - for _ in range(250): - names.append( - "".join(choice(string.ascii_lowercase) for _ in range(randint(3, 12))) - ) - return names - - -us_states = [ - "AL", - "AK", - "AZ", - "AR", - "CA", - "CO", - "CT", - "DE", - "FL", - "GA", - "HI", - "ID", - "IL", - "IN", - "IA", - "KS", - "KY", - "LA", - "ME", - "MD", - "MA", - "MI", - "MN", - "MS", - "MO", - "MT", - "NE", - "NV", - "NH", - "NJ", - "NM", - "NY", - "NC", - "ND", - "OH", - "OK", - "OR", - "PA", - "RI", - "SC", - "SD", - "TN", - "TX", - "UT", - "VT", - "VA", - "WA", - "WV", - "WI", - "WY", - "other", -] + return list(BirthNamesGeneratorFactory.make().generate())