From 0a5d2ae4b72f08dede759417d9c46e4f73d2d92a Mon Sep 17 00:00:00 2001 From: ofekisr Date: Thu, 30 Dec 2021 13:26:16 +0200 Subject: [PATCH] refactor: replace the way the birth_names data is generated --- .../common/example_data_generator/__init__.py | 25 +++++ tests/common/example_data_generator/base.py | 34 +++++++ .../birth_names_generator.py | 80 ++++++++++++++++ .../birth_names_generator_factory.py | 65 +++++++++++++ tests/common/example_data_generator/consts.py | 69 ++++++++++++++ .../string_generator.py | 42 +++++++++ .../string_generator_factory.py | 46 ++++++++++ .../fixtures/birth_names_dashboard.py | 92 +------------------ 8 files changed, 365 insertions(+), 88 deletions(-) create mode 100644 tests/common/example_data_generator/__init__.py create mode 100644 tests/common/example_data_generator/base.py create mode 100644 tests/common/example_data_generator/birth_names_generator.py create mode 100644 tests/common/example_data_generator/birth_names_generator_factory.py create mode 100644 tests/common/example_data_generator/consts.py create mode 100644 tests/common/example_data_generator/string_generator.py create mode 100644 tests/common/example_data_generator/string_generator_factory.py diff --git a/tests/common/example_data_generator/__init__.py b/tests/common/example_data_generator/__init__.py new file mode 100644 index 0000000000000..dc74f5a8fa252 --- /dev/null +++ b/tests/common/example_data_generator/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/common/example_data_generator/base.py b/tests/common/example_data_generator/base.py new file mode 100644 index 0000000000000..2d149a4999f90 --- /dev/null +++ b/tests/common/example_data_generator/base.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable + + +class ExampleDataGenerator(ABC): + @abstractmethod + def generate(self) -> Iterable[Dict[Any, Any]]: + pass diff --git a/tests/common/example_data_generator/birth_names_generator.py b/tests/common/example_data_generator/birth_names_generator.py new file mode 100644 index 0000000000000..93f389372b5e4 --- /dev/null +++ b/tests/common/example_data_generator/birth_names_generator.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime +from random import choice, randint +from typing import Any, Dict, Iterable + +from tests.common.example_data_generator.base import ExampleDataGenerator +from tests.common.example_data_generator.consts import US_STATES +from tests.common.example_data_generator.string_generator import StringGenerator + +GIRL = "girl" +BOY = "boy" + + +class BirthNamesGenerator(ExampleDataGenerator): + _names_generator: StringGenerator + _start_year: int + _until_not_include_year: int + _rows_per_year: int + + def __init__( + self, + names_generator: StringGenerator, + start_year: int, + years_amount: int, + rows_per_year: int, + ) -> None: + assert start_year > -1 + assert years_amount > 0 + self._names_generator = names_generator + self._start_year = start_year + self._until_not_include_year = start_year + years_amount + self._rows_per_year = rows_per_year + + def generate(self) -> Iterable[Dict[Any, Any]]: + for year in range(self._start_year, self._until_not_include_year): + ds = self._make_year(year) + for _ in range(self._rows_per_year): + yield self.generate_row(ds) + + def _make_year(self, year: int): + return datetime(year, 1, 1, 0, 0, 0) + + def generate_row(self, dt: datetime) -> Dict[Any, Any]: + gender = choice([BOY, GIRL]) + num = randint(1, 100000) + return { + "ds": dt, + "gender": gender, + "name": self._names_generator.generate(), + "num": num, + "state": choice(US_STATES), + "num_boys": num if gender == BOY else 0, + "num_girls": num if gender == GIRL else 0, + } diff --git a/tests/common/example_data_generator/birth_names_generator_factory.py b/tests/common/example_data_generator/birth_names_generator_factory.py new file mode 100644 index 0000000000000..c5f65cc89edf7 --- /dev/null +++ b/tests/common/example_data_generator/birth_names_generator_factory.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from abc import ABC, abstractmethod + +from tests.common.example_data_generator.birth_names_generator import ( + BirthNamesGenerator, +) +from tests.common.example_data_generator.string_generator_factory import ( + StringGeneratorFactory, +) + + +class BirthNamesGeneratorFactory(ABC): + __factory: BirthNamesGeneratorFactory + + @abstractmethod + def _make(self) -> BirthNamesGenerator: + pass + + @classmethod + def make(cls) -> BirthNamesGenerator: + return cls._get_instance()._make() + + @classmethod + def set_instance(cls, factory: BirthNamesGeneratorFactory) -> None: + cls.__factory = factory + + @classmethod + def _get_instance(cls) -> BirthNamesGeneratorFactory: + if not hasattr(cls, "_BirthNamesGeneratorFactory__factory"): + cls.__factory = BirthNamesGeneratorFactoryImpl() + return cls.__factory + + +MIN_NAME_LEN = 3 +MAX_NAME_SIZE = 10 +START_YEAR = 1960 +YEARS_AMOUNT = 60 +ROW_PER_YEAR = 20 + + +class BirthNamesGeneratorFactoryImpl(BirthNamesGeneratorFactory): + def _make(self) -> BirthNamesGenerator: + string_generator = StringGeneratorFactory.make_lowercase_based( + MIN_NAME_LEN, MAX_NAME_SIZE + ) + return BirthNamesGenerator( + string_generator, START_YEAR, YEARS_AMOUNT, ROW_PER_YEAR + ) diff --git a/tests/common/example_data_generator/consts.py b/tests/common/example_data_generator/consts.py new file mode 100644 index 0000000000000..2432d60b6de49 --- /dev/null +++ b/tests/common/example_data_generator/consts.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +US_STATES = [ + "AL", + "AK", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "HI", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VT", + "VA", + "WA", + "WV", + "WI", + "WY", + "other", +] diff --git a/tests/common/example_data_generator/string_generator.py b/tests/common/example_data_generator/string_generator.py new file mode 100644 index 0000000000000..101f49c9c47b6 --- /dev/null +++ b/tests/common/example_data_generator/string_generator.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from random import choices, randint + + +class StringGenerator: + _seed_letters: str + _min_length: int + _max_length: int + + def __init__(self, seed_letters: str, min_length: int, max_length: int): + self._seed_letters = seed_letters + self._min_length = min_length + self._max_length = max_length + + def generate(self) -> str: + rv_string_length = randint(self._min_length, self._max_length) + randomized_letters = choices(self._seed_letters, k=rv_string_length) + return "".join(randomized_letters) diff --git a/tests/common/example_data_generator/string_generator_factory.py b/tests/common/example_data_generator/string_generator_factory.py new file mode 100644 index 0000000000000..caca7e2a20e42 --- /dev/null +++ b/tests/common/example_data_generator/string_generator_factory.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import string + +from tests.common.example_data_generator.string_generator import StringGenerator + + +class StringGeneratorFactory: + @classmethod + def make( + cls, seed_letters: str, min_length: int, max_length: int + ) -> StringGenerator: + cls.__validate_arguments(seed_letters, min_length, max_length) + return StringGenerator(seed_letters, min_length, max_length) + + @classmethod + def make_lowercase_based(cls, min_length: int, max_length: int) -> StringGenerator: + return cls.make(string.ascii_lowercase, min_length, max_length) + + @classmethod + def make_ascii_letters_based( + cls, min_length: int, max_length: int + ) -> StringGenerator: + return cls.make(string.ascii_letters, min_length, max_length) + + @staticmethod + def __validate_arguments( + seed_letters: str, min_length: int, max_length: int + ) -> None: + assert seed_letters, "seed_letters is empty" + assert min_length > -1, "min_length is negative" + assert max_length > min_length, "max_length is not bigger then min_length" diff --git a/tests/integration_tests/fixtures/birth_names_dashboard.py b/tests/integration_tests/fixtures/birth_names_dashboard.py index 56d1bc2870288..25046886b0c8e 100644 --- a/tests/integration_tests/fixtures/birth_names_dashboard.py +++ b/tests/integration_tests/fixtures/birth_names_dashboard.py @@ -14,10 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json -import string -from datetime import date, datetime -from random import choice, getrandbits, randint, random, uniform from typing import Any, Dict, List, Optional import pandas as pd @@ -31,6 +27,9 @@ from superset.models.dashboard import Dashboard from superset.models.slice import Slice from superset.utils.core import get_example_database, get_example_default_schema +from tests.common.example_data_generator.birth_names_generator_factory import ( + BirthNamesGeneratorFactory, +) from tests.integration_tests.dashboard_utils import create_table_metadata from tests.integration_tests.test_app import app @@ -148,87 +147,4 @@ def _get_dataframe(database: Database) -> DataFrame: def _get_birth_names_data() -> List[Dict[Any, Any]]: - data = [] - names = generate_names() - for year in range(1960, 2020): - ds = datetime(year, 1, 1, 0, 0, 0) - for _ in range(20): - gender = "boy" if choice([True, False]) else "girl" - num = randint(1, 100000) - data.append( - { - "ds": ds, - "gender": gender, - "name": choice(names), - "num": num, - "state": choice(us_states), - "num_boys": num if gender == "boy" else 0, - "num_girls": num if gender == "girl" else 0, - } - ) - - return data - - -def generate_names() -> List[str]: - names = [] - for _ in range(250): - names.append( - "".join(choice(string.ascii_lowercase) for _ in range(randint(3, 12))) - ) - return names - - -us_states = [ - "AL", - "AK", - "AZ", - "AR", - "CA", - "CO", - "CT", - "DE", - "FL", - "GA", - "HI", - "ID", - "IL", - "IN", - "IA", - "KS", - "KY", - "LA", - "ME", - "MD", - "MA", - "MI", - "MN", - "MS", - "MO", - "MT", - "NE", - "NV", - "NH", - "NJ", - "NM", - "NY", - "NC", - "ND", - "OH", - "OK", - "OR", - "PA", - "RI", - "SC", - "SD", - "TN", - "TX", - "UT", - "VT", - "VA", - "WA", - "WV", - "WI", - "WY", - "other", -] + return list(BirthNamesGeneratorFactory.make().generate())