From ef08520091870423371bbbe54492d09066d399f7 Mon Sep 17 00:00:00 2001 From: Vlad Emelianov Date: Wed, 27 Nov 2024 03:27:49 +0300 Subject: [PATCH] Move lite packages to a separate product --- .github/workflows/integration_full.yml | 2 -- mypy_boto3_builder/cli_parser.py | 7 ---- mypy_boto3_builder/enums/product.py | 16 ++++++++-- mypy_boto3_builder/enums/product_type.py | 1 + .../generators/aioboto3_generator.py | 25 ++++++++------- .../generators/aiobotocore_generator.py | 29 +++++++++-------- .../generators/base_generator.py | 18 +++++++---- .../generators/boto3_generator.py | 32 +++++++++++-------- mypy_boto3_builder/main.py | 25 ++++++++------- mypy_boto3_builder/type_defs.py | 23 +++++++++++++ scripts/integration.py | 1 - 11 files changed, 112 insertions(+), 67 deletions(-) create mode 100644 mypy_boto3_builder/type_defs.py diff --git a/.github/workflows/integration_full.yml b/.github/workflows/integration_full.yml index e7c14a51..c519245c 100644 --- a/.github/workflows/integration_full.yml +++ b/.github/workflows/integration_full.yml @@ -29,13 +29,11 @@ jobs: --product boto3 boto3-full \ -s s3 ec2 \ --no-smart-version \ - --skip-lite \ --output-type=wheel uv run mypy_boto3_builder mypy_boto3_output \ --product aioboto3 aiobotocore aiobotocore-full \ -s s3 \ --no-smart-version \ - --skip-lite \ --output-type=wheel - name: Install full packages run: | diff --git a/mypy_boto3_builder/cli_parser.py b/mypy_boto3_builder/cli_parser.py index d1e61762..dffa78cb 100644 --- a/mypy_boto3_builder/cli_parser.py +++ b/mypy_boto3_builder/cli_parser.py @@ -109,7 +109,6 @@ class CLINamespace: skip_published: bool = False disable_smart_version: bool = False download_static_stubs: bool = True - skip_lite_package: bool = False def parse_args(args: Sequence[str]) -> CLINamespace: @@ -214,11 +213,6 @@ def parse_args(args: Sequence[str]) -> CLINamespace: action="store_true", help="List supported boto3 service names.", ) - parser.add_argument( - "--skip-lite", - action="store_true", - help="Skip generation for *-lite packages.", - ) result = parser.parse_args(args) if result.installed: @@ -236,5 +230,4 @@ def parse_args(args: Sequence[str]) -> CLINamespace: skip_published=result.skip_published, disable_smart_version=result.no_smart_version, download_static_stubs=result.download_static_stubs, - skip_lite_package=result.skip_lite, ) diff --git a/mypy_boto3_builder/enums/product.py b/mypy_boto3_builder/enums/product.py index 8fbc508a..a4407bee 100644 --- a/mypy_boto3_builder/enums/product.py +++ b/mypy_boto3_builder/enums/product.py @@ -16,14 +16,17 @@ class Product(Enum): """ boto3 = "boto3" + boto3_lite = "boto3-lite" boto3_services = "boto3-services" boto3_full = "boto3-full" boto3_docs = "boto3-docs" aiobotocore = "aiobotocore" + aiobotocore_lite = "aiobotocore-lite" aiobotocore_services = "aiobotocore-services" aiobotocore_docs = "aiobotocore-docs" aiobotocore_full = "aiobotocore-full" aioboto3 = "aioboto3" + aioboto3_lite = "aioboto3-lite" aioboto3_docs = "aioboto3-docs" def __str__(self) -> str: @@ -37,16 +40,23 @@ def get_library(self) -> ProductLibrary: Get library name. """ match self: - case Product.boto3 | Product.boto3_services | Product.boto3_full | Product.boto3_docs: + case ( + Product.boto3 + | Product.boto3_lite + | Product.boto3_services + | Product.boto3_full + | Product.boto3_docs + ): return ProductLibrary.boto3 case ( Product.aiobotocore + | Product.aiobotocore_lite | Product.aiobotocore_services | Product.aiobotocore_full | Product.aiobotocore_docs ): return ProductLibrary.aiobotocore - case Product.aioboto3 | Product.aioboto3_docs: + case Product.aioboto3 | Product.aioboto3_lite | Product.aioboto3_docs: return ProductLibrary.aioboto3 def get_type(self) -> ProductType: @@ -56,6 +66,8 @@ def get_type(self) -> ProductType: match self: case Product.boto3 | Product.aiobotocore | Product.aioboto3: return ProductType.stubs + case Product.boto3_lite | Product.aiobotocore_lite | Product.aioboto3_lite: + return ProductType.stubs_lite case Product.boto3_services | Product.aiobotocore_services: return ProductType.service_stubs case Product.boto3_docs | Product.aiobotocore_docs | Product.aioboto3_docs: diff --git a/mypy_boto3_builder/enums/product_type.py b/mypy_boto3_builder/enums/product_type.py index de63e9d6..ad66261c 100644 --- a/mypy_boto3_builder/enums/product_type.py +++ b/mypy_boto3_builder/enums/product_type.py @@ -13,6 +13,7 @@ class ProductType(Enum): """ stubs = "stubs" + stubs_lite = "stubs_lite" service_stubs = "service_stubs" docs = "docs" full = "full" diff --git a/mypy_boto3_builder/generators/aioboto3_generator.py b/mypy_boto3_builder/generators/aioboto3_generator.py index c41e17b7..ba0e41ca 100644 --- a/mypy_boto3_builder/generators/aioboto3_generator.py +++ b/mypy_boto3_builder/generators/aioboto3_generator.py @@ -1,5 +1,5 @@ """ -AioBoto3 stubs generator. +Generator for types-aioboto3 packages. Copyright 2024 Vlad Emelianov """ @@ -27,7 +27,7 @@ class AioBoto3Generator(BaseGenerator): """ - AioBoto3 stubs generator. + Generator for types-aioboto3 packages. """ service_package_data = TypesAioBoto3PackageData @@ -38,7 +38,7 @@ def _get_static_files_path(self) -> Path: StaticStubsPullURL.types_aioboto3, ) - def get_postprocessor(self, service_package: ServicePackage) -> AioBoto3Postprocessor: + def _get_postprocessor(self, service_package: ServicePackage) -> AioBoto3Postprocessor: """ Get postprocessor for service package. """ @@ -48,17 +48,20 @@ def generate_stubs(self) -> list[Package]: """ Generate `types-aioboto3` package. """ - packages: list[Package] = [] - package: Package | None = self._generate_stubs() + package = self._generate_stubs() if package: - packages.append(package) + return [package] - if not self.config.skip_lite_package: - package = self._generate_stubs_lite() - if package: - packages.append(package) + return [] - return packages + def generate_stubs_lite(self) -> list[Package]: + """ + Generate `types-aioboto3-lite` package. + """ + package = self._generate_stubs_lite() + if package: + return [package] + return [] def _generate_stubs(self) -> TypesAioBoto3Package | None: package_data = TypesAioBoto3PackageData diff --git a/mypy_boto3_builder/generators/aiobotocore_generator.py b/mypy_boto3_builder/generators/aiobotocore_generator.py index 393d13a0..3f776867 100644 --- a/mypy_boto3_builder/generators/aiobotocore_generator.py +++ b/mypy_boto3_builder/generators/aiobotocore_generator.py @@ -1,5 +1,5 @@ """ -AioBotocore stubs/docs generator. +Generator for types-aiobotocore packages. Copyright 2024 Vlad Emelianov """ @@ -32,7 +32,7 @@ class AioBotocoreGenerator(BaseGenerator): """ - AioBotocore stubs/docs generator. + Generator for types-aiobotocore packages. """ service_package_data = TypesAioBotocorePackageData @@ -44,7 +44,7 @@ def _get_static_files_path(self) -> Path: StaticStubsPullURL.types_aiobotocore, ) - def get_postprocessor(self, service_package: ServicePackage) -> AioBotocorePostprocessor: + def _get_postprocessor(self, service_package: ServicePackage) -> AioBotocorePostprocessor: """ Get postprocessor for service package. """ @@ -52,19 +52,22 @@ def get_postprocessor(self, service_package: ServicePackage) -> AioBotocorePostp def generate_stubs(self) -> list[Package]: """ - Generate `aiobotocore-stubs` package. + Generate `types-aiobotocore` package. """ - packages: list[Package] = [] - package: Package | None = self._generate_stubs() + package = self._generate_stubs() if package: - packages.append(package) + return [package] - if not self.config.skip_lite_package: - package = self._generate_stubs_lite() - if package: - packages.append(package) + return [] - return packages + def generate_stubs_lite(self) -> list[Package]: + """ + Generate `types-aiobotocore-lite` package. + """ + package = self._generate_stubs_lite() + if package: + return [package] + return [] def _generate_stubs(self) -> TypesAioBotocorePackage | None: package_data = TypesAioBotocorePackageData @@ -127,7 +130,7 @@ def generate_docs(self) -> None: def generate_full_stubs(self) -> list[Package]: """ - Generate full stubs. + Generate `types-aiobotocore-full` package. """ package_data = TypesAioBotocoreFullPackageData try: diff --git a/mypy_boto3_builder/generators/base_generator.py b/mypy_boto3_builder/generators/base_generator.py index 8f43a857..df259d5a 100644 --- a/mypy_boto3_builder/generators/base_generator.py +++ b/mypy_boto3_builder/generators/base_generator.py @@ -59,7 +59,7 @@ def __init__( self.master_service_names = master_service_names self.config = config self.logger = get_logger() - self.version = version or self.get_library_version() + self.version = version or self._get_library_version() self.cleanup = cleanup self.package_writer = PackageWriter( output_path=self.output_path, @@ -125,12 +125,12 @@ def _get_or_download_static_files_path( return self._downloaded_static_files_path @abstractmethod - def get_postprocessor(self, service_package: ServicePackage) -> BasePostprocessor: + def _get_postprocessor(self, service_package: ServicePackage) -> BasePostprocessor: """ Get postprocessor for service package. """ - def get_library_version(self) -> str: + def _get_library_version(self) -> str: """ Get underlying library version. """ @@ -153,14 +153,18 @@ def generate_stubs(self) -> Sequence[Package]: """ Generate main stubs. """ - raise NotImplementedError("Method should be implemented in child class") + + @abstractmethod + def generate_stubs_lite(self) -> Sequence[Package]: + """ + Generate main stubs. + """ @abstractmethod def generate_full_stubs(self) -> Sequence[Package]: """ Generate full stubs. """ - raise NotImplementedError("Method should be implemented in child class") @abstractmethod def generate_custom_stubs(self) -> Sequence[Package]: @@ -210,6 +214,8 @@ def generate_product(self, product_type: ProductType) -> None: match product_type: case ProductType.stubs: packages.extend(self.generate_stubs()) + case ProductType.stubs_lite: + packages.extend(self.generate_stubs_lite()) case ProductType.service_stubs: packages.extend(self.generate_service_stubs()) case ProductType.docs: @@ -237,7 +243,7 @@ def _parse_service_package( parser = ServicePackageParser(service_name, package_data, version) service_package = parser.parse() - postprocessor = self.get_postprocessor(service_package) + postprocessor = self._get_postprocessor(service_package) postprocessor.generate_docstrings() postprocessor.process_package() diff --git a/mypy_boto3_builder/generators/boto3_generator.py b/mypy_boto3_builder/generators/boto3_generator.py index be8bd5ad..ad672801 100644 --- a/mypy_boto3_builder/generators/boto3_generator.py +++ b/mypy_boto3_builder/generators/boto3_generator.py @@ -1,5 +1,5 @@ """ -Boto3 stubs/docs generator. +Generator for boto3-stubs packages. Copyright 2024 Vlad Emelianov """ @@ -35,13 +35,13 @@ class Boto3Generator(BaseGenerator): """ - Boto3 stubs/docs generator. + Generator for boto3-stubs packages. """ service_package_data = Boto3StubsPackageData service_template_path = TemplatePath.boto3_stubs_service - def get_postprocessor(self, service_package: ServicePackage) -> BotocorePostprocessor: + def _get_postprocessor(self, service_package: ServicePackage) -> BotocorePostprocessor: """ Get postprocessor for service package. """ @@ -72,7 +72,7 @@ def _get_static_files_path(self) -> Path: StaticStubsPullURL.boto3_stubs, ) - def _generate_boto3_stubs(self) -> Boto3StubsPackage | None: + def _generate_stubs(self) -> Boto3StubsPackage | None: package_data = Boto3StubsPackageData try: version = self._get_package_version(package_data.PYPI_NAME, self.version) @@ -89,7 +89,7 @@ def _generate_boto3_stubs(self) -> Boto3StubsPackage | None: static_files_path=self._get_static_files_path(), ) - def _generate_boto3_stubs_lite(self) -> Boto3StubsPackage | None: + def _generate_stubs_lite(self) -> Boto3StubsPackage | None: package_data = Boto3StubsLitePackageData try: version = self._get_package_version(package_data.PYPI_NAME, self.version) @@ -108,27 +108,31 @@ def _generate_boto3_stubs_lite(self) -> Boto3StubsPackage | None: def generate_stubs(self) -> list[Package]: """ - Generate main stubs. + Generate `boto3-stubs` package. """ result: list[Package] = [] package: Package | None = None - package = self._generate_boto3_stubs() + package = self._generate_stubs() if package: result.append(package) - if not self.config.skip_lite_package and self.is_package(): + if not self.is_package(): package = self._generate_master() if package: result.append(package) - if not self.config.skip_lite_package: - package = self._generate_boto3_stubs_lite() - if package: - result.append(package) - return result + def generate_stubs_lite(self) -> list[Package]: + """ + Generate `boto3-stubs-lite` package. + """ + package = self._generate_stubs_lite() + if package: + return [package] + return [] + def generate_docs(self) -> None: """ Generate service and master docs. @@ -156,7 +160,7 @@ def generate_docs(self) -> None: def generate_full_stubs(self) -> list[Package]: """ - Generate full stubs. + Generate `boto3-stubs-full` package. """ package_data = Boto3StubsFullPackageData try: diff --git a/mypy_boto3_builder/main.py b/mypy_boto3_builder/main.py index b24ffabe..291385fa 100644 --- a/mypy_boto3_builder/main.py +++ b/mypy_boto3_builder/main.py @@ -21,6 +21,7 @@ from mypy_boto3_builder.jinja_manager import JinjaManager from mypy_boto3_builder.logger import get_logger from mypy_boto3_builder.service_name import ServiceName, ServiceNameCatalog +from mypy_boto3_builder.type_defs import GeneratorKwargs from mypy_boto3_builder.utils.boto3_utils import get_botocore_session from mypy_boto3_builder.utils.botocore_changelog import BotocoreChangelog from mypy_boto3_builder.utils.strings import get_anchor_link, get_botocore_class_name, get_copyright @@ -105,18 +106,18 @@ def get_available_service_names(session: BotocoreSession) -> list[ServiceName]: return result -def get_generator_cls(product: Product) -> type[BaseGenerator]: +def get_generator(product: Product, kwargs: GeneratorKwargs) -> BaseGenerator: """ Get Generator class for a product. """ library = product.get_library() match library: case ProductLibrary.boto3: - return Boto3Generator + return Boto3Generator(**kwargs) case ProductLibrary.aiobotocore: - return AioBotocoreGenerator + return AioBotocoreGenerator(**kwargs) case ProductLibrary.aioboto3: - return AioBoto3Generator + return AioBoto3Generator(**kwargs) def generate_product( @@ -134,13 +135,15 @@ def generate_product( service_names -- Selected service names master_service_names -- Service names included in master """ - generator_cls = get_generator_cls(product) - generator = generator_cls( - service_names=service_names, - master_service_names=master_service_names, - config=args, - version=args.build_version, - cleanup=True, + generator = get_generator( + product, + { + "service_names": service_names, + "master_service_names": master_service_names, + "config": args, + "version": args.build_version, + "cleanup": True, + }, ) generator.generate_product(product.get_type()) generator.cleanup_temporary_files() diff --git a/mypy_boto3_builder/type_defs.py b/mypy_boto3_builder/type_defs.py new file mode 100644 index 00000000..b7d68d33 --- /dev/null +++ b/mypy_boto3_builder/type_defs.py @@ -0,0 +1,23 @@ +""" +Type definitions for builder. + +Copyright 2024 Vlad Emelianov +""" + +from collections.abc import Sequence +from typing import TypedDict + +from mypy_boto3_builder.cli_parser import CLINamespace +from mypy_boto3_builder.service_name import ServiceName + + +class GeneratorKwargs(TypedDict): + """ + Generator keyword arguments. + """ + + service_names: Sequence[ServiceName] + master_service_names: Sequence[ServiceName] + config: CLINamespace + version: str + cleanup: bool diff --git a/scripts/integration.py b/scripts/integration.py index 8e55a33a..c3509843 100755 --- a/scripts/integration.py +++ b/scripts/integration.py @@ -216,7 +216,6 @@ def build_packages( skip_published=False, partial_overload=False, list_services=False, - skip_lite_package=True, ) )