Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: The file structure of stubs resembles the "package" path. #106

Merged
merged 15 commits into from
May 2, 2024
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/safeds_stubgen/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,9 @@
"""Safe-DS stubs generator."""

from __future__ import annotations

from ._helpers import is_internal

__all__ = [
"is_internal",
]
2 changes: 2 additions & 0 deletions src/safeds_stubgen/_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def is_internal(name: str) -> bool:
return name.startswith("_")
5 changes: 1 addition & 4 deletions src/safeds_stubgen/api_analyzer/_ast_visitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import mypy.types as mp_types

import safeds_stubgen.api_analyzer._types as sds_types
from safeds_stubgen import is_internal
from safeds_stubgen.docstring_parsing import ResultDocstring

from ._api import (
Expand Down Expand Up @@ -1187,10 +1188,6 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module |
return None


def is_internal(name: str) -> bool:
return name.startswith("_")


def result_name_generator() -> Generator:
"""Generate a name for callable type parameters starting from 'a' until 'zz'."""
while True:
Expand Down
18 changes: 12 additions & 6 deletions src/safeds_stubgen/api_analyzer/cli/_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import TYPE_CHECKING

from safeds_stubgen.api_analyzer import get_api
from safeds_stubgen.stubs_generator import generate_stubs
from safeds_stubgen.stubs_generator import StubsStringGenerator, create_stub_files, generate_stub_data

if TYPE_CHECKING:
from safeds_stubgen.docstring_parsing import DocstringStyle
Expand All @@ -17,7 +17,7 @@ def cli() -> None:
if args.verbose:
logging.basicConfig(level=logging.INFO)

_run_api_command(args.src, args.out, args.docstyle, args.testrun, args.naming_convert)
_run_stub_generator(args.src, args.out, args.docstyle, args.testrun, args.naming_convert)


def _get_args() -> argparse.Namespace:
Expand Down Expand Up @@ -64,15 +64,15 @@ def _get_args() -> argparse.Namespace:
return parser.parse_args()


def _run_api_command(
def _run_stub_generator(
src_dir_path: Path,
out_dir_path: Path,
docstring_style: DocstringStyle,
is_test_run: bool,
convert_identifiers: bool,
) -> None:
"""
List the API of a package.
Create API data of a package and Safe-DS stub files.

Parameters
----------
Expand All @@ -83,8 +83,14 @@ def _run_api_command(
is_test_run : bool
Set True if files in test directories should be parsed too.
"""
api = get_api(src_dir_path, docstring_style, is_test_run)
# Generate the API data
api = get_api(root=src_dir_path, docstring_style=docstring_style, is_test_run=is_test_run)
# Create an API file
out_file_api = out_dir_path.joinpath(f"{src_dir_path.stem}__api.json")
api.to_json_file(out_file_api)

generate_stubs(api, out_dir_path, convert_identifiers)
# Generate the stub data
stubs_generator = StubsStringGenerator(api=api, convert_identifiers=convert_identifiers)
stub_data = generate_stub_data(stubs_generator=stubs_generator, out_path=out_dir_path)
# Create the stub files
create_stub_files(stubs_generator=stubs_generator, stubs_data=stub_data, out_path=out_dir_path)
7 changes: 5 additions & 2 deletions src/safeds_stubgen/stubs_generator/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@

from __future__ import annotations

from ._generate_stubs import generate_stubs
from ._generate_stubs import NamingConvention, StubsStringGenerator, create_stub_files, generate_stub_data

__all__ = [
"generate_stubs",
"create_stub_files",
"generate_stub_data",
"NamingConvention",
"StubsStringGenerator",
]
59 changes: 26 additions & 33 deletions src/safeds_stubgen/stubs_generator/_generate_stubs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from types import NoneType
from typing import TYPE_CHECKING

from safeds_stubgen import is_internal
from safeds_stubgen.api_analyzer import (
API,
Attribute,
Expand Down Expand Up @@ -34,33 +35,27 @@ class NamingConvention(IntEnum):
SAFE_DS = 2


def generate_stubs(api: API, out_path: Path, convert_identifiers: bool) -> None:
def generate_stub_data(
stubs_generator: StubsStringGenerator,
out_path: Path,
) -> list[tuple[Path, str, str]]:
"""Generate Safe-DS stubs.

Generates stub files from an API object and writes them to the out_path path.
Generates stub data from an API object.

Parameters
----------
api
The API object from which the stubs
stubs_generator
The class for generating the stubs.
out_path
The path in which the stub files should be created. If no such path exists this function creates the directory
files.
convert_identifiers
Set this True if the identifiers should be converted to Safe-DS standard (UpperCamelCase for classes and
camelCase for everything else).
"""
naming_convention = NamingConvention.SAFE_DS if convert_identifiers else NamingConvention.PYTHON
stubs_generator = StubsStringGenerator(api, naming_convention)
stubs_data = _generate_stubs_data(api, out_path, stubs_generator)
_generate_stubs_files(stubs_data, out_path, stubs_generator, naming_convention)


def _generate_stubs_data(
api: API,
out_path: Path,
stubs_generator: StubsStringGenerator,
) -> list[tuple[Path, str, str]]:
Returns
-------
A list of tuples, which are 1. the path of the stub file, 2. the name of the stub file and 3. its content.
"""
api = stubs_generator.api
stubs_data: list[tuple[Path, str, str]] = []
for module in api.modules.values():
if module.name == "__init__":
Expand All @@ -69,7 +64,7 @@ def _generate_stubs_data(
log_msg = f"Creating stub data for {module.id}"
logging.info(log_msg)

module_text = stubs_generator(module)
module_text, package_info = stubs_generator(module)

# Each text block we create ends with "\n", therefore, if there is only the package information
# the file would look like this: "package path.to.myPackage\n" or this:
Expand All @@ -83,17 +78,17 @@ def _generate_stubs_data(
if len(splitted_text) <= 2 or (len(splitted_text) == 3 and splitted_text[1].startswith("package ")):
continue

module_dir = Path(out_path / module.id)
module_dir = Path(out_path / package_info.replace(".", "/"))
stubs_data.append((module_dir, module.name, module_text))
return stubs_data


def _generate_stubs_files(
def create_stub_files(
stubs_generator: StubsStringGenerator,
stubs_data: list[tuple[Path, str, str]],
out_path: Path,
stubs_generator: StubsStringGenerator,
naming_convention: NamingConvention,
) -> None:
naming_convention = stubs_generator.naming_convention
for module_dir, module_name, module_text in stubs_data:
log_msg = f"Creating stub file for {module_dir}"
logging.info(log_msg)
Expand All @@ -102,7 +97,8 @@ def _generate_stubs_files(
module_dir.mkdir(parents=True, exist_ok=True)

# Create and open module file
file_path = Path(module_dir / f"{module_name}.sdsstub")
public_module_name = module_name.lstrip("_")
file_path = Path(module_dir / f"{public_module_name}.sdsstub")
Path(file_path).touch()

with file_path.open("w") as f:
Expand Down Expand Up @@ -179,19 +175,19 @@ class StubsStringGenerator:
method.
"""

def __init__(self, api: API, naming_convention: NamingConvention) -> None:
def __init__(self, api: API, convert_identifiers: bool) -> None:
self.api = api
self.naming_convention = naming_convention
self.naming_convention = NamingConvention.SAFE_DS if convert_identifiers else NamingConvention.PYTHON
self.classes_outside_package: set[str] = set()

def __call__(self, module: Module) -> str:
def __call__(self, module: Module) -> tuple[str, str]:
self.module_imports: set[str] = set()
self._current_todo_msgs: set[str] = set()
self.module = module
self.class_generics: list = []
return self._create_module_string()

def _create_module_string(self) -> str:
def _create_module_string(self) -> tuple[str, str]:
# Create package info
package_info = self._get_shortest_public_reexport()
package_info_camel_case = _convert_name_to_convention(package_info, self.naming_convention)
Expand Down Expand Up @@ -223,7 +219,7 @@ def _create_module_string(self) -> str:
# Create imports - We have to create them last, since we have to check all used types in this module first
module_header += self._create_imports_string()

return docstring + module_header + module_text
return f"{docstring}{module_header}{module_text}", package_info

def _create_imports_string(self) -> str:
if not self.module_imports:
Expand Down Expand Up @@ -1137,15 +1133,12 @@ def _replace_if_safeds_keyword(keyword: str) -> str:
"sub",
"super",
"_",
"unknown",
}:
return f"`{keyword}`"
return keyword


def is_internal(name: str) -> bool:
return name.startswith("_")


def _convert_name_to_convention(
name: str,
naming_convention: NamingConvention,
Expand Down
84 changes: 36 additions & 48 deletions tests/safeds_stubgen/stubs_generator/test_generate_stubs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,17 @@
import pytest
from safeds_stubgen.api_analyzer import get_api
from safeds_stubgen.docstring_parsing import DocstringStyle
from safeds_stubgen.stubs_generator import generate_stubs
from safeds_stubgen.stubs_generator import NamingConvention, StubsStringGenerator, create_stub_files, generate_stub_data

# noinspection PyProtectedMember
from safeds_stubgen.stubs_generator._generate_stubs import (
NamingConvention,
StubsStringGenerator,
_convert_name_to_convention,
_generate_stubs_data,
_generate_stubs_files,
)
from safeds_stubgen.stubs_generator._generate_stubs import _convert_name_to_convention

if TYPE_CHECKING:
from collections.abc import Generator

from syrupy import SnapshotAssertion


# Setup - Run API to create stub files
_lib_dir = Path(__file__).parent.parent.parent
_test_package_name = "various_modules_package"
Expand All @@ -33,50 +28,43 @@
_docstring_package_dir = Path(_lib_dir / "data" / _docstring_package_name)

api = get_api(_test_package_dir, is_test_run=True)
stubs_generator = StubsStringGenerator(api, naming_convention=NamingConvention.SAFE_DS)
stubs_data = _generate_stubs_data(api, _out_dir, stubs_generator)
stubs_generator = StubsStringGenerator(api=api, convert_identifiers=True)
stubs_data = generate_stub_data(stubs_generator=stubs_generator, out_path=_out_dir)


def test_file_creation() -> None:
_generate_stubs_files(stubs_data, _out_dir, stubs_generator, naming_convention=NamingConvention.SAFE_DS)
_assert_file_creation_recursive(
python_path=Path(_test_package_dir / "file_creation"),
stub_path=Path(_out_dir_stubs / "file_creation"),
)


def _assert_file_creation_recursive(python_path: Path, stub_path: Path) -> None:
assert python_path.is_dir()
assert stub_path.is_dir()

python_files: list[Path] = list(python_path.iterdir())
stub_files: list[Path] = list(stub_path.iterdir())

# Remove __init__ files and private files without public reexported content.
# We reexport public content from _module_3 and _module_6, not from empty_module, _module_2 and _module_4.
actual_python_files = []
for item in python_files:
if not (item.is_file() and item.stem in {"__init__", "_module_2", "_module_4"}):
actual_python_files.append(item)

assert len(actual_python_files) == len(stub_files)

actual_python_files.sort(key=lambda x: x.stem)
stub_files.sort(key=lambda x: x.stem)

for py_item, stub_item in zip(actual_python_files, stub_files, strict=True):
if py_item.is_file():
assert stub_item.is_dir()
stub_files = list(stub_item.iterdir())
assert len(stub_files) == 1
assert stub_files[0].stem == py_item.stem
else:
_assert_file_creation_recursive(py_item, stub_item)
data_to_test: list[tuple[str, str]] = [
("/".join(stub_data[0].parts), stub_data[1]) for stub_data in stubs_data if "file_creation" in str(stub_data[0])
]
data_to_test.sort(key=lambda x: x[1])

expected_files: list[tuple[str, str]] = [
# We reexport these three modules from another package into the file_creation package.
("tests/data/various_modules_package/file_creation", "_reexported_from_another_package"),
("tests/data/various_modules_package/file_creation", "_reexported_from_another_package_2"),
("tests/data/various_modules_package/file_creation", "_reexported_from_another_package_3"),
# module_1 is public
("tests/data/various_modules_package/file_creation/module_1", "module_1"),
# _module_6 has a public reexport in the file_creation package
("tests/data/various_modules_package/file_creation", "_module_6"),
# module_5 is publich
("tests/data/various_modules_package/file_creation/package_1/module_5", "module_5"),
# _module_3 is not created, even though it is reexported, since it's also reexported in the parent
# package.
# _module_2 is not created, since the reexport is still private
# _module_4 is not created, since it has no (public) reexport
]
expected_files.sort(key=lambda x: x[1])

assert len(data_to_test) == len(expected_files)
for data_tuple, expected_tuple in zip(data_to_test, expected_files, strict=True):
assert data_tuple[0].endswith(expected_tuple[0])
assert data_tuple[1] == expected_tuple[1]


def test_file_creation_limited_stubs_outside_package(snapshot_sds_stub: SnapshotAssertion) -> None:
# Somehow the stubs get overwritten by other tests, therefore we have to call the function before asserting
generate_stubs(api, _out_dir, convert_identifiers=True)
create_stub_files(stubs_generator=stubs_generator, stubs_data=stubs_data, out_path=_out_dir)

path = Path(_out_dir / "tests/data/main_package/another_path/another_module/another_module.sdsstub")
assert path.is_file()

Expand Down Expand Up @@ -161,8 +149,8 @@ def test_stub_docstring_creation(
docstring_style=docstring_style,
is_test_run=True,
)
docstring_stubs_generator = StubsStringGenerator(docstring_api, naming_convention=NamingConvention.SAFE_DS)
docstring_stubs_data = _generate_stubs_data(docstring_api, _out_dir, docstring_stubs_generator)
docstring_stubs_generator = StubsStringGenerator(api=docstring_api, convert_identifiers=True)
docstring_stubs_data = generate_stub_data(stubs_generator=docstring_stubs_generator, out_path=_out_dir)

for stub_text in docstring_stubs_data:
if stub_text[1] == filename:
Expand Down