From c1780aa8b61581fafc95fe499a8929a1317bfa91 Mon Sep 17 00:00:00 2001 From: Jesse Michel <jmmichel@mit.edu> Date: Sat, 18 Feb 2023 16:56:16 -0500 Subject: [PATCH 01/21] exist tokenizer over-escapes --- tap/utils.py | 3 +-- tests/test_utils.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tap/utils.py b/tap/utils.py index 602db57..8a3c2ce 100644 --- a/tap/utils.py +++ b/tap/utils.py @@ -184,7 +184,6 @@ def tokenize_source(obj: object) -> Generator: """Returns a generator for the tokens of the object's source code.""" source = inspect.getsource(obj) token_generator = tokenize.generate_tokens(StringIO(source).readline) - return token_generator @@ -206,7 +205,7 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj): line_to_tokens.setdefault(start_line, []).append({ 'token_type': token_type, - 'token': token, + 'token': bytes(token, encoding='ascii').decode('unicode-escape'), 'start_line': start_line, 'start_column': start_column, 'end_line': end_line, diff --git a/tests/test_utils.py b/tests/test_utils.py index 837cc8a..61362a3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -294,6 +294,19 @@ class TripleQuoteMultiline: class_variables['hi'] = {'comment': 'Hello there'} self.assertEqual(get_class_variables(TripleQuoteMultiline), class_variables) + def test_comments_with_quotes(self): + class MultiquoteMultiline: + bar: int = 0 + '\'\'biz baz\'' + + hi: str + "\"Hello there\"\"" + + class_variables = OrderedDict() + class_variables['bar'] = {'comment': "''biz baz'"} + class_variables['hi'] = {'comment': '"Hello there""'} + self.assertEqual(get_class_variables(MultiquoteMultiline), class_variables) + def test_single_quote_multiline(self): class SingleQuoteMultiline: bar: int = 0 From d7f7f9defcf3038a31e2aebf292ae4df56ba336d Mon Sep 17 00:00:00 2001 From: Alexander Shadchin <shadchin@yandex-team.com> Date: Sun, 11 Aug 2024 16:28:40 +0300 Subject: [PATCH 02/21] Fix `test_add_subparsers_twice` with Python 3.12.5 --- tests/test_subparser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_subparser.py b/tests/test_subparser.py index b73cb24..b618fcc 100644 --- a/tests/test_subparser.py +++ b/tests/test_subparser.py @@ -134,8 +134,12 @@ def configure(self): self.add_subparsers(help="sub-command1 help") self.add_subparsers(help="sub-command2 help") - with self.assertRaises(SystemExit): - Args().parse_args([]) + if sys.version_info >= (3, 12, 5): + with self.assertRaises(ArgumentError): + Args().parse_args([]) + else: + with self.assertRaises(SystemExit): + Args().parse_args([]) def test_add_subparsers_with_add_argument(self): class SubparserA(Tap): From 48a4ac2914d62bd7781703d6698329755e73fb2b Mon Sep 17 00:00:00 2001 From: Jesse Michel <jmmichel@mit.edu> Date: Sat, 24 Aug 2024 17:20:27 -0400 Subject: [PATCH 03/21] merge --- src/tap/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 4cf9d42..08fdd4b 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -203,7 +203,6 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in """Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code.""" line_to_tokens = {} for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj): -<<<<<<< HEAD:tap/utils.py line_to_tokens.setdefault(start_line, []).append({ 'token_type': token_type, 'token': bytes(token, encoding='ascii').decode('unicode-escape'), @@ -213,19 +212,6 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in 'end_column': end_column, 'line': line }) -======= - line_to_tokens.setdefault(start_line, []).append( - { - "token_type": token_type, - "token": token, - "start_line": start_line, - "start_column": start_column, - "end_line": end_line, - "end_column": end_column, - "line": line, - } - ) ->>>>>>> main:src/tap/utils.py return line_to_tokens From bf9818808944a27d1ec9437c55a82fe93ced7fae Mon Sep 17 00:00:00 2001 From: Jesse Michel <jmmichel@mit.edu> Date: Sat, 24 Aug 2024 17:46:55 -0400 Subject: [PATCH 04/21] add fix for quotation marks in docstrings (issue 97) --- src/tap/utils.py | 17 +++++++++++++++-- tests/test_utils.py | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 08fdd4b..7695c65 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -205,7 +205,7 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj): line_to_tokens.setdefault(start_line, []).append({ 'token_type': token_type, - 'token': bytes(token, encoding='ascii').decode('unicode-escape'), + 'token': token, 'start_line': start_line, 'start_column': start_column, 'end_line': end_line, @@ -241,8 +241,21 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: and token["token"][:1] in {'"', "'"} ): sep = " " if variable_to_comment[class_variable]["comment"] else "" + + # Identify the quote character (single or double) quote_char = token["token"][:1] - variable_to_comment[class_variable]["comment"] += sep + token["token"].strip(quote_char).strip() + + # Identify the number of quote characters at the start of the string + num_quote_chars = len(token["token"]) - len(token["token"].lstrip(quote_char)) + + # Remove the number of quote characters at the start of the string and the end of the string + token["token"] = token["token"][num_quote_chars:-num_quote_chars] + + # Remove the unicode escape sequences (e.g. "\"") + token["token"] = bytes(token["token"], encoding='ascii').decode('unicode-escape') + + # Add the token to the comment, stripping whitespace + variable_to_comment[class_variable]["comment"] += sep + token["token"].strip() # Match class variable class_variable = None diff --git a/tests/test_utils.py b/tests/test_utils.py index e16d8d3..68b2d8c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -308,7 +308,7 @@ class MultiquoteMultiline: hi: str "\"Hello there\"\"" - class_variables = OrderedDict() + class_variables = {} class_variables['bar'] = {'comment': "''biz baz'"} class_variables['hi'] = {'comment': '"Hello there""'} self.assertEqual(get_class_variables(MultiquoteMultiline), class_variables) From 6405e3006dca76a9a962aa475bc4c4f7d98365e5 Mon Sep 17 00:00:00 2001 From: Jesse Michel <jmmichel@mit.edu> Date: Sat, 24 Aug 2024 18:36:08 -0400 Subject: [PATCH 05/21] include additional test case --- tests/test_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 68b2d8c..ab12c81 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -313,6 +313,17 @@ class MultiquoteMultiline: class_variables['hi'] = {'comment': '"Hello there""'} self.assertEqual(get_class_variables(MultiquoteMultiline), class_variables) + def test_multiline_argument(self): + class MultilineArgument: + bar: str = ( + "This is a multiline argument" + " that should not be included in the docstring" + ) + ("""biz baz""") + + class_variables = {"bar": {"comment": "biz baz"}} + self.assertEqual(get_class_variables(MultilineArgument), class_variables) + def test_single_quote_multiline(self): class SingleQuoteMultiline: bar: int = 0 From 69f78d38ebadcc09d4d3425f4fc4dc3bba1b08d9 Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sat, 24 Aug 2024 16:18:08 -0700 Subject: [PATCH 06/21] Fixing comment extraction in the case of multiline assign statements using ast --- src/tap/utils.py | 60 +++++++++++++++++++++++++++++++++++++++++++-- tests/test_utils.py | 2 +- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 7695c65..7e22c86 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -1,4 +1,5 @@ from argparse import ArgumentParser, ArgumentTypeError +import ast from base64 import b64encode, b64decode import copy from functools import wraps @@ -10,6 +11,7 @@ import re import subprocess import sys +import textwrap import tokenize from typing import ( Any, @@ -24,6 +26,7 @@ Union, ) from typing_inspect import get_args as typing_inspect_get_args, get_origin as typing_inspect_get_origin +import warnings if sys.version_info >= (3, 10): from types import UnionType @@ -216,6 +219,52 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in return line_to_tokens +def get_subsequent_assign_lines(cls: type) -> set[int]: + """For all multiline assign statements, get the line numbers after the first line of the assignment.""" + # Get source code of class + source = inspect.getsource(cls) + + # Parse source code using ast (with an if statement to avoid indentation errors) + source = f"if True:\n{textwrap.indent(source, ' ')}" + body = ast.parse(source).body[0] + + # Set up warning message + parse_warning = ( + "Could not parse class source code to extract comments. " + "Comments in the help string may be incorrect." + ) + + # Check for correct parsing + if not isinstance(body, ast.If): + warnings.warn(parse_warning) + return set() + + # Extract if body + if_body = body.body + + # Check for a single body + if len(if_body) != 1: + warnings.warn(parse_warning) + return set() + + # Extract class body + cls_body = if_body[0] + + # Check for a single class definition + if not isinstance(cls_body, ast.ClassDef): + warnings.warn(parse_warning) + return set() + + # Get line numbers of assign statements + assign_lines = set() + for node in cls_body.body: + if isinstance(node, (ast.Assign, ast.AnnAssign)): + # Get line number of assign statement excluding the first line (and minus 1 for the if statement) + assign_lines |= set(range(node.lineno, node.end_lineno)) + + return assign_lines + + def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: """Returns a dictionary mapping class variables to their additional information (currently just comments).""" # Get mapping from line number to tokens @@ -224,12 +273,19 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: # Get class variable column number class_variable_column = get_class_column(cls) + # For all multiline assign statements, get the line numbers after the first line of the assignment + # This is used to avoid identifying comments in multiline assign statements + subsequent_assign_lines = get_subsequent_assign_lines(cls) + # Extract class variables class_variable = None variable_to_comment = {} - for tokens in line_to_tokens.values(): - for i, token in enumerate(tokens): + for line, tokens in line_to_tokens.items(): + # Skip assign lines after the first line of multiline assign statements + if line in subsequent_assign_lines: + continue + for i, token in enumerate(tokens): # Skip whitespace if token["token"].strip() == "": continue diff --git a/tests/test_utils.py b/tests/test_utils.py index ab12c81..33729ca 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -319,7 +319,7 @@ class MultilineArgument: "This is a multiline argument" " that should not be included in the docstring" ) - ("""biz baz""") + """biz baz""" class_variables = {"bar": {"comment": "biz baz"}} self.assertEqual(get_class_variables(MultilineArgument), class_variables) From 3980c817a85fceb4b9092fbb92642e96fdbb6870 Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sat, 24 Aug 2024 16:31:28 -0700 Subject: [PATCH 07/21] Fixing Python 3.8 type hint issue --- src/tap/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 7e22c86..9f22c34 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -22,6 +22,7 @@ List, Literal, Optional, + Set, Tuple, Union, ) @@ -219,7 +220,7 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in return line_to_tokens -def get_subsequent_assign_lines(cls: type) -> set[int]: +def get_subsequent_assign_lines(cls: type) -> Set[int]: """For all multiline assign statements, get the line numbers after the first line of the assignment.""" # Get source code of class source = inspect.getsource(cls) From a7d851b5b360db15b0eb3aed9ecea41c28055b18 Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Wed, 28 Aug 2024 03:16:51 +0200 Subject: [PATCH 08/21] ignore Tap from class variables parsing --- src/tap/tap.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/tap/tap.py b/src/tap/tap.py index 09a5134..79f6147 100644 --- a/src/tap/tap.py +++ b/src/tap/tap.py @@ -483,7 +483,9 @@ def parse_args( return self @classmethod - def _get_from_self_and_super(cls, extract_func: Callable[[type], dict]) -> Union[Dict[str, Any], Dict]: + def _get_from_self_and_super( + cls, extract_func: Callable[[type], dict], *, need_tap_cls: bool = True, + ) -> Union[Dict[str, Any], Dict]: """Returns a dictionary mapping variable names to values. Variables and values are extracted from classes using key starting @@ -494,6 +496,7 @@ def _get_from_self_and_super(cls, extract_func: Callable[[type], dict]) -> Union Super classes are traversed through breadth first search. :param extract_func: A function that extracts from a class a dictionary mapping variables to values. + :param need_tap_cls: If False, variables from the Tap and ArgumentParser classes are ignored. :return: A dictionary mapping variable names to values from the class dict. """ visited = set() @@ -503,6 +506,9 @@ def _get_from_self_and_super(cls, extract_func: Callable[[type], dict]) -> Union while len(super_classes) > 0: super_class = super_classes.pop(0) + if not need_tap_cls and super_class is Tap: + break + if super_class not in visited and issubclass(super_class, Tap): super_dictionary = extract_func(super_class) @@ -521,7 +527,8 @@ def _get_from_self_and_super(cls, extract_func: Callable[[type], dict]) -> Union def _get_class_dict(self) -> Dict[str, Any]: """Returns a dictionary mapping class variable names to values from the class dict.""" class_dict = self._get_from_self_and_super( - extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())) + extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())), + need_tap_cls=False, ) class_dict = { var: val @@ -529,9 +536,7 @@ def _get_class_dict(self) -> Dict[str, Any]: if not ( var.startswith("_") or callable(val) - or isinstance(val, staticmethod) - or isinstance(val, classmethod) - or isinstance(val, property) + or isinstance(val, (staticmethod, classmethod, property)) ) } @@ -539,7 +544,10 @@ def _get_class_dict(self) -> Dict[str, Any]: def _get_annotations(self) -> Dict[str, Any]: """Returns a dictionary mapping variable names to their type annotations.""" - return self._get_from_self_and_super(extract_func=lambda super_class: dict(get_type_hints(super_class))) + return self._get_from_self_and_super( + extract_func=lambda super_class: dict(get_type_hints(super_class)), + need_tap_cls=False, + ) def _get_class_variables(self) -> dict: """Returns a dictionary mapping class variables names to their additional information.""" @@ -547,7 +555,7 @@ def _get_class_variables(self) -> dict: try: class_variables = self._get_from_self_and_super( - extract_func=lambda super_class: get_class_variables(super_class) + extract_func=get_class_variables, need_tap_cls=False, ) # Handle edge-case of source code modification while code is running @@ -588,7 +596,8 @@ def as_dict(self) -> Dict[str, Any]: self_dict = self.__dict__ class_dict = self._get_from_self_and_super( - extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())) + extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())), + need_tap_cls=False, ) class_dict = {key: val for key, val in class_dict.items() if key not in self_dict} stored_dict = {**self_dict, **class_dict} From 03c9dc4d71ded3f1c930ebd8d3f8a84e32286eca Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Wed, 28 Aug 2024 03:30:45 +0200 Subject: [PATCH 09/21] only calls inspect.getsource and tokenize.generated tokens once --- src/tap/utils.py | 59 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 9f22c34..872cc1f 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -18,6 +18,7 @@ Callable, Dict, Generator, + Iterable, Iterator, List, Literal, @@ -184,17 +185,20 @@ def is_positional_arg(*name_or_flags) -> bool: return not is_option_arg(*name_or_flags) +def _tokenize_source(source: str) -> Generator[tokenize.TokenInfo, None, None]: + """Returns a generator for the tokens of the object's source code, given the source code.""" + return tokenize.generate_tokens(StringIO(source).readline) + + def tokenize_source(obj: object) -> Generator: """Returns a generator for the tokens of the object's source code.""" - source = inspect.getsource(obj) - token_generator = tokenize.generate_tokens(StringIO(source).readline) - return token_generator + return _tokenize_source(inspect.getsource(obj)) -def get_class_column(obj: type) -> int: - """Determines the column number for class variables in a class.""" +def _get_class_column(tokens: Iterable[tokenize.TokenInfo]) -> int: + """Determines the column number for class variables in a class, given the tokens of the class.""" first_line = 1 - for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj): + for token_type, token, (start_line, start_column), (end_line, end_column), line in tokens: if token.strip() == "@": first_line += 1 if start_line <= first_line or token.strip() == "": @@ -203,10 +207,18 @@ def get_class_column(obj: type) -> int: return start_column -def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, int]]]]: - """Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code.""" +def get_class_column(cls: type) -> int: + """Determines the column number for class variables in a class.""" + return _get_class_column(tokenize_source(cls)) + + +def _source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, List[Dict[str, Union[str, int]]]]: + """ + Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code, + given the tokens of the object's source code. + """ line_to_tokens = {} - for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj): + for token_type, token, (start_line, start_column), (end_line, end_column), line in tokens: line_to_tokens.setdefault(start_line, []).append({ 'token_type': token_type, 'token': token, @@ -220,13 +232,19 @@ def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, in return line_to_tokens -def get_subsequent_assign_lines(cls: type) -> Set[int]: - """For all multiline assign statements, get the line numbers after the first line of the assignment.""" - # Get source code of class - source = inspect.getsource(cls) +def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, int]]]]: + """Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code.""" + return _source_line_to_tokens(tokenize_source(obj)) + + +def _get_subsequent_assign_lines(source_cls: str) -> Set[int]: + """ + For all multiline assign statements, get the line numbers after the first line of the assignment, + given the source code of the object. + """ # Parse source code using ast (with an if statement to avoid indentation errors) - source = f"if True:\n{textwrap.indent(source, ' ')}" + source = f"if True:\n{textwrap.indent(source_cls, ' ')}" body = ast.parse(source).body[0] # Set up warning message @@ -265,18 +283,25 @@ def get_subsequent_assign_lines(cls: type) -> Set[int]: return assign_lines +def get_subsequent_assign_lines(cls: type) -> Set[int]: + """For all multiline assign statements, get the line numbers after the first line of the assignment.""" + return _get_subsequent_assign_lines(inspect.getsource(cls)) def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: """Returns a dictionary mapping class variables to their additional information (currently just comments).""" + # Get the source code and tokens of the class + source_cls = inspect.getsource(cls) + tokens = tuple(_tokenize_source(source_cls)) + # Get mapping from line number to tokens - line_to_tokens = source_line_to_tokens(cls) + line_to_tokens = _source_line_to_tokens(tokens) # Get class variable column number - class_variable_column = get_class_column(cls) + class_variable_column = _get_class_column(tokens) # For all multiline assign statements, get the line numbers after the first line of the assignment # This is used to avoid identifying comments in multiline assign statements - subsequent_assign_lines = get_subsequent_assign_lines(cls) + subsequent_assign_lines = _get_subsequent_assign_lines(source_cls) # Extract class variables class_variable = None From 96182f8b147c235aa7fd1df0959d68c8bcf803d7 Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Wed, 28 Aug 2024 03:49:55 +0200 Subject: [PATCH 10/21] remove the need_tap argument since it is not used --- src/tap/tap.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/tap/tap.py b/src/tap/tap.py index 79f6147..c925c6e 100644 --- a/src/tap/tap.py +++ b/src/tap/tap.py @@ -483,9 +483,7 @@ def parse_args( return self @classmethod - def _get_from_self_and_super( - cls, extract_func: Callable[[type], dict], *, need_tap_cls: bool = True, - ) -> Union[Dict[str, Any], Dict]: + def _get_from_self_and_super( cls, extract_func: Callable[[type], dict]) -> Union[Dict[str, Any], Dict]: """Returns a dictionary mapping variable names to values. Variables and values are extracted from classes using key starting @@ -496,7 +494,6 @@ def _get_from_self_and_super( Super classes are traversed through breadth first search. :param extract_func: A function that extracts from a class a dictionary mapping variables to values. - :param need_tap_cls: If False, variables from the Tap and ArgumentParser classes are ignored. :return: A dictionary mapping variable names to values from the class dict. """ visited = set() @@ -506,10 +503,7 @@ def _get_from_self_and_super( while len(super_classes) > 0: super_class = super_classes.pop(0) - if not need_tap_cls and super_class is Tap: - break - - if super_class not in visited and issubclass(super_class, Tap): + if super_class not in visited and issubclass(super_class, Tap) and super_class is not Tap: super_dictionary = extract_func(super_class) # Update only unseen variables to avoid overriding subclass values @@ -527,8 +521,7 @@ def _get_from_self_and_super( def _get_class_dict(self) -> Dict[str, Any]: """Returns a dictionary mapping class variable names to values from the class dict.""" class_dict = self._get_from_self_and_super( - extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())), - need_tap_cls=False, + extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())) ) class_dict = { var: val @@ -545,8 +538,7 @@ def _get_class_dict(self) -> Dict[str, Any]: def _get_annotations(self) -> Dict[str, Any]: """Returns a dictionary mapping variable names to their type annotations.""" return self._get_from_self_and_super( - extract_func=lambda super_class: dict(get_type_hints(super_class)), - need_tap_cls=False, + extract_func=lambda super_class: dict(get_type_hints(super_class)) ) def _get_class_variables(self) -> dict: @@ -554,9 +546,7 @@ def _get_class_variables(self) -> dict: class_variable_names = {**self._get_annotations(), **self._get_class_dict()}.keys() try: - class_variables = self._get_from_self_and_super( - extract_func=get_class_variables, need_tap_cls=False, - ) + class_variables = self._get_from_self_and_super(extract_func=get_class_variables) # Handle edge-case of source code modification while code is running variables_to_add = class_variable_names - class_variables.keys() @@ -597,7 +587,6 @@ def as_dict(self) -> Dict[str, Any]: self_dict = self.__dict__ class_dict = self._get_from_self_and_super( extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())), - need_tap_cls=False, ) class_dict = {key: val for key, val in class_dict.items() if key not in self_dict} stored_dict = {**self_dict, **class_dict} From a9f931572f6ae568b45677b53dffb21c6c2ed141 Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Wed, 28 Aug 2024 03:55:03 +0200 Subject: [PATCH 11/21] formatting --- src/tap/tap.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tap/tap.py b/src/tap/tap.py index c925c6e..c2460a7 100644 --- a/src/tap/tap.py +++ b/src/tap/tap.py @@ -483,7 +483,7 @@ def parse_args( return self @classmethod - def _get_from_self_and_super( cls, extract_func: Callable[[type], dict]) -> Union[Dict[str, Any], Dict]: + def _get_from_self_and_super(cls, extract_func: Callable[[type], dict]) -> Union[Dict[str, Any], Dict]: """Returns a dictionary mapping variable names to values. Variables and values are extracted from classes using key starting @@ -537,9 +537,7 @@ def _get_class_dict(self) -> Dict[str, Any]: def _get_annotations(self) -> Dict[str, Any]: """Returns a dictionary mapping variable names to their type annotations.""" - return self._get_from_self_and_super( - extract_func=lambda super_class: dict(get_type_hints(super_class)) - ) + return self._get_from_self_and_super(extract_func=lambda super_class: dict(get_type_hints(super_class))) def _get_class_variables(self) -> dict: """Returns a dictionary mapping class variables names to their additional information.""" @@ -586,7 +584,7 @@ def as_dict(self) -> Dict[str, Any]: self_dict = self.__dict__ class_dict = self._get_from_self_and_super( - extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())), + extract_func=lambda super_class: dict(getattr(super_class, "__dict__", dict())) ) class_dict = {key: val for key, val in class_dict.items() if key not in self_dict} stored_dict = {**self_dict, **class_dict} From d3b1874c18c37e841755fee97bbaf888df23c5ba Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Thu, 29 Aug 2024 19:53:28 +0200 Subject: [PATCH 12/21] Remove unused functions, adjust the tests for the new ones --- src/tap/utils.py | 35 +++++++++-------------------------- tests/test_utils.py | 20 ++++++++++++++------ 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 872cc1f..793a3e0 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -185,17 +185,12 @@ def is_positional_arg(*name_or_flags) -> bool: return not is_option_arg(*name_or_flags) -def _tokenize_source(source: str) -> Generator[tokenize.TokenInfo, None, None]: +def tokenize_source(source: str) -> Generator[tokenize.TokenInfo, None, None]: """Returns a generator for the tokens of the object's source code, given the source code.""" return tokenize.generate_tokens(StringIO(source).readline) -def tokenize_source(obj: object) -> Generator: - """Returns a generator for the tokens of the object's source code.""" - return _tokenize_source(inspect.getsource(obj)) - - -def _get_class_column(tokens: Iterable[tokenize.TokenInfo]) -> int: +def get_class_column(tokens: Iterable[tokenize.TokenInfo]) -> int: """Determines the column number for class variables in a class, given the tokens of the class.""" first_line = 1 for token_type, token, (start_line, start_column), (end_line, end_column), line in tokens: @@ -205,14 +200,10 @@ def _get_class_column(tokens: Iterable[tokenize.TokenInfo]) -> int: continue return start_column + raise ValueError("Could not find any class variables in the class.") -def get_class_column(cls: type) -> int: - """Determines the column number for class variables in a class.""" - return _get_class_column(tokenize_source(cls)) - - -def _source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, List[Dict[str, Union[str, int]]]]: +def source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, List[Dict[str, Union[str, int]]]]: """ Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code, given the tokens of the object's source code. @@ -232,12 +223,7 @@ def _source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, Li return line_to_tokens -def source_line_to_tokens(obj: object) -> Dict[int, List[Dict[str, Union[str, int]]]]: - """Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code.""" - return _source_line_to_tokens(tokenize_source(obj)) - - -def _get_subsequent_assign_lines(source_cls: str) -> Set[int]: +def get_subsequent_assign_lines(source_cls: str) -> Set[int]: """ For all multiline assign statements, get the line numbers after the first line of the assignment, given the source code of the object. @@ -283,25 +269,22 @@ def _get_subsequent_assign_lines(source_cls: str) -> Set[int]: return assign_lines -def get_subsequent_assign_lines(cls: type) -> Set[int]: - """For all multiline assign statements, get the line numbers after the first line of the assignment.""" - return _get_subsequent_assign_lines(inspect.getsource(cls)) def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: """Returns a dictionary mapping class variables to their additional information (currently just comments).""" # Get the source code and tokens of the class source_cls = inspect.getsource(cls) - tokens = tuple(_tokenize_source(source_cls)) + tokens = tuple(tokenize_source(source_cls)) # Get mapping from line number to tokens - line_to_tokens = _source_line_to_tokens(tokens) + line_to_tokens = source_line_to_tokens(tokens) # Get class variable column number - class_variable_column = _get_class_column(tokens) + class_variable_column = get_class_column(tokens) # For all multiline assign statements, get the line numbers after the first line of the assignment # This is used to avoid identifying comments in multiline assign statements - subsequent_assign_lines = _get_subsequent_assign_lines(source_cls) + subsequent_assign_lines = get_subsequent_assign_lines(source_cls) # Extract class variables class_variable = None diff --git a/tests/test_utils.py b/tests/test_utils.py index 33729ca..c349b49 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ from argparse import ArgumentTypeError +import inspect import json import os import subprocess @@ -11,6 +12,7 @@ get_class_column, get_class_variables, GitInfo, + tokenize_source, type_to_str, get_literals, TupleTypeEnforcer, @@ -145,7 +147,8 @@ def test_column_simple(self): class SimpleColumn: arg = 2 - self.assertEqual(get_class_column(SimpleColumn), 12) + tokens = tokenize_source(inspect.getsource(SimpleColumn)) + self.assertEqual(get_class_column(tokens), 12) def test_column_comment(self): class CommentColumn: @@ -158,28 +161,32 @@ class CommentColumn: arg = 2 - self.assertEqual(get_class_column(CommentColumn), 12) + tokens = tokenize_source(inspect.getsource(CommentColumn)) + self.assertEqual(get_class_column(tokens), 12) def test_column_space(self): class SpaceColumn: arg = 2 - self.assertEqual(get_class_column(SpaceColumn), 12) + tokens = tokenize_source(inspect.getsource(SpaceColumn)) + self.assertEqual(get_class_column(tokens), 12) def test_column_method(self): class FuncColumn: def func(self): pass - self.assertEqual(get_class_column(FuncColumn), 12) + tokens = tokenize_source(inspect.getsource(FuncColumn)) + self.assertEqual(get_class_column(tokens), 12) def test_dataclass(self): @class_decorator class DataclassColumn: arg: int = 5 - self.assertEqual(get_class_column(DataclassColumn), 12) + tokens = tokenize_source(inspect.getsource(DataclassColumn)) + self.assertEqual(get_class_column(tokens), 12) def test_dataclass_method(self): def wrapper(f): @@ -191,7 +198,8 @@ class DataclassColumn: def func(self): pass - self.assertEqual(get_class_column(DataclassColumn), 12) + tokens = tokenize_source(inspect.getsource(DataclassColumn)) + self.assertEqual(get_class_column(tokens), 12) class ClassVariableTests(TestCase): From e64c747377e09f77c21341ec823dbea87667feee Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Thu, 29 Aug 2024 19:58:05 +0200 Subject: [PATCH 13/21] Fix edge case where node.end_lineno is None --- src/tap/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tap/utils.py b/src/tap/utils.py index 793a3e0..d5e1d16 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -264,6 +264,11 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: assign_lines = set() for node in cls_body.body: if isinstance(node, (ast.Assign, ast.AnnAssign)): + # Check if the end line number is found + if node.end_lineno is None: + warnings.warn(parse_warning) + return set() + # Get line number of assign statement excluding the first line (and minus 1 for the if statement) assign_lines |= set(range(node.lineno, node.end_lineno)) From 09cb610410e6b4addc6929dcf41b2680d964feca Mon Sep 17 00:00:00 2001 From: arnaud-ma <arnaudma.code@gmail.com> Date: Thu, 29 Aug 2024 20:56:09 +0200 Subject: [PATCH 14/21] replace return to continue in the loop --- src/tap/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index d5e1d16..c664517 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -267,7 +267,7 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: # Check if the end line number is found if node.end_lineno is None: warnings.warn(parse_warning) - return set() + continue # Get line number of assign statement excluding the first line (and minus 1 for the if statement) assign_lines |= set(range(node.lineno, node.end_lineno)) From b78111e97b5e58b28d59c341d0f8f95bbec89981 Mon Sep 17 00:00:00 2001 From: Jesse Michel <jmmichel@mit.edu> Date: Sun, 8 Sep 2024 17:39:04 -0400 Subject: [PATCH 15/21] Improve documentation, typing compliance, and removed deprecated code --- src/tap/tap.py | 16 ++++------- src/tap/utils.py | 69 +++++++++++++----------------------------------- 2 files changed, 23 insertions(+), 62 deletions(-) diff --git a/src/tap/tap.py b/src/tap/tap.py index c2460a7..5c8668b 100644 --- a/src/tap/tap.py +++ b/src/tap/tap.py @@ -26,7 +26,6 @@ TupleTypeEnforcer, define_python_object_encoder, as_python_object, - fix_py36_copy, enforce_reproducibility, PathLike, ) @@ -227,7 +226,7 @@ def _add_argument(self, *name_or_flags, **kwargs) -> None: # Handle Tuple type (with type args) by extracting types of Tuple elements and enforcing them elif get_origin(var_type) in (Tuple, tuple) and len(get_args(var_type)) > 0: loop = False - types = get_args(var_type) + types = list(get_args(var_type)) # Handle Tuple[type, ...] if len(types) == 2 and types[1] == Ellipsis: @@ -331,7 +330,7 @@ def add_subparser(self, flag: str, subparser_type: type, **kwargs) -> None: self._subparser_buffer.append((flag, subparser_type, kwargs)) def _add_subparsers(self) -> None: - """Add each of the subparsers to the Tap object. """ + """Add each of the subparsers to the Tap object.""" # Initialize the _subparsers object if not already created if self._subparsers is None and len(self._subparser_buffer) > 0: self._subparsers = super(Tap, self).add_subparsers() @@ -345,7 +344,7 @@ def add_subparsers(self, **kwargs) -> None: self._subparsers = super().add_subparsers(**kwargs) def _configure(self) -> None: - """Executes the user-defined configuration. """ + """Executes the user-defined configuration.""" # Call the user-defined configuration self.configure() @@ -526,11 +525,7 @@ def _get_class_dict(self) -> Dict[str, Any]: class_dict = { var: val for var, val in class_dict.items() - if not ( - var.startswith("_") - or callable(val) - or isinstance(val, (staticmethod, classmethod, property)) - ) + if not (var.startswith("_") or callable(val) or isinstance(val, (staticmethod, classmethod, property))) } return class_dict @@ -712,7 +707,6 @@ def __str__(self) -> str: """ return pformat(self.as_dict()) - @fix_py36_copy def __deepcopy__(self, memo: Dict[int, Any] = None) -> TapType: """Deepcopy the Tap object.""" copied = type(self).__new__(type(self)) @@ -722,7 +716,7 @@ def __deepcopy__(self, memo: Dict[int, Any] = None) -> TapType: memo[id(self)] = copied - for (k, v) in self.__dict__.items(): + for k, v in self.__dict__.items(): copied.__dict__[k] = deepcopy(v, memo) return copied diff --git a/src/tap/utils.py b/src/tap/utils.py index c664517..3e6ef7c 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -146,7 +146,7 @@ def get_argument_name(*name_or_flags) -> str: return "help" if len(name_or_flags) > 1: - name_or_flags = [n_or_f for n_or_f in name_or_flags if n_or_f.startswith("--")] + name_or_flags = tuple(n_or_f for n_or_f in name_or_flags if n_or_f.startswith("--")) if len(name_or_flags) != 1: raise ValueError(f"There should only be a single canonical name for argument {name_or_flags}!") @@ -204,39 +204,33 @@ def get_class_column(tokens: Iterable[tokenize.TokenInfo]) -> int: def source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, List[Dict[str, Union[str, int]]]]: - """ - Gets a dictionary mapping from line number to a dictionary of tokens on that line for an object's source code, - given the tokens of the object's source code. - """ + """Extract a map from each line number to list of mappings providing information about each token.""" line_to_tokens = {} for token_type, token, (start_line, start_column), (end_line, end_column), line in tokens: - line_to_tokens.setdefault(start_line, []).append({ - 'token_type': token_type, - 'token': token, - 'start_line': start_line, - 'start_column': start_column, - 'end_line': end_line, - 'end_column': end_column, - 'line': line - }) + line_to_tokens.setdefault(start_line, []).append( + { + "token_type": token_type, + "token": token, + "start_line": start_line, + "start_column": start_column, + "end_line": end_line, + "end_column": end_column, + "line": line, + } + ) return line_to_tokens def get_subsequent_assign_lines(source_cls: str) -> Set[int]: - """ - For all multiline assign statements, get the line numbers after the first line of the assignment, - given the source code of the object. - """ - + """For all multiline assign statements, get the line numbers after the first line in the assignment.""" # Parse source code using ast (with an if statement to avoid indentation errors) source = f"if True:\n{textwrap.indent(source_cls, ' ')}" body = ast.parse(source).body[0] # Set up warning message parse_warning = ( - "Could not parse class source code to extract comments. " - "Comments in the help string may be incorrect." + "Could not parse class source code to extract comments. Comments in the help string may be incorrect." ) # Check for correct parsing @@ -322,7 +316,7 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: token["token"] = token["token"][num_quote_chars:-num_quote_chars] # Remove the unicode escape sequences (e.g. "\"") - token["token"] = bytes(token["token"], encoding='ascii').decode('unicode-escape') + token["token"] = bytes(token["token"], encoding="ascii").decode("unicode-escape") # Add the token to the comment, stripping whitespace variable_to_comment[class_variable]["comment"] += sep + token["token"].strip() @@ -351,7 +345,7 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: return variable_to_comment -def get_literals(literal: Literal, variable: str) -> Tuple[Callable[[str], Any], List[str]]: +def get_literals(literal: Literal, variable: str) -> Tuple[Callable[[str], Any], List[type]]: """Extracts the values from a Literal type and ensures that the values are all primitive types.""" literals = list(get_args(literal)) @@ -476,7 +470,7 @@ def default(self, obj: Any) -> Any: class UnpicklableObject: - """A class that serves as a placeholder for an object that could not be pickled. """ + """A class that serves as a placeholder for an object that could not be pickled.""" def __eq__(self, other): return isinstance(other, UnpicklableObject) @@ -508,33 +502,6 @@ def as_python_object(dct: Any) -> Any: return dct -def fix_py36_copy(func: Callable) -> Callable: - """Decorator that fixes functions using Python 3.6 deepcopy of ArgumentParsers. - - Based on https://stackoverflow.com/questions/6279305/typeerror-cannot-deepcopy-this-pattern-object - """ - if sys.version_info[:2] > (3, 6): - return func - - @wraps(func) - def wrapper(*args, **kwargs): - re_type = type(re.compile("")) - has_prev_val = re_type in copy._deepcopy_dispatch - prev_val = copy._deepcopy_dispatch.get(re_type, None) - copy._deepcopy_dispatch[type(re.compile(""))] = lambda r, _: r - - result = func(*args, **kwargs) - - if has_prev_val: - copy._deepcopy_dispatch[re_type] = prev_val - else: - del copy._deepcopy_dispatch[re_type] - - return result - - return wrapper - - def enforce_reproducibility( saved_reproducibility_data: Optional[Dict[str, str]], current_reproducibility_data: Dict[str, str], path: PathLike ) -> None: From 02f4358ac7fd15b0f03c4d814c0db22741ddc6cc Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sat, 28 Sep 2024 15:26:23 -0700 Subject: [PATCH 16/21] Fixing an issue to enable extraction of hashtag comments from the end of multiline assign statements --- src/tap/utils.py | 43 ++++++++++++++++++++++++++++++++----------- tests/test_utils.py | 17 +++++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 3e6ef7c..7a1e5b9 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -222,8 +222,12 @@ def source_line_to_tokens(tokens: Iterable[tokenize.TokenInfo]) -> Dict[int, Lis return line_to_tokens -def get_subsequent_assign_lines(source_cls: str) -> Set[int]: - """For all multiline assign statements, get the line numbers after the first line in the assignment.""" +def get_subsequent_assign_lines(source_cls: str) -> Tuple[Set[int], Set[int]]: + """For all multiline assign statements, get the line numbers after the first line in the assignment. + + :param source_cls: The source code of the class. + :return: A set of intermediate line numbers for multiline assign statements and a set of final line numbers. + """ # Parse source code using ast (with an if statement to avoid indentation errors) source = f"if True:\n{textwrap.indent(source_cls, ' ')}" body = ast.parse(source).body[0] @@ -236,7 +240,7 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: # Check for correct parsing if not isinstance(body, ast.If): warnings.warn(parse_warning) - return set() + return set(), set() # Extract if body if_body = body.body @@ -244,7 +248,7 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: # Check for a single body if len(if_body) != 1: warnings.warn(parse_warning) - return set() + return set(), set() # Extract class body cls_body = if_body[0] @@ -252,10 +256,11 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: # Check for a single class definition if not isinstance(cls_body, ast.ClassDef): warnings.warn(parse_warning) - return set() + return set(), set() # Get line numbers of assign statements - assign_lines = set() + intermediate_assign_lines = set() + final_assign_lines = set() for node in cls_body.body: if isinstance(node, (ast.Assign, ast.AnnAssign)): # Check if the end line number is found @@ -263,10 +268,15 @@ def get_subsequent_assign_lines(source_cls: str) -> Set[int]: warnings.warn(parse_warning) continue - # Get line number of assign statement excluding the first line (and minus 1 for the if statement) - assign_lines |= set(range(node.lineno, node.end_lineno)) + # Only consider multiline assign statements + if node.end_lineno > node.lineno: + # Get intermediate line number of assign statement excluding the first line (and minus 1 for the if statement) + intermediate_assign_lines |= set(range(node.lineno, node.end_lineno - 1)) + + # If multiline assign statement, get the line number of the last line (and minus 1 for the if statement) + final_assign_lines.add(node.end_lineno - 1) - return assign_lines + return intermediate_assign_lines, final_assign_lines def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: @@ -283,14 +293,25 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: # For all multiline assign statements, get the line numbers after the first line of the assignment # This is used to avoid identifying comments in multiline assign statements - subsequent_assign_lines = get_subsequent_assign_lines(source_cls) + intermediate_assign_lines, final_assign_lines = get_subsequent_assign_lines(source_cls) # Extract class variables class_variable = None variable_to_comment = {} for line, tokens in line_to_tokens.items(): + # If this is the final line of a multiline assign, extract any potential comments + if line in final_assign_lines: + # Find the comment (if it exists) + for token in tokens: + print(token) + if token["token_type"] == tokenize.COMMENT: + # Leave out "#" and whitespace from comment + variable_to_comment[class_variable]["comment"] = token["token"][1:].strip() + break + continue + # Skip assign lines after the first line of multiline assign statements - if line in subsequent_assign_lines: + if line in intermediate_assign_lines: continue for i, token in enumerate(tokens): diff --git a/tests/test_utils.py b/tests/test_utils.py index c349b49..8f707bd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -332,6 +332,23 @@ class MultilineArgument: class_variables = {"bar": {"comment": "biz baz"}} self.assertEqual(get_class_variables(MultilineArgument), class_variables) + def test_multiline_argument_with_final_hashtag_comment(self): + class MultilineArgument: + bar: str = ( + "This is a multiline argument" + " that should not be included in the docstring" + ) # biz baz + barr: str = ( + "This is a multiline argument" + " that should not be included in the docstring") # bar baz + barrr: str = ( # meow + "This is a multiline argument" # blah + " that should not be included in the docstring" # grrrr + ) # yay! + + class_variables = {"bar": {"comment": "biz baz"}, "barr": {"comment": "bar baz"}, "barrr": {"comment": "yay!"}} + self.assertEqual(get_class_variables(MultilineArgument), class_variables) + def test_single_quote_multiline(self): class SingleQuoteMultiline: bar: int = 0 From fdc0000bbc32b348cef386f5319d4e2c449cae21 Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sat, 28 Sep 2024 15:43:57 -0700 Subject: [PATCH 17/21] Fixing multiline comment test for python 3.8 and removing print statement --- src/tap/utils.py | 1 - tests/test_utils.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tap/utils.py b/src/tap/utils.py index 7a1e5b9..2bf788c 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -303,7 +303,6 @@ def get_class_variables(cls: type) -> Dict[str, Dict[str, str]]: if line in final_assign_lines: # Find the comment (if it exists) for token in tokens: - print(token) if token["token_type"] == tokenize.COMMENT: # Leave out "#" and whitespace from comment variable_to_comment[class_variable]["comment"] = token["token"][1:].strip() diff --git a/tests/test_utils.py b/tests/test_utils.py index 8f707bd..dd27283 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -333,7 +333,7 @@ class MultilineArgument: self.assertEqual(get_class_variables(MultilineArgument), class_variables) def test_multiline_argument_with_final_hashtag_comment(self): - class MultilineArgument: + class MultilineArgumentWithHashTagComment: bar: str = ( "This is a multiline argument" " that should not be included in the docstring" @@ -347,7 +347,7 @@ class MultilineArgument: ) # yay! class_variables = {"bar": {"comment": "biz baz"}, "barr": {"comment": "bar baz"}, "barrr": {"comment": "yay!"}} - self.assertEqual(get_class_variables(MultilineArgument), class_variables) + self.assertEqual(get_class_variables(MultilineArgumentWithHashTagComment), class_variables) def test_single_quote_multiline(self): class SingleQuoteMultiline: From 1a3af2a9e48fa2f0b2af25359f658cd55dd65a6b Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sat, 28 Sep 2024 15:49:04 -0700 Subject: [PATCH 18/21] Literally one is actually three --- tests/test_integration.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 7ff7795..97d82b4 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1247,19 +1247,19 @@ class TupleClassTap(Tap): self.assertEqual(args.tup, true_args) - def test_tuple_literally_one(self): + def test_tuple_literally_three(self): class LiterallyOne(Tap): - tup: Tuple[Literal[1]] + tup: Tuple[Literal[3]] - input_args = ("1",) - true_args = (1,) + input_args = ("3",) + true_args = (3,) args = LiterallyOne().parse_args(["--tup", *input_args]) self.assertEqual(args.tup, true_args) with self.assertRaises(SystemExit): - LiterallyOne().parse_args(["--tup", "2"]) + LiterallyOne().parse_args(["--tup", "5"]) def test_tuple_literally_two(self): class LiterallyTwo(Tap): From 7ab915d418dc18114485d0ecf2e79a609b6c91e5 Mon Sep 17 00:00:00 2001 From: Kush Dubey <kushdubey63@gmail.com> Date: Thu, 10 Oct 2024 21:47:21 -0700 Subject: [PATCH 19/21] Fix #150 --- tests/test_to_tap_class.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 7c30da6..2bc10f7 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -24,9 +24,14 @@ _IS_PYDANTIC_V1 = Version(pydantic.__version__) < Version("2.0.0") -# To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 +# To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 -> 3.13 _OPTIONS_TITLE = "options" if not sys.version_info < (3, 10) else "optional arguments" _ARG_LIST_DOTS = "..." if not sys.version_info < (3, 9) else "[ARG_LIST ...]" +_ARG_WITH_ALIAS = ( + "-arg, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME" + if not sys.version_info < (3, 13) + else "-arg ARGUMENT_WITH_REALLY_LONG_NAME, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME" +) @dataclasses.dataclass @@ -416,7 +421,7 @@ def test_subclasser_complex_help_message(class_or_function_: Any): {description} {_OPTIONS_TITLE}: - -arg ARGUMENT_WITH_REALLY_LONG_NAME, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME + {_ARG_WITH_ALIAS} (Union[float, int], default=3) This argument has a long name and will be aliased with a short one --arg_int ARG_INT (int, required) some integer From 00dcff83141eca9153413f8f6714224dee4fdeb7 Mon Sep 17 00:00:00 2001 From: Kush Dubey <kushdubey63@gmail.com> Date: Thu, 10 Oct 2024 21:58:30 -0700 Subject: [PATCH 20/21] Add Python 3.13 to the testing workflow --- .github/workflows/tests.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8f4b3b6..6585066 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@main diff --git a/pyproject.toml b/pyproject.toml index d272ae0..d802aec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Typing :: Typed", From fe0d2d8c0a022786dd1c976ad1ec85c2b3d03c2e Mon Sep 17 00:00:00 2001 From: Kyle Swanson <swansonk.14@gmail.com> Date: Sun, 20 Oct 2024 22:51:26 -0700 Subject: [PATCH 21/21] Removing Python 3.8 due to end-of-life --- .github/workflows/tests.yml | 2 +- LICENSE.txt | 2 +- README.md | 2 +- pyproject.toml | 3 +-- src/tap/utils.py | 2 +- tests/test_actions.py | 3 --- tests/test_integration.py | 3 --- tests/test_tapify.py | 3 --- tests/test_to_tap_class.py | 10 +++------- 9 files changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6585066..8a00450 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@main diff --git a/LICENSE.txt b/LICENSE.txt index cbeb803..5e8cd4d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2022 Jesse Michel and Kyle Swanson +Copyright (c) 2024 Jesse Michel and Kyle Swanson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index a8cfefd..b4d80c4 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Running `python square.py --num 2` will print `The square of your number is 4.0. ## Installation -Tap requires Python 3.8+ +Tap requires Python 3.9+ To install Tap from PyPI run: diff --git a/pyproject.toml b/pyproject.toml index d802aec..0506d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,10 +21,9 @@ dependencies = [ "packaging", "typing-inspect >= 0.7.1", ] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/src/tap/utils.py b/src/tap/utils.py index 2bf788c..8a69467 100644 --- a/src/tap/utils.py +++ b/src/tap/utils.py @@ -558,7 +558,7 @@ def enforce_reproducibility( raise ValueError(f"{no_reproducibility_message}: Uncommitted changes " f"in current args.") -# TODO: remove this once typing_inspect.get_origin is fixed for Python 3.8, 3.9, and 3.10 +# TODO: remove this once typing_inspect.get_origin is fixed for Python 3.9 and 3.10 # https://github.com/ilevkivskyi/typing_inspect/issues/64 # https://github.com/ilevkivskyi/typing_inspect/issues/65 def get_origin(tp: Any) -> Any: diff --git a/tests/test_actions.py b/tests/test_actions.py index 8eb6003..cb60dd8 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -171,7 +171,6 @@ def configure(self): # tried redirecting stderr using unittest.mock.patch # VersionTap().parse_args(['--version']) - @unittest.skipIf(sys.version_info < (3, 8), 'action="extend" introduced in argparse in Python 3.8') def test_actions_extend(self): class ExtendTap(Tap): arg = [1, 2] @@ -185,7 +184,6 @@ def configure(self): args = ExtendTap().parse_args("--arg a b --arg a --arg c d".split()) self.assertEqual(args.arg, [1, 2] + "a b a c d".split()) - @unittest.skipIf(sys.version_info < (3, 8), 'action="extend" introduced in argparse in Python 3.8') def test_actions_extend_list(self): class ExtendListTap(Tap): arg: List = ["hi"] @@ -196,7 +194,6 @@ def configure(self): args = ExtendListTap().parse_args("--arg yo yo --arg yoyo --arg yo yo".split()) self.assertEqual(args.arg, "hi yo yo yoyo yo yo".split()) - @unittest.skipIf(sys.version_info < (3, 8), 'action="extend" introduced in argparse in Python 3.8') def test_actions_extend_list_int(self): class ExtendListIntTap(Tap): arg: List[int] = [0] diff --git a/tests/test_integration.py b/tests/test_integration.py index 97d82b4..e97a44e 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -120,9 +120,6 @@ def test_both_assigned_okay(self): class ParameterizedStandardCollectionTests(TestCase): - @unittest.skipIf( - sys.version_info < (3, 9), "Parameterized standard collections (e.g., list[int]) introduced in Python 3.9" - ) def test_parameterized_standard_collection(self): class ParameterizedStandardCollectionTap(Tap): arg_list_str: list[str] diff --git a/tests/test_tapify.py b/tests/test_tapify.py index 7e2c2da..9a575f2 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -321,9 +321,6 @@ def __eq__(self, other: str) -> bool: self.assertEqual(output, "complex things require 1 0 Person(jesse)") - @unittest.skipIf( - sys.version_info < (3, 9), "Parameterized standard collections (e.g., list[int]) introduced in Python 3.9" - ) def test_tapify_complex_types_parameterized_standard(self): def concat(complexity: list[int], requires: tuple[int, int], intelligence: Person) -> str: return f'{" ".join(map(str, complexity))} {requires[0]} {requires[1]} {intelligence}' diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 2bc10f7..c26ba1a 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -24,9 +24,9 @@ _IS_PYDANTIC_V1 = Version(pydantic.__version__) < Version("2.0.0") -# To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 -> 3.13 +# To properly test the help message, we need to know how argparse formats it. It changed from 3.9 -> 3.10 -> 3.13 _OPTIONS_TITLE = "options" if not sys.version_info < (3, 10) else "optional arguments" -_ARG_LIST_DOTS = "..." if not sys.version_info < (3, 9) else "[ARG_LIST ...]" +_ARG_LIST_DOTS = "..." _ARG_WITH_ALIAS = ( "-arg, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME" if not sys.version_info < (3, 13) @@ -472,8 +472,6 @@ def test_subclasser_complex_help_message(class_or_function_: Any): "--arg_int 1 --baz X --foo b", SystemExit( "error: argument {a,b}: invalid choice: 'X' (choose from 'a', 'b')" - if sys.version_info >= (3, 9) - else "error: invalid choice: 'X' (choose from 'a', 'b')" ), ), ( @@ -493,14 +491,12 @@ def test_subclasser_subparser( _test_subclasser(subclasser_subparser, class_or_function_, args_string_and_arg_to_expected_value, test_call=False) -# @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") @pytest.mark.parametrize( "args_string_and_description_and_expected_message", [ ( "-h", "Script description", - # foo help likely missing b/c class nesting. In a demo in a Python 3.8 env, foo help appears in -h f""" usage: pytest [--foo] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] {{a,b}} ... @@ -513,7 +509,7 @@ def test_subclasser_subparser( b b help {_OPTIONS_TITLE}: - --foo (bool, default=False) {'' if sys.version_info < (3, 9) else 'foo help'} + --foo (bool, default=False) foo help --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) --arg_list [ARG_LIST {_ARG_LIST_DOTS}]