Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix some tokenizer error #3815

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

fix some tokenizer error #3815

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2ee44e
fix some tokenizer error
charles-cooper Feb 26, 2024
4f8c0c8
fix lint
charles-cooper Feb 26, 2024
74b6534
swap two lines
charles-cooper Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions vyper/ast/parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ def parse_to_ast_with_settings(
"""
if "\x00" in source_code:
raise ParserException("No null bytes (\\x00) allowed in the source code.")
settings, class_types, for_loop_annotations, reformatted_code = pre_parse(source_code)
settings, class_types, for_loop_annotations, python_source = pre_parse(source_code)
try:
py_ast = python_ast.parse(reformatted_code)
py_ast = python_ast.parse(python_source)
except SyntaxError as e:
# TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors
raise SyntaxException(str(e), source_code, e.lineno, e.offset) from e
Expand All @@ -72,6 +72,7 @@ def parse_to_ast_with_settings(

annotate_python_ast(
py_ast,
python_source,
source_code,
class_types,
for_loop_annotations,
Expand Down Expand Up @@ -117,7 +118,8 @@ def dict_to_ast(ast_struct: Union[Dict, List]) -> Union[vy_ast.VyperNode, List]:

def annotate_python_ast(
parsed_ast: python_ast.AST,
source_code: str,
python_source: str, # vyper code after pre-parsing
original_source: str, # original vyper code
modification_offsets: ModificationOffsets,
for_loop_annotations: dict,
source_id: int = 0,
Expand All @@ -144,9 +146,11 @@ def annotate_python_ast(
The annotated and optimized AST.
"""

tokens = asttokens.ASTTokens(source_code, tree=cast(Optional[python_ast.Module], parsed_ast))
tokens = asttokens.ASTTokens(
original_source, tree=cast(Optional[python_ast.Module], parsed_ast)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
original_source, tree=cast(Optional[python_ast.Module], parsed_ast)
python_source, tree=cast(Optional[python_ast.Module], parsed_ast)

This seems to work for the failing tests.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm okay it does not work for other tests.

)
visitor = AnnotatingVisitor(
source_code,
python_source,
modification_offsets,
for_loop_annotations,
tokens,
Expand Down
Loading