diff --git a/poetry.lock b/poetry.lock index 1080b489a..7efcd2ac2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -42,7 +42,6 @@ tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.i [[package]] category = "dev" description = "Backport of functools.lru_cache" -marker = "python_version < \"3.2\"" name = "backports.functools-lru-cache" optional = false python-versions = ">=2.6" @@ -55,7 +54,6 @@ testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pyt [[package]] category = "dev" description = "Backport of new features in Python's tempfile module" -marker = "python_version >= \"2.7\" and python_version < \"2.8\"" name = "backports.tempfile" optional = false python-versions = "*" @@ -67,7 +65,6 @@ version = "1.0" [[package]] category = "dev" description = "Backport of new features in Python's weakref module" -marker = "python_version >= \"2.7\" and python_version < \"2.8\"" name = "backports.weakref" optional = false python-versions = "*" @@ -76,7 +73,6 @@ version = "1.0.post1" [[package]] category = "dev" description = "Python package for providing Mozilla's CA Bundle." -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "certifi" optional = false python-versions = "*" @@ -96,7 +92,6 @@ six = "*" [[package]] category = "dev" description = "Universal encoding detector for Python 2 and 3" -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "chardet" optional = false python-versions = "*" @@ -108,15 +103,14 @@ description = "Cleo allows you to create beautiful and testable command-line int name = "cleo" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.7.6" +version = "0.8.1" [package.dependencies] -clikit = ">=0.4.0,<0.5.0" +clikit = ">=0.6.0,<0.7.0" [[package]] category = "dev" description = "Composable command line interface toolkit" -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "click" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -128,37 +122,27 @@ description = "CliKit is a group of utilities to build beautiful and testable co name = "clikit" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.4.3" +version = "0.6.2" [package.dependencies] pastel = ">=0.2.0,<0.3.0" pylev = ">=1.3,<2.0" - -[package.dependencies.enum34] -python = ">=2.7,<2.8" -version = ">=1.1,<2.0" - -[package.dependencies.typing] -python = ">=2.7,<2.8 || >=3.4,<3.5" -version = ">=3.6,<4.0" - -[package.dependencies.typing-extensions] -python = ">=3.5.0,<3.5.4" -version = ">=3.6,<4.0" +crashtest = {version = ">=0.3.0,<0.4.0", markers = "python_version >= \"3.6\" and python_version < \"4.0\""} +enum34 = {version = ">=1.1,<2.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\""} +typing = {version = ">=3.6,<4.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\""} +typing-extensions = {version = ">=3.6,<4.0", markers = "python_version >= \"3.5\" and python_full_version < \"3.5.4\""} [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\" and python_version != \"3.4\" or platform_system == \"Windows\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.4.3" [[package]] -category = "dev" +category = "main" description = "Updated configparser from Python 3.7 for Python 2.6+." -marker = "python_version < \"3\"" name = "configparser" optional = false python-versions = ">=2.6" @@ -169,9 +153,8 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2)", "pytest-flake8", "pytest-black-multipy"] [[package]] -category = "dev" +category = "main" description = "Backports and enhancements for the contextlib module" -marker = "python_version < \"3.4\"" name = "contextlib2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -188,6 +171,14 @@ version = "5.2" [package.extras] toml = ["toml"] +[[package]] +category = "dev" +description = "Manage Python errors with ease" +name = "crashtest" +optional = false +python-versions = ">=3.6,<4.0" +version = "0.3.0" + [[package]] category = "dev" description = "Distribution utilities" @@ -199,7 +190,6 @@ version = "0.3.1" [[package]] category = "main" description = "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4" -marker = "python_version >= \"2.7\" and python_version < \"2.8\"" name = "enum34" optional = false python-versions = "*" @@ -216,7 +206,6 @@ version = "3.0.12" [[package]] category = "dev" description = "Python function signatures from PEP362 for Python 2.6, 2.7 and 3.2+" -marker = "python_version < \"3.0\"" name = "funcsigs" optional = false python-versions = "*" @@ -225,7 +214,6 @@ version = "1.0.2" [[package]] category = "main" description = "Backport of the functools module from Python 3.2.3 for use on 2.7 and PyPy." -marker = "python_version >= \"2.7\" and python_version < \"2.8\"" name = "functools32" optional = false python-versions = "*" @@ -233,12 +221,11 @@ version = "3.2.3-2" [[package]] category = "dev" -description = "Backport of the concurrent.futures package from Python 3" -marker = "python_version < \"3.2\"" +description = "Backport of the concurrent.futures package from Python 3.2" name = "futures" optional = false -python-versions = ">=2.6, <3" -version = "3.3.0" +python-versions = "*" +version = "3.1.1" [[package]] category = "dev" @@ -254,16 +241,14 @@ license = ["editdistance"] [[package]] category = "dev" description = "Internationalized Domain Names in Applications (IDNA)" -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "idna" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.10" [[package]] -category = "dev" +category = "main" description = "Read metadata from Python packages" -marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" @@ -271,18 +256,9 @@ version = "1.7.0" [package.dependencies] zipp = ">=0.5" - -[package.dependencies.configparser] -python = "<3" -version = ">=3.5" - -[package.dependencies.contextlib2] -python = "<3" -version = "*" - -[package.dependencies.pathlib2] -python = "<3" -version = "*" +configparser = {version = ">=3.5", markers = "python_version < \"3\""} +contextlib2 = {version = "*", markers = "python_version < \"3\""} +pathlib2 = {version = "*", markers = "python_version < \"3\""} [package.extras] docs = ["sphinx", "rst.linker"] @@ -291,32 +267,17 @@ testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] [[package]] category = "dev" description = "Read resources from Python packages" -marker = "python_version < \"3.7\"" name = "importlib-resources" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" version = "3.0.0" [package.dependencies] -[package.dependencies.contextlib2] -python = "<3" -version = "*" - -[package.dependencies.pathlib2] -python = "<3" -version = "*" - -[package.dependencies.singledispatch] -python = "<3.4" -version = "*" - -[package.dependencies.typing] -python = "<3.5" -version = "*" - -[package.dependencies.zipp] -python = "<3.8" -version = ">=0.4" +contextlib2 = {version = "*", markers = "python_version < \"3\""} +pathlib2 = {version = "*", markers = "python_version < \"3\""} +singledispatch = {version = "*", markers = "python_version < \"3.4\""} +typing = {version = "*", markers = "python_version < \"3.5\""} +zipp = {version = ">=0.4", markers = "python_version < \"3.8\""} [package.extras] docs = ["sphinx", "rst.linker", "jaraco.packaging"] @@ -330,13 +291,8 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "4.3.21" [package.dependencies] -[package.dependencies."backports.functools-lru-cache"] -python = "<3.2" -version = "*" - -[package.dependencies.futures] -python = "<3.2" -version = "*" +"backports.functools-lru-cache" = {version = "*", markers = "python_version < \"3.2\""} +futures = {version = "*", markers = "python_version < \"3.2\""} [package.extras] pipfile = ["pipreqs", "requirementslib"] @@ -347,7 +303,6 @@ xdg_home = ["appdirs (>=1.4.0)"] [[package]] category = "dev" description = "An implementation of JSON Schema validation for Python" -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "jsonschema" optional = false python-versions = "*" @@ -356,7 +311,6 @@ version = "3.2.0" [package.dependencies] attrs = ">=17.4.0" pyrsistent = ">=0.14.0" -setuptools = "*" six = ">=1.11.0" [package.extras] @@ -366,7 +320,6 @@ format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator [[package]] category = "dev" description = "Rolling backport of unittest.mock for all Pythons" -marker = "python_version < \"3.0\"" name = "mock" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -374,10 +327,7 @@ version = "3.0.5" [package.dependencies] six = "*" - -[package.dependencies.funcsigs] -python = "<3.3" -version = ">=1" +funcsigs = {version = ">=1", markers = "python_version < \"3.3\""} [package.extras] build = ["twine", "wheel", "blurb"] @@ -387,7 +337,6 @@ test = ["pytest", "pytest-cov"] [[package]] category = "dev" description = "More routines for operating on iterables, beyond itertools" -marker = "python_version <= \"2.7\"" name = "more-itertools" optional = false python-versions = "*" @@ -396,15 +345,6 @@ version = "5.0.0" [package.dependencies] six = ">=1.0.0,<2.0.0" -[[package]] -category = "dev" -description = "More routines for operating on iterables, beyond itertools" -marker = "python_version > \"2.7\"" -name = "more-itertools" -optional = false -python-versions = ">=3.5" -version = "8.4.0" - [[package]] category = "dev" description = "Node.js virtual environment builder" @@ -436,7 +376,6 @@ version = "0.2.0" [[package]] category = "main" description = "Object-oriented filesystem paths" -marker = "python_version < \"3.4\" and sys_platform != \"win32\" or python_version < \"3.6\" or python_version >= \"2.7\" and python_version < \"2.8\"" name = "pathlib2" optional = false python-versions = "*" @@ -444,10 +383,7 @@ version = "2.3.5" [package.dependencies] six = "*" - -[package.dependencies.scandir] -python = "<3.5" -version = "*" +scandir = {version = "*", markers = "python_version < \"3.5\""} [[package]] category = "dev" @@ -459,14 +395,8 @@ version = "0.8.2" [package.dependencies] toml = "*" - -[package.dependencies.importlib_metadata] -python = "<3.8" -version = "*" - -[package.dependencies.zipp] -python = "<3.8" -version = "*" +importlib_metadata = {version = "*", markers = "python_version < \"3.8\""} +zipp = {version = "*", markers = "python_version < \"3.8\""} [[package]] category = "dev" @@ -477,9 +407,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.13.1" [package.dependencies] -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] @@ -501,18 +429,9 @@ pyyaml = "*" six = "*" toml = "*" virtualenv = ">=15.2" - -[package.dependencies.futures] -python = "<3.2" -version = "*" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = "*" - -[package.dependencies.importlib-resources] -python = "<3.7" -version = "*" +futures = {version = "*", markers = "python_version < \"3.2\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +importlib-resources = {version = "*", markers = "python_version < \"3.7\""} [[package]] category = "dev" @@ -541,7 +460,6 @@ version = "2.4.7" [[package]] category = "dev" description = "Persistent/Functional/Immutable data structures" -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "pyrsistent" optional = false python-versions = "*" @@ -566,31 +484,19 @@ pluggy = ">=0.12,<1.0" py = ">=1.5.0" six = ">=1.10.0" wcwidth = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\" and python_version != \"3.4\""} +funcsigs = {version = ">=1.0", markers = "python_version < \"3.0\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +pathlib2 = {version = ">=2.2.0", markers = "python_version < \"3.6\""} [[package.dependencies.more-itertools]] -python = "<2.8" +markers = "python_version <= \"2.7\"" version = ">=4.0.0,<6.0.0" [[package.dependencies.more-itertools]] -python = ">=2.8" +markers = "python_version > \"2.7\"" version = ">=4.0.0" -[package.dependencies.colorama] -python = "<3.4.0 || >=3.5.0" -version = "*" - -[package.dependencies.funcsigs] -python = "<3.0" -version = ">=1.0" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12" - -[package.dependencies.pathlib2] -python = "<3.6" -version = ">=2.2.0" - [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "nose", "requests", "mock"] @@ -619,10 +525,7 @@ version = "2.0.0" [package.dependencies] pytest = ">=2.7" - -[package.dependencies.mock] -python = "<3.0" -version = "*" +mock = {version = "*", markers = "python_version < \"3.0\""} [package.extras] dev = ["pre-commit", "tox"] @@ -638,7 +541,6 @@ version = "5.3.1" [[package]] category = "dev" description = "Python HTTP for Humans." -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -657,7 +559,6 @@ socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [[package]] category = "main" description = "scandir, a better directory iterator and faster os.walk()" -marker = "python_version < \"2.8\" and sys_platform != \"win32\" and python_version >= \"2.7\" or python_version >= \"2.7\" and python_version < \"2.8\"" name = "scandir" optional = false python-versions = "*" @@ -666,7 +567,6 @@ version = "1.10.0" [[package]] category = "dev" description = "This library brings functools.singledispatch from Python 3.4 to Python 2.6-3.3." -marker = "python_version < \"3.4\"" name = "singledispatch" optional = false python-versions = "*" @@ -700,17 +600,9 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.5.11" [package.dependencies] -[package.dependencies.enum34] -python = ">=2.7,<2.8" -version = ">=1.1,<2.0" - -[package.dependencies.functools32] -python = ">=2.7,<2.8" -version = ">=3.2.3,<4.0.0" - -[package.dependencies.typing] -python = ">=2.7,<2.8 || >=3.4,<3.5" -version = ">=3.6,<4.0" +enum34 = {version = ">=1.1,<2.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\""} +functools32 = {version = ">=3.2.3,<4.0.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\""} +typing = {version = ">=3.6,<4.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\""} [[package]] category = "dev" @@ -721,7 +613,6 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" version = "3.17.1" [package.dependencies] -colorama = ">=0.4.1" filelock = ">=3.0.0" packaging = ">=14" pluggy = ">=0.12.0" @@ -729,10 +620,8 @@ py = ">=1.4.17" six = ">=1.14.0" toml = ">=0.9.4" virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2,<20.0.3 || >20.0.3,<20.0.4 || >20.0.4,<20.0.5 || >20.0.5,<20.0.6 || >20.0.6,<20.0.7 || >20.0.7" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12,<2" +colorama = {version = ">=0.4.1", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = ">=0.12,<2", markers = "python_version < \"3.8\""} [package.extras] docs = ["sphinx (>=2.0.0)", "towncrier (>=18.5.0)", "pygments-github-lexers (>=0.0.5)", "sphinxcontrib-autoprogram (>=0.1.5)"] @@ -741,7 +630,6 @@ testing = ["freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "pytest (>=4.0.0)", "py [[package]] category = "main" description = "Type Hints for Python" -marker = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version < \"3.5\" or python_version >= \"3.4\" and python_version < \"3.5\"" name = "typing" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -750,7 +638,6 @@ version = "3.7.4.3" [[package]] category = "dev" description = "Backported and Experimental Type Hints for Python 3.5+" -marker = "python_version >= \"3.5.0\" and python_version < \"3.5.4\"" name = "typing-extensions" optional = false python-versions = "*" @@ -759,7 +646,6 @@ version = "3.7.4.2" [[package]] category = "dev" description = "HTTP library with thread-safe connection pooling, file post, and more." -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" @@ -773,7 +659,6 @@ socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] category = "dev" description = "A command line tool, to simplify vendoring pure Python dependencies." -marker = "python_version >= \"3.8\" and python_version < \"3.9\"" name = "vendoring" optional = false python-versions = "~= 3.8.0" @@ -798,18 +683,9 @@ appdirs = ">=1.4.3,<2" distlib = ">=0.3.1,<1" filelock = ">=3.0.0,<4" six = ">=1.9.0,<2" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12,<2" - -[package.dependencies.importlib-resources] -python = "<3.7" -version = ">=1.0" - -[package.dependencies.pathlib2] -python = "<3.4" -version = ">=2.3.3,<3" +importlib-metadata = {version = ">=0.12,<2", markers = "python_version < \"3.8\""} +importlib-resources = {version = ">=1.0", markers = "python_version < \"3.7\""} +pathlib2 = {version = ">=2.3.3,<3", markers = "python_version < \"3.4\" and sys_platform != \"win32\""} [package.extras] docs = ["sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)", "proselint (>=0.10.2)"] @@ -824,30 +700,26 @@ python-versions = "*" version = "0.2.5" [package.dependencies] -[package.dependencies."backports.functools-lru-cache"] -python = "<3.2" -version = ">=1.2.1" +"backports.functools-lru-cache" = {version = ">=1.2.1", markers = "python_version < \"3.2\""} [[package]] -category = "dev" +category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=2.7" version = "1.2.0" [package.dependencies] -[package.dependencies.contextlib2] -python = "<3.4" -version = "*" +contextlib2 = {version = "*", markers = "python_version < \"3.4\""} [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] [metadata] -content-hash = "84d8293078aa84a1b4262a97eb085109da938103b72705f9cae8fa3d81f99cd9" +content-hash = "101cc0df9d423447400dc91a0c7bc038dac1158c609a6da2c7db54b4fff75423" +lock-version = "1.0" python-versions = "~2.7 || ^3.5" [metadata.files] @@ -892,16 +764,16 @@ chardet = [ {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] cleo = [ - {file = "cleo-0.7.6-py2.py3-none-any.whl", hash = "sha256:9443d67e5b2da79b32d820ae41758dd6a25618345cb10b9a022a695e26b291b9"}, - {file = "cleo-0.7.6.tar.gz", hash = "sha256:99cf342406f3499cec43270fcfaf93c126c5164092eca201dfef0f623360b409"}, + {file = "cleo-0.8.1-py2.py3-none-any.whl", hash = "sha256:141cda6dc94a92343be626bb87a0b6c86ae291dfc732a57bf04310d4b4201753"}, + {file = "cleo-0.8.1.tar.gz", hash = "sha256:3d0e22d30117851b45970b6c14aca4ab0b18b1b53c8af57bed13208147e4069f"}, ] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] clikit = [ - {file = "clikit-0.4.3-py2.py3-none-any.whl", hash = "sha256:71e321b7795a2a6c4888629f43365d52db071737e668ab16861121d7dd3ada09"}, - {file = "clikit-0.4.3.tar.gz", hash = "sha256:6e2d7e115e7c7b35bceb0209109935ab2f9ab50910e9ff2293f7fa0b7abf973e"}, + {file = "clikit-0.6.2-py2.py3-none-any.whl", hash = "sha256:71268e074e68082306e23d7369a7b99f824a0ef926e55ba2665e911f7208489e"}, + {file = "clikit-0.6.2.tar.gz", hash = "sha256:442ee5db9a14120635c5990bcdbfe7c03ada5898291f0c802f77be71569ded59"}, ] colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, @@ -951,6 +823,10 @@ coverage = [ {file = "coverage-5.2-cp39-cp39-win_amd64.whl", hash = "sha256:10f2a618a6e75adf64329f828a6a5b40244c1c50f5ef4ce4109e904e69c71bd2"}, {file = "coverage-5.2.tar.gz", hash = "sha256:1874bdc943654ba46d28f179c1846f5710eda3aeb265ff029e0ac2b52daae404"}, ] +crashtest = [ + {file = "crashtest-0.3.0-py3-none-any.whl", hash = "sha256:06069a9267c54be31c42b03574b72407bf780e13c82cb0238f24ea69cf25b6dd"}, + {file = "crashtest-0.3.0.tar.gz", hash = "sha256:e9c06cc96400939ab5327123a3f699078eaad8a6283247d7b2ae0f6afffadf14"}, +] distlib = [ {file = "distlib-0.3.1-py2.py3-none-any.whl", hash = "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb"}, {file = "distlib-0.3.1.zip", hash = "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1"}, @@ -973,8 +849,9 @@ functools32 = [ {file = "functools32-3.2.3-2.zip", hash = "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0"}, ] futures = [ - {file = "futures-3.3.0-py2-none-any.whl", hash = "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16"}, - {file = "futures-3.3.0.tar.gz", hash = "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794"}, + {file = "futures-3.1.1-py2-none-any.whl", hash = "sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f"}, + {file = "futures-3.1.1-py3-none-any.whl", hash = "sha256:3a44f286998ae64f0cc083682fcfec16c406134a81a589a5de445d7bb7c2751b"}, + {file = "futures-3.1.1.tar.gz", hash = "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd"}, ] identify = [ {file = "identify-1.4.24-py2.py3-none-any.whl", hash = "sha256:5519601b70c831011fb425ffd214101df7639ba3980f24dc283f7675b19127b3"}, @@ -1008,8 +885,6 @@ more-itertools = [ {file = "more-itertools-5.0.0.tar.gz", hash = "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4"}, {file = "more_itertools-5.0.0-py2-none-any.whl", hash = "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc"}, {file = "more_itertools-5.0.0-py3-none-any.whl", hash = "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"}, - {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, - {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, ] nodeenv = [ {file = "nodeenv-1.4.0-py2.py3-none-any.whl", hash = "sha256:4b0b77afa3ba9b54f4b6396e60b0c83f59eaeb2d63dc3cc7a70f7f4af96c82bc"}, diff --git a/poetry/core/_vendor/lark-parser.LICENSE b/poetry/core/_vendor/lark-parser.LICENSE new file mode 100644 index 000000000..efcb9665f --- /dev/null +++ b/poetry/core/_vendor/lark-parser.LICENSE @@ -0,0 +1,19 @@ +Copyright © 2017 Erez Shinan + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/poetry/core/_vendor/lark/__init__.py b/poetry/core/_vendor/lark/__init__.py new file mode 100644 index 000000000..8ddab96a8 --- /dev/null +++ b/poetry/core/_vendor/lark/__init__.py @@ -0,0 +1,9 @@ +from .tree import Tree +from .visitors import Transformer, Visitor, v_args, Discard +from .visitors import InlineTransformer, inline_args # XXX Deprecated +from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken, + UnexpectedInput, UnexpectedCharacters, LarkError) +from .lexer import Token +from .lark import Lark + +__version__ = "0.9.0" diff --git a/poetry/core/_vendor/lark/__pyinstaller/__init__.py b/poetry/core/_vendor/lark/__pyinstaller/__init__.py new file mode 100644 index 000000000..fa02fc923 --- /dev/null +++ b/poetry/core/_vendor/lark/__pyinstaller/__init__.py @@ -0,0 +1,6 @@ +# For usage of lark with PyInstaller. See https://pyinstaller-sample-hook.readthedocs.io/en/latest/index.html + +import os + +def get_hook_dirs(): + return [os.path.dirname(__file__)] \ No newline at end of file diff --git a/poetry/core/_vendor/lark/__pyinstaller/hook-lark.py b/poetry/core/_vendor/lark/__pyinstaller/hook-lark.py new file mode 100644 index 000000000..cf3d8e3d1 --- /dev/null +++ b/poetry/core/_vendor/lark/__pyinstaller/hook-lark.py @@ -0,0 +1,14 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2017-2020, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License (version 2 +# or later) with exception for distributing the bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +# +# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) +#----------------------------------------------------------------------------- + +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('lark') diff --git a/poetry/core/_vendor/lark/common.py b/poetry/core/_vendor/lark/common.py new file mode 100644 index 000000000..c44f9cef3 --- /dev/null +++ b/poetry/core/_vendor/lark/common.py @@ -0,0 +1,29 @@ +from .utils import Serialize +from .lexer import TerminalDef + +###{standalone + +class LexerConf(Serialize): + __serialize_fields__ = 'tokens', 'ignore', 'g_regex_flags' + __serialize_namespace__ = TerminalDef, + + def __init__(self, tokens, ignore=(), postlex=None, callbacks=None, g_regex_flags=0): + self.tokens = tokens + self.ignore = ignore + self.postlex = postlex + self.callbacks = callbacks or {} + self.g_regex_flags = g_regex_flags + + def _deserialize(self): + self.callbacks = {} # TODO + +###} + +class ParserConf: + def __init__(self, rules, callbacks, start): + assert isinstance(start, list) + self.rules = rules + self.callbacks = callbacks + self.start = start + + diff --git a/poetry/core/_vendor/lark/exceptions.py b/poetry/core/_vendor/lark/exceptions.py new file mode 100644 index 000000000..1c5e533e4 --- /dev/null +++ b/poetry/core/_vendor/lark/exceptions.py @@ -0,0 +1,119 @@ +from .utils import STRING_TYPE + +###{standalone +class LarkError(Exception): + pass + +class GrammarError(LarkError): + pass + +class ParseError(LarkError): + pass + +class LexError(LarkError): + pass + +class UnexpectedEOF(ParseError): + def __init__(self, expected): + self.expected = expected + + message = ("Unexpected end-of-input. Expected one of: \n\t* %s\n" % '\n\t* '.join(x.name for x in self.expected)) + super(UnexpectedEOF, self).__init__(message) + + +class UnexpectedInput(LarkError): + pos_in_stream = None + + def get_context(self, text, span=40): + pos = self.pos_in_stream + start = max(pos - span, 0) + end = pos + span + before = text[start:pos].rsplit('\n', 1)[-1] + after = text[pos:end].split('\n', 1)[0] + return before + after + '\n' + ' ' * len(before) + '^\n' + + def match_examples(self, parse_fn, examples, token_type_match_fallback=False): + """ Given a parser instance and a dictionary mapping some label with + some malformed syntax examples, it'll return the label for the + example that bests matches the current error. + """ + assert self.state is not None, "Not supported for this exception" + + candidate = (None, False) + for label, example in examples.items(): + assert not isinstance(example, STRING_TYPE) + + for malformed in example: + try: + parse_fn(malformed) + except UnexpectedInput as ut: + if ut.state == self.state: + try: + if ut.token == self.token: # Try exact match first + return label + + if token_type_match_fallback: + # Fallback to token types match + if (ut.token.type == self.token.type) and not candidate[-1]: + candidate = label, True + + except AttributeError: + pass + if not candidate[0]: + candidate = label, False + + return candidate[0] + + +class UnexpectedCharacters(LexError, UnexpectedInput): + def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None, token_history=None): + message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column) + + self.line = line + self.column = column + self.allowed = allowed + self.considered_tokens = considered_tokens + self.pos_in_stream = lex_pos + self.state = state + + message += '\n\n' + self.get_context(seq) + if allowed: + message += '\nExpecting: %s\n' % allowed + if token_history: + message += '\nPrevious tokens: %s\n' % ', '.join(repr(t) for t in token_history) + + super(UnexpectedCharacters, self).__init__(message) + + + +class UnexpectedToken(ParseError, UnexpectedInput): + def __init__(self, token, expected, considered_rules=None, state=None, puppet=None): + self.token = token + self.expected = expected # XXX str shouldn't necessary + self.line = getattr(token, 'line', '?') + self.column = getattr(token, 'column', '?') + self.considered_rules = considered_rules + self.state = state + self.pos_in_stream = getattr(token, 'pos_in_stream', None) + self.puppet = puppet + + message = ("Unexpected token %r at line %s, column %s.\n" + "Expected one of: \n\t* %s\n" + % (token, self.line, self.column, '\n\t* '.join(self.expected))) + + super(UnexpectedToken, self).__init__(message) + +class VisitError(LarkError): + """VisitError is raised when visitors are interrupted by an exception + + It provides the following attributes for inspection: + - obj: the tree node or token it was processing when the exception was raised + - orig_exc: the exception that cause it to fail + """ + def __init__(self, rule, obj, orig_exc): + self.obj = obj + self.orig_exc = orig_exc + + message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc) + super(VisitError, self).__init__(message) +###} diff --git a/poetry/core/_vendor/lark/grammar.py b/poetry/core/_vendor/lark/grammar.py new file mode 100644 index 000000000..bb8435138 --- /dev/null +++ b/poetry/core/_vendor/lark/grammar.py @@ -0,0 +1,108 @@ +from .utils import Serialize + +###{standalone + +class Symbol(Serialize): + __slots__ = ('name',) + + is_term = NotImplemented + + def __init__(self, name): + self.name = name + + def __eq__(self, other): + assert isinstance(other, Symbol), other + return self.is_term == other.is_term and self.name == other.name + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.name) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.name) + + fullrepr = property(__repr__) + + +class Terminal(Symbol): + __serialize_fields__ = 'name', 'filter_out' + + is_term = True + + def __init__(self, name, filter_out=False): + self.name = name + self.filter_out = filter_out + + @property + def fullrepr(self): + return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out) + + + +class NonTerminal(Symbol): + __serialize_fields__ = 'name', + + is_term = False + + + +class RuleOptions(Serialize): + __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'template_source', 'empty_indices' + + def __init__(self, keep_all_tokens=False, expand1=False, priority=None, template_source=None, empty_indices=()): + self.keep_all_tokens = keep_all_tokens + self.expand1 = expand1 + self.priority = priority + self.template_source = template_source + self.empty_indices = empty_indices + + def __repr__(self): + return 'RuleOptions(%r, %r, %r, %r)' % ( + self.keep_all_tokens, + self.expand1, + self.priority, + self.template_source + ) + + +class Rule(Serialize): + """ + origin : a symbol + expansion : a list of symbols + order : index of this expansion amongst all rules of the same name + """ + __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash') + + __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options' + __serialize_namespace__ = Terminal, NonTerminal, RuleOptions + + def __init__(self, origin, expansion, order=0, alias=None, options=None): + self.origin = origin + self.expansion = expansion + self.alias = alias + self.order = order + self.options = options or RuleOptions() + self._hash = hash((self.origin, tuple(self.expansion))) + + def _deserialize(self): + self._hash = hash((self.origin, tuple(self.expansion))) + + def __str__(self): + return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion)) + + def __repr__(self): + return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options) + + def __hash__(self): + return self._hash + + def __eq__(self, other): + if not isinstance(other, Rule): + return False + return self.origin == other.origin and self.expansion == other.expansion + + + +###} diff --git a/poetry/core/_vendor/lark/grammars/common.lark b/poetry/core/_vendor/lark/grammars/common.lark new file mode 100644 index 000000000..a675ca410 --- /dev/null +++ b/poetry/core/_vendor/lark/grammars/common.lark @@ -0,0 +1,50 @@ +// +// Numbers +// + +DIGIT: "0".."9" +HEXDIGIT: "a".."f"|"A".."F"|DIGIT + +INT: DIGIT+ +SIGNED_INT: ["+"|"-"] INT +DECIMAL: INT "." INT? | "." INT + +// float = /-?\d+(\.\d+)?([eE][+-]?\d+)?/ +_EXP: ("e"|"E") SIGNED_INT +FLOAT: INT _EXP | DECIMAL _EXP? +SIGNED_FLOAT: ["+"|"-"] FLOAT + +NUMBER: FLOAT | INT +SIGNED_NUMBER: ["+"|"-"] NUMBER + +// +// Strings +// +_STRING_INNER: /.*?/ +_STRING_ESC_INNER: _STRING_INNER /(? 0 + + def handle_NL(self, token): + if self.paren_level > 0: + return + + yield token + + indent_str = token.rsplit('\n', 1)[1] # Tabs and spaces + indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len + + if indent > self.indent_level[-1]: + self.indent_level.append(indent) + yield Token.new_borrow_pos(self.INDENT_type, indent_str, token) + else: + while indent < self.indent_level[-1]: + self.indent_level.pop() + yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token) + + assert indent == self.indent_level[-1], '%s != %s' % (indent, self.indent_level[-1]) + + def _process(self, stream): + for token in stream: + if token.type == self.NL_type: + for t in self.handle_NL(token): + yield t + else: + yield token + + if token.type in self.OPEN_PAREN_types: + self.paren_level += 1 + elif token.type in self.CLOSE_PAREN_types: + self.paren_level -= 1 + assert self.paren_level >= 0 + + while len(self.indent_level) > 1: + self.indent_level.pop() + yield Token(self.DEDENT_type, '') + + assert self.indent_level == [0], self.indent_level + + def process(self, stream): + self.paren_level = 0 + self.indent_level = [0] + return self._process(stream) + + # XXX Hack for ContextualLexer. Maybe there's a more elegant solution? + @property + def always_accept(self): + return (self.NL_type,) + +###} diff --git a/poetry/core/_vendor/lark/lark.py b/poetry/core/_vendor/lark/lark.py new file mode 100644 index 000000000..2b783cb20 --- /dev/null +++ b/poetry/core/_vendor/lark/lark.py @@ -0,0 +1,405 @@ +from __future__ import absolute_import + +import sys, os, pickle, hashlib, logging +from io import open + + +from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS +from .load_grammar import load_grammar +from .tree import Tree +from .common import LexerConf, ParserConf + +from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken +from .parse_tree_builder import ParseTreeBuilder +from .parser_frontends import get_frontend +from .grammar import Rule + +import re +try: + import regex +except ImportError: + regex = None + +###{standalone + +class LarkOptions(Serialize): + """Specifies the options for Lark + + """ + OPTIONS_DOC = """ +# General + + start - The start symbol. Either a string, or a list of strings for + multiple possible starts (Default: "start") + debug - Display debug information, such as warnings (default: False) + transformer - Applies the transformer to every parse tree (equivlent to + applying it after the parse, but faster) + propagate_positions - Propagates (line, column, end_line, end_column) + attributes into all tree branches. + maybe_placeholders - When True, the `[]` operator returns `None` when not matched. + When `False`, `[]` behaves like the `?` operator, + and returns no value at all. + (default=`False`. Recommended to set to `True`) + regex - When True, uses the `regex` module instead of the stdlib `re`. + cache - Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. + LALR only for now. + When `False`, does nothing (default) + When `True`, caches to a temporary file in the local directory + When given a string, caches to the path pointed by the string + + g_regex_flags - Flags that are applied to all terminals + (both regex and strings) + keep_all_tokens - Prevent the tree builder from automagically + removing "punctuation" tokens (default: False) + +# Algorithm + + parser - Decides which parser engine to use + Accepts "earley" or "lalr". (Default: "earley") + (there is also a "cyk" option for legacy) + + lexer - Decides whether or not to use a lexer stage + "auto" (default): Choose for me based on the parser + "standard": Use a standard lexer + "contextual": Stronger lexer (only works with parser="lalr") + "dynamic": Flexible and powerful (only with parser="earley") + "dynamic_complete": Same as dynamic, but tries *every* variation + of tokenizing possible. + + ambiguity - Decides how to handle ambiguity in the parse. + Only relevant if parser="earley" + "resolve": The parser will automatically choose the simplest + derivation (it chooses consistently: greedy for + tokens, non-greedy for rules) + "explicit": The parser will return all derivations wrapped + in "_ambig" tree nodes (i.e. a forest). + +# Domain Specific + + postlex - Lexer post-processing (Default: None) Only works with the + standard and contextual lexers. + priority - How priorities should be evaluated - auto, none, normal, + invert (Default: auto) + lexer_callbacks - Dictionary of callbacks for the lexer. May alter + tokens during lexing. Use with caution. + edit_terminals - A callback + """ + if __doc__: + __doc__ += OPTIONS_DOC + + _defaults = { + 'debug': False, + 'keep_all_tokens': False, + 'tree_class': None, + 'cache': False, + 'postlex': None, + 'parser': 'earley', + 'lexer': 'auto', + 'transformer': None, + 'start': 'start', + 'priority': 'auto', + 'ambiguity': 'auto', + 'regex': False, + 'propagate_positions': False, + 'lexer_callbacks': {}, + 'maybe_placeholders': False, + 'edit_terminals': None, + 'g_regex_flags': 0, + } + + def __init__(self, options_dict): + o = dict(options_dict) + + options = {} + for name, default in self._defaults.items(): + if name in o: + value = o.pop(name) + if isinstance(default, bool) and name != 'cache': + value = bool(value) + else: + value = default + + options[name] = value + + if isinstance(options['start'], STRING_TYPE): + options['start'] = [options['start']] + + self.__dict__['options'] = options + + assert self.parser in ('earley', 'lalr', 'cyk', None) + + if self.parser == 'earley' and self.transformer: + raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.' + 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)') + + if o: + raise ValueError("Unknown options: %s" % o.keys()) + + def __getattr__(self, name): + try: + return self.options[name] + except KeyError as e: + raise AttributeError(e) + + def __setattr__(self, name, value): + assert name in self.options + self.options[name] = value + + def serialize(self, memo): + return self.options + + @classmethod + def deserialize(cls, data, memo): + return cls(data) + + +class Lark(Serialize): + def __init__(self, grammar, **options): + """ + grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax) + options : a dictionary controlling various aspects of Lark. + """ + + self.options = LarkOptions(options) + + # Set regex or re module + use_regex = self.options.regex + if use_regex: + if regex: + self.re = regex + else: + raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.') + else: + self.re = re + + # Some, but not all file-like objects have a 'name' attribute + try: + self.source = grammar.name + except AttributeError: + self.source = '' + + # Drain file-like objects to get their contents + try: + read = grammar.read + except AttributeError: + pass + else: + grammar = read() + + assert isinstance(grammar, STRING_TYPE) + + cache_fn = None + if self.options.cache: + if self.options.parser != 'lalr': + raise NotImplementedError("cache only works with parser='lalr' for now") + if isinstance(self.options.cache, STRING_TYPE): + cache_fn = self.options.cache + else: + if self.options.cache is not True: + raise ValueError("cache must be bool or str") + unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals') + from . import __version__ + options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable) + s = grammar + options_str + __version__ + md5 = hashlib.md5(s.encode()).hexdigest() + cache_fn = '.lark_cache_%s.tmp' % md5 + + if FS.exists(cache_fn): + logging.debug('Loading grammar from cache: %s', cache_fn) + with FS.open(cache_fn, 'rb') as f: + self._load(f, self.options.transformer, self.options.postlex) + return + + if self.options.lexer == 'auto': + if self.options.parser == 'lalr': + self.options.lexer = 'contextual' + elif self.options.parser == 'earley': + self.options.lexer = 'dynamic' + elif self.options.parser == 'cyk': + self.options.lexer = 'standard' + else: + assert False, self.options.parser + lexer = self.options.lexer + assert lexer in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer) + + if self.options.ambiguity == 'auto': + if self.options.parser == 'earley': + self.options.ambiguity = 'resolve' + else: + disambig_parsers = ['earley', 'cyk'] + assert self.options.parser in disambig_parsers, ( + 'Only %s supports disambiguation right now') % ', '.join(disambig_parsers) + + if self.options.priority == 'auto': + if self.options.parser in ('earley', 'cyk', ): + self.options.priority = 'normal' + elif self.options.parser in ('lalr', ): + self.options.priority = None + elif self.options.priority in ('invert', 'normal'): + assert self.options.parser in ('earley', 'cyk'), "priorities are not supported for LALR at this time" + + assert self.options.priority in ('auto', None, 'normal', 'invert'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority) + assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"' + assert self.options.ambiguity in ('resolve', 'explicit', 'auto', ) + + # Parse the grammar file and compose the grammars (TODO) + self.grammar = load_grammar(grammar, self.source, self.re) + + # Compile the EBNF grammar into BNF + self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start) + + if self.options.edit_terminals: + for t in self.terminals: + self.options.edit_terminals(t) + + self._terminals_dict = {t.name:t for t in self.terminals} + + # If the user asked to invert the priorities, negate them all here. + # This replaces the old 'resolve__antiscore_sum' option. + if self.options.priority == 'invert': + for rule in self.rules: + if rule.options.priority is not None: + rule.options.priority = -rule.options.priority + # Else, if the user asked to disable priorities, strip them from the + # rules. This allows the Earley parsers to skip an extra forest walk + # for improved performance, if you don't need them (or didn't specify any). + elif self.options.priority == None: + for rule in self.rules: + if rule.options.priority is not None: + rule.options.priority = None + + # TODO Deprecate lexer_callbacks? + lexer_callbacks = dict(self.options.lexer_callbacks) + if self.options.transformer: + t = self.options.transformer + for term in self.terminals: + if hasattr(t, term.name): + lexer_callbacks[term.name] = getattr(t, term.name) + + self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.g_regex_flags) + + if self.options.parser: + self.parser = self._build_parser() + elif lexer: + self.lexer = self._build_lexer() + + if cache_fn: + logging.debug('Saving grammar to cache: %s', cache_fn) + with FS.open(cache_fn, 'wb') as f: + self.save(f) + + if __init__.__doc__: + __init__.__doc__ += "\nOptions:\n" + LarkOptions.OPTIONS_DOC + + __serialize_fields__ = 'parser', 'rules', 'options' + + def _build_lexer(self): + return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, g_regex_flags=self.lexer_conf.g_regex_flags) + + def _prepare_callbacks(self): + self.parser_class = get_frontend(self.options.parser, self.options.lexer) + self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class or Tree, self.options.propagate_positions, self.options.keep_all_tokens, self.options.parser!='lalr' and self.options.ambiguity=='explicit', self.options.maybe_placeholders) + self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer) + + def _build_parser(self): + self._prepare_callbacks() + parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) + return self.parser_class(self.lexer_conf, parser_conf, self.re, options=self.options) + + def save(self, f): + data, m = self.memo_serialize([TerminalDef, Rule]) + pickle.dump({'data': data, 'memo': m}, f) + + @classmethod + def load(cls, f): + inst = cls.__new__(cls) + return inst._load(f) + + def _load(self, f, transformer=None, postlex=None): + if isinstance(f, dict): + d = f + else: + d = pickle.load(f) + memo = d['memo'] + data = d['data'] + + assert memo + memo = SerializeMemoizer.deserialize(memo, {'Rule': Rule, 'TerminalDef': TerminalDef}, {}) + options = dict(data['options']) + if transformer is not None: + options['transformer'] = transformer + if postlex is not None: + options['postlex'] = postlex + self.options = LarkOptions.deserialize(options, memo) + self.re = regex if self.options.regex else re + self.rules = [Rule.deserialize(r, memo) for r in data['rules']] + self.source = '' + self._prepare_callbacks() + self.parser = self.parser_class.deserialize(data['parser'], memo, self._callbacks, self.options.postlex, self.re) + return self + + @classmethod + def _load_from_dict(cls, data, memo, transformer=None, postlex=None): + inst = cls.__new__(cls) + return inst._load({'data': data, 'memo': memo}, transformer, postlex) + + @classmethod + def open(cls, grammar_filename, rel_to=None, **options): + """Create an instance of Lark with the grammar given by its filename + + If rel_to is provided, the function will find the grammar filename in relation to it. + + Example: + + >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr") + Lark(...) + + """ + if rel_to: + basepath = os.path.dirname(rel_to) + grammar_filename = os.path.join(basepath, grammar_filename) + with open(grammar_filename, encoding='utf8') as f: + return cls(f, **options) + + def __repr__(self): + return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source, self.options.parser, self.options.lexer) + + + def lex(self, text): + "Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'" + if not hasattr(self, 'lexer'): + self.lexer = self._build_lexer() + stream = self.lexer.lex(text) + if self.options.postlex: + return self.options.postlex.process(stream) + return stream + + def get_terminal(self, name): + "Get information about a terminal" + return self._terminals_dict[name] + + def parse(self, text, start=None, on_error=None): + """Parse the given text, according to the options provided. + + Parameters: + start: str - required if Lark was given multiple possible start symbols (using the start option). + on_error: function - if provided, will be called on UnexpectedToken error. Return true to resume parsing. LALR only. + + Returns a tree, unless specified otherwise. + """ + try: + return self.parser.parse(text, start=start) + except UnexpectedToken as e: + if on_error is None: + raise + + while True: + if not on_error(e): + raise e + try: + return e.puppet.resume_parse() + except UnexpectedToken as e2: + e = e2 + + +###} diff --git a/poetry/core/_vendor/lark/lexer.py b/poetry/core/_vendor/lark/lexer.py new file mode 100644 index 000000000..bff5de9e8 --- /dev/null +++ b/poetry/core/_vendor/lark/lexer.py @@ -0,0 +1,395 @@ +## Lexer Implementation + +import re + +from .utils import Str, classify, get_regexp_width, Py36, Serialize +from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken + +###{standalone + +class Pattern(Serialize): + + def __init__(self, value, flags=()): + self.value = value + self.flags = frozenset(flags) + + def __repr__(self): + return repr(self.to_regexp()) + + # Pattern Hashing assumes all subclasses have a different priority! + def __hash__(self): + return hash((type(self), self.value, self.flags)) + def __eq__(self, other): + return type(self) == type(other) and self.value == other.value and self.flags == other.flags + + def to_regexp(self): + raise NotImplementedError() + + if Py36: + # Python 3.6 changed syntax for flags in regular expression + def _get_flags(self, value): + for f in self.flags: + value = ('(?%s:%s)' % (f, value)) + return value + + else: + def _get_flags(self, value): + for f in self.flags: + value = ('(?%s)' % f) + value + return value + + +class PatternStr(Pattern): + __serialize_fields__ = 'value', 'flags' + + type = "str" + + def to_regexp(self): + return self._get_flags(re.escape(self.value)) + + @property + def min_width(self): + return len(self.value) + max_width = min_width + +class PatternRE(Pattern): + __serialize_fields__ = 'value', 'flags', '_width' + + type = "re" + + def to_regexp(self): + return self._get_flags(self.value) + + _width = None + def _get_width(self): + if self._width is None: + self._width = get_regexp_width(self.to_regexp()) + return self._width + + @property + def min_width(self): + return self._get_width()[0] + @property + def max_width(self): + return self._get_width()[1] + + +class TerminalDef(Serialize): + __serialize_fields__ = 'name', 'pattern', 'priority' + __serialize_namespace__ = PatternStr, PatternRE + + def __init__(self, name, pattern, priority=1): + assert isinstance(pattern, Pattern), pattern + self.name = name + self.pattern = pattern + self.priority = priority + + def __repr__(self): + return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern) + + + +class Token(Str): + __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') + + def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): + try: + self = super(Token, cls).__new__(cls, value) + except UnicodeDecodeError: + value = value.decode('latin1') + self = super(Token, cls).__new__(cls, value) + + self.type = type_ + self.pos_in_stream = pos_in_stream + self.value = value + self.line = line + self.column = column + self.end_line = end_line + self.end_column = end_column + self.end_pos = end_pos + return self + + def update(self, type_=None, value=None): + return Token.new_borrow_pos( + type_ if type_ is not None else self.type, + value if value is not None else self.value, + self + ) + + @classmethod + def new_borrow_pos(cls, type_, value, borrow_t): + return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) + + def __reduce__(self): + return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column, )) + + def __repr__(self): + return 'Token(%s, %r)' % (self.type, self.value) + + def __deepcopy__(self, memo): + return Token(self.type, self.value, self.pos_in_stream, self.line, self.column) + + def __eq__(self, other): + if isinstance(other, Token) and self.type != other.type: + return False + + return Str.__eq__(self, other) + + __hash__ = Str.__hash__ + + +class LineCounter: + def __init__(self): + self.newline_char = '\n' + self.char_pos = 0 + self.line = 1 + self.column = 1 + self.line_start_pos = 0 + + def feed(self, token, test_newline=True): + """Consume a token and calculate the new line & column. + + As an optional optimization, set test_newline=False is token doesn't contain a newline. + """ + if test_newline: + newlines = token.count(self.newline_char) + if newlines: + self.line += newlines + self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1 + + self.char_pos += len(token) + self.column = self.char_pos - self.line_start_pos + 1 + +class _Lex: + "Built to serve both Lexer and ContextualLexer" + def __init__(self, lexer, state=None): + self.lexer = lexer + self.state = state + + def lex(self, stream, newline_types, ignore_types): + newline_types = frozenset(newline_types) + ignore_types = frozenset(ignore_types) + line_ctr = LineCounter() + last_token = None + + while line_ctr.char_pos < len(stream): + lexer = self.lexer + res = lexer.match(stream, line_ctr.char_pos) + if not res: + allowed = {v for m, tfi in lexer.mres for v in tfi.values()} - ignore_types + if not allowed: + allowed = {""} + raise UnexpectedCharacters(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column, allowed=allowed, state=self.state, token_history=last_token and [last_token]) + + value, type_ = res + + if type_ not in ignore_types: + t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) + line_ctr.feed(value, type_ in newline_types) + t.end_line = line_ctr.line + t.end_column = line_ctr.column + t.end_pos = line_ctr.char_pos + if t.type in lexer.callback: + t = lexer.callback[t.type](t) + if not isinstance(t, Token): + raise ValueError("Callbacks must return a token (returned %r)" % t) + yield t + last_token = t + else: + if type_ in lexer.callback: + t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) + lexer.callback[type_](t2) + line_ctr.feed(value, type_ in newline_types) + + + + +class UnlessCallback: + def __init__(self, mres): + self.mres = mres + + def __call__(self, t): + for mre, type_from_index in self.mres: + m = mre.match(t.value) + if m: + t.type = type_from_index[m.lastindex] + break + return t + +class CallChain: + def __init__(self, callback1, callback2, cond): + self.callback1 = callback1 + self.callback2 = callback2 + self.cond = cond + + def __call__(self, t): + t2 = self.callback1(t) + return self.callback2(t) if self.cond(t2) else t2 + + + + + +def _create_unless(terminals, g_regex_flags, re_): + tokens_by_type = classify(terminals, lambda t: type(t.pattern)) + assert len(tokens_by_type) <= 2, tokens_by_type.keys() + embedded_strs = set() + callback = {} + for retok in tokens_by_type.get(PatternRE, []): + unless = [] # {} + for strtok in tokens_by_type.get(PatternStr, []): + if strtok.priority > retok.priority: + continue + s = strtok.pattern.value + m = re_.match(retok.pattern.to_regexp(), s, g_regex_flags) + if m and m.group(0) == s: + unless.append(strtok) + if strtok.pattern.flags <= retok.pattern.flags: + embedded_strs.add(strtok) + if unless: + callback[retok.name] = UnlessCallback(build_mres(unless, g_regex_flags, re_, match_whole=True)) + + terminals = [t for t in terminals if t not in embedded_strs] + return terminals, callback + + +def _build_mres(terminals, max_size, g_regex_flags, match_whole, re_): + # Python sets an unreasonable group limit (currently 100) in its re module + # Worse, the only way to know we reached it is by catching an AssertionError! + # This function recursively tries less and less groups until it's successful. + postfix = '$' if match_whole else '' + mres = [] + while terminals: + try: + mre = re_.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]), g_regex_flags) + except AssertionError: # Yes, this is what Python provides us.. :/ + return _build_mres(terminals, max_size//2, g_regex_flags, match_whole, re_) + + # terms_from_name = {t.name: t for t in terminals[:max_size]} + mres.append((mre, {i:n for n,i in mre.groupindex.items()} )) + terminals = terminals[max_size:] + return mres + +def build_mres(terminals, g_regex_flags, re_, match_whole=False): + return _build_mres(terminals, len(terminals), g_regex_flags, match_whole, re_) + +def _regexp_has_newline(r): + r"""Expressions that may indicate newlines in a regexp: + - newlines (\n) + - escaped newline (\\n) + - anything but ([^...]) + - any-char (.) when the flag (?s) exists + - spaces (\s) + """ + return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r) + +class Lexer(object): + """Lexer interface + + Method Signatures: + lex(self, stream) -> Iterator[Token] + """ + lex = NotImplemented + + +class TraditionalLexer(Lexer): + + def __init__(self, terminals, re_, ignore=(), user_callbacks={}, g_regex_flags=0): + assert all(isinstance(t, TerminalDef) for t in terminals), terminals + + terminals = list(terminals) + + self.re = re_ + # Sanitization + for t in terminals: + try: + self.re.compile(t.pattern.to_regexp(), g_regex_flags) + except self.re.error: + raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) + + if t.pattern.min_width == 0: + raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern)) + + assert set(ignore) <= {t.name for t in terminals} + + # Init + self.newline_types = [t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())] + self.ignore_types = list(ignore) + + terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) + self.terminals = terminals + self.user_callbacks = user_callbacks + self.build(g_regex_flags) + + def build(self, g_regex_flags=0): + terminals, self.callback = _create_unless(self.terminals, g_regex_flags, re_=self.re) + assert all(self.callback.values()) + + for type_, f in self.user_callbacks.items(): + if type_ in self.callback: + # Already a callback there, probably UnlessCallback + self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_) + else: + self.callback[type_] = f + + self.mres = build_mres(terminals, g_regex_flags, self.re) + + def match(self, stream, pos): + for mre, type_from_index in self.mres: + m = mre.match(stream, pos) + if m: + return m.group(0), type_from_index[m.lastindex] + + def lex(self, stream): + return _Lex(self).lex(stream, self.newline_types, self.ignore_types) + + + + +class ContextualLexer(Lexer): + + def __init__(self, terminals, states, re_, ignore=(), always_accept=(), user_callbacks={}, g_regex_flags=0): + self.re = re_ + tokens_by_name = {} + for t in terminals: + assert t.name not in tokens_by_name, t + tokens_by_name[t.name] = t + + lexer_by_tokens = {} + self.lexers = {} + for state, accepts in states.items(): + key = frozenset(accepts) + try: + lexer = lexer_by_tokens[key] + except KeyError: + accepts = set(accepts) | set(ignore) | set(always_accept) + state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name] + lexer = TraditionalLexer(state_tokens, re_=self.re, ignore=ignore, user_callbacks=user_callbacks, g_regex_flags=g_regex_flags) + lexer_by_tokens[key] = lexer + + self.lexers[state] = lexer + + self.root_lexer = TraditionalLexer(terminals, re_=self.re, ignore=ignore, user_callbacks=user_callbacks, g_regex_flags=g_regex_flags) + + def lex(self, stream, get_parser_state): + parser_state = get_parser_state() + l = _Lex(self.lexers[parser_state], parser_state) + try: + for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types): + yield x + parser_state = get_parser_state() + l.lexer = self.lexers[parser_state] + l.state = parser_state # For debug only, no need to worry about multithreading + except UnexpectedCharacters as e: + # In the contextual lexer, UnexpectedCharacters can mean that the terminal is defined, + # but not in the current context. + # This tests the input against the global context, to provide a nicer error. + root_match = self.root_lexer.match(stream, e.pos_in_stream) + if not root_match: + raise + + value, type_ = root_match + t = Token(type_, value, e.pos_in_stream, e.line, e.column) + raise UnexpectedToken(t, e.allowed, state=e.state) + +###} diff --git a/poetry/core/_vendor/lark/load_grammar.py b/poetry/core/_vendor/lark/load_grammar.py new file mode 100644 index 000000000..407d8d16d --- /dev/null +++ b/poetry/core/_vendor/lark/load_grammar.py @@ -0,0 +1,947 @@ +"Parses and creates Grammar objects" + +import os.path +import sys +from copy import copy, deepcopy +from io import open + +from .utils import bfs, eval_escaping +from .lexer import Token, TerminalDef, PatternStr, PatternRE + +from .parse_tree_builder import ParseTreeBuilder +from .parser_frontends import LALR_TraditionalLexer +from .common import LexerConf, ParserConf +from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol +from .utils import classify, suppress, dedup_list, Str +from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken + +from .tree import Tree, SlottedTree as ST +from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive +inline_args = v_args(inline=True) + +__path__ = os.path.dirname(__file__) +IMPORT_PATHS = [os.path.join(__path__, 'grammars')] + +EXT = '.lark' + +_RE_FLAGS = 'imslux' + +_EMPTY = Symbol('__empty__') + +_TERMINAL_NAMES = { + '.' : 'DOT', + ',' : 'COMMA', + ':' : 'COLON', + ';' : 'SEMICOLON', + '+' : 'PLUS', + '-' : 'MINUS', + '*' : 'STAR', + '/' : 'SLASH', + '\\' : 'BACKSLASH', + '|' : 'VBAR', + '?' : 'QMARK', + '!' : 'BANG', + '@' : 'AT', + '#' : 'HASH', + '$' : 'DOLLAR', + '%' : 'PERCENT', + '^' : 'CIRCUMFLEX', + '&' : 'AMPERSAND', + '_' : 'UNDERSCORE', + '<' : 'LESSTHAN', + '>' : 'MORETHAN', + '=' : 'EQUAL', + '"' : 'DBLQUOTE', + '\'' : 'QUOTE', + '`' : 'BACKQUOTE', + '~' : 'TILDE', + '(' : 'LPAR', + ')' : 'RPAR', + '{' : 'LBRACE', + '}' : 'RBRACE', + '[' : 'LSQB', + ']' : 'RSQB', + '\n' : 'NEWLINE', + '\r\n' : 'CRLF', + '\t' : 'TAB', + ' ' : 'SPACE', +} + +# Grammar Parser +TERMINALS = { + '_LPAR': r'\(', + '_RPAR': r'\)', + '_LBRA': r'\[', + '_RBRA': r'\]', + '_LBRACE': r'\{', + '_RBRACE': r'\}', + 'OP': '[+*]|[?](?![a-z])', + '_COLON': ':', + '_COMMA': ',', + '_OR': r'\|', + '_DOT': r'\.(?!\.)', + '_DOTDOT': r'\.\.', + 'TILDE': '~', + 'RULE': '!?[_?]?[a-z][_a-z0-9]*', + 'TERMINAL': '_?[A-Z][_A-Z0-9]*', + 'STRING': r'"(\\"|\\\\|[^"\n])*?"i?', + 'REGEXP': r'/(?!/)(\\/|\\\\|[^/\n])*?/[%s]*' % _RE_FLAGS, + '_NL': r'(\r?\n)+\s*', + 'WS': r'[ \t]+', + 'COMMENT': r'\s*//[^\n]*', + '_TO': '->', + '_IGNORE': r'%ignore', + '_DECLARE': r'%declare', + '_IMPORT': r'%import', + 'NUMBER': r'[+-]?\d+', +} + +RULES = { + 'start': ['_list'], + '_list': ['_item', '_list _item'], + '_item': ['rule', 'term', 'statement', '_NL'], + + 'rule': ['RULE template_params _COLON expansions _NL', + 'RULE template_params _DOT NUMBER _COLON expansions _NL'], + 'template_params': ['_LBRACE _template_params _RBRACE', + ''], + '_template_params': ['RULE', + '_template_params _COMMA RULE'], + 'expansions': ['alias', + 'expansions _OR alias', + 'expansions _NL _OR alias'], + + '?alias': ['expansion _TO RULE', 'expansion'], + 'expansion': ['_expansion'], + + '_expansion': ['', '_expansion expr'], + + '?expr': ['atom', + 'atom OP', + 'atom TILDE NUMBER', + 'atom TILDE NUMBER _DOTDOT NUMBER', + ], + + '?atom': ['_LPAR expansions _RPAR', + 'maybe', + 'value'], + + 'value': ['terminal', + 'nonterminal', + 'literal', + 'range', + 'template_usage'], + + 'terminal': ['TERMINAL'], + 'nonterminal': ['RULE'], + + '?name': ['RULE', 'TERMINAL'], + + 'maybe': ['_LBRA expansions _RBRA'], + 'range': ['STRING _DOTDOT STRING'], + + 'template_usage': ['RULE _LBRACE _template_args _RBRACE'], + '_template_args': ['value', + '_template_args _COMMA value'], + + 'term': ['TERMINAL _COLON expansions _NL', + 'TERMINAL _DOT NUMBER _COLON expansions _NL'], + 'statement': ['ignore', 'import', 'declare'], + 'ignore': ['_IGNORE expansions _NL'], + 'declare': ['_DECLARE _declare_args _NL'], + 'import': ['_IMPORT _import_path _NL', + '_IMPORT _import_path _LPAR name_list _RPAR _NL', + '_IMPORT _import_path _TO name _NL'], + + '_import_path': ['import_lib', 'import_rel'], + 'import_lib': ['_import_args'], + 'import_rel': ['_DOT _import_args'], + '_import_args': ['name', '_import_args _DOT name'], + + 'name_list': ['_name_list'], + '_name_list': ['name', '_name_list _COMMA name'], + + '_declare_args': ['name', '_declare_args name'], + 'literal': ['REGEXP', 'STRING'], +} + +@inline_args +class EBNF_to_BNF(Transformer_InPlace): + def __init__(self): + self.new_rules = [] + self.rules_by_expr = {} + self.prefix = 'anon' + self.i = 0 + self.rule_options = None + + def _add_recurse_rule(self, type_, expr): + if expr in self.rules_by_expr: + return self.rules_by_expr[expr] + + new_name = '__%s_%s_%d' % (self.prefix, type_, self.i) + self.i += 1 + t = NonTerminal(new_name) + tree = ST('expansions', [ST('expansion', [expr]), ST('expansion', [t, expr])]) + self.new_rules.append((new_name, tree, self.rule_options)) + self.rules_by_expr[expr] = t + return t + + def expr(self, rule, op, *args): + if op.value == '?': + empty = ST('expansion', []) + return ST('expansions', [rule, empty]) + elif op.value == '+': + # a : b c+ d + # --> + # a : b _c d + # _c : _c c | c; + return self._add_recurse_rule('plus', rule) + elif op.value == '*': + # a : b c* d + # --> + # a : b _c? d + # _c : _c c | c; + new_name = self._add_recurse_rule('star', rule) + return ST('expansions', [new_name, ST('expansion', [])]) + elif op.value == '~': + if len(args) == 1: + mn = mx = int(args[0]) + else: + mn, mx = map(int, args) + if mx < mn or mn < 0: + raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx)) + return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx+1)]) + assert False, op + + def maybe(self, rule): + keep_all_tokens = self.rule_options and self.rule_options.keep_all_tokens + + def will_not_get_removed(sym): + if isinstance(sym, NonTerminal): + return not sym.name.startswith('_') + if isinstance(sym, Terminal): + return keep_all_tokens or not sym.filter_out + assert False + + if any(rule.scan_values(will_not_get_removed)): + empty = _EMPTY + else: + empty = ST('expansion', []) + + return ST('expansions', [rule, empty]) + + +class SimplifyRule_Visitor(Visitor): + + @staticmethod + def _flatten(tree): + while True: + to_expand = [i for i, child in enumerate(tree.children) + if isinstance(child, Tree) and child.data == tree.data] + if not to_expand: + break + tree.expand_kids_by_index(*to_expand) + + def expansion(self, tree): + # rules_list unpacking + # a : b (c|d) e + # --> + # a : b c e | b d e + # + # In AST terms: + # expansion(b, expansions(c, d), e) + # --> + # expansions( expansion(b, c, e), expansion(b, d, e) ) + + self._flatten(tree) + + for i, child in enumerate(tree.children): + if isinstance(child, Tree) and child.data == 'expansions': + tree.data = 'expansions' + tree.children = [self.visit(ST('expansion', [option if i==j else other + for j, other in enumerate(tree.children)])) + for option in dedup_list(child.children)] + self._flatten(tree) + break + + def alias(self, tree): + rule, alias_name = tree.children + if rule.data == 'expansions': + aliases = [] + for child in tree.children[0].children: + aliases.append(ST('alias', [child, alias_name])) + tree.data = 'expansions' + tree.children = aliases + + def expansions(self, tree): + self._flatten(tree) + # Ensure all children are unique + if len(set(tree.children)) != len(tree.children): + tree.children = dedup_list(tree.children) # dedup is expensive, so try to minimize its use + + +class RuleTreeToText(Transformer): + def expansions(self, x): + return x + def expansion(self, symbols): + return symbols, None + def alias(self, x): + (expansion, _alias), alias = x + assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed + return expansion, alias.value + + +@inline_args +class CanonizeTree(Transformer_InPlace): + def tokenmods(self, *args): + if len(args) == 1: + return list(args) + tokenmods, value = args + return tokenmods + [value] + +class PrepareAnonTerminals(Transformer_InPlace): + "Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them" + + def __init__(self, terminals): + self.terminals = terminals + self.term_set = {td.name for td in self.terminals} + self.term_reverse = {td.pattern: td for td in terminals} + self.i = 0 + + + @inline_args + def pattern(self, p): + value = p.value + if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags: + raise GrammarError(u'Conflicting flags for the same terminal: %s' % p) + + term_name = None + + if isinstance(p, PatternStr): + try: + # If already defined, use the user-defined terminal name + term_name = self.term_reverse[p].name + except KeyError: + # Try to assign an indicative anon-terminal name + try: + term_name = _TERMINAL_NAMES[value] + except KeyError: + if value.isalnum() and value[0].isalpha() and value.upper() not in self.term_set: + with suppress(UnicodeEncodeError): + value.upper().encode('ascii') # Make sure we don't have unicode in our terminal names + term_name = value.upper() + + if term_name in self.term_set: + term_name = None + + elif isinstance(p, PatternRE): + if p in self.term_reverse: # Kind of a wierd placement.name + term_name = self.term_reverse[p].name + else: + assert False, p + + if term_name is None: + term_name = '__ANON_%d' % self.i + self.i += 1 + + if term_name not in self.term_set: + assert p not in self.term_reverse + self.term_set.add(term_name) + termdef = TerminalDef(term_name, p) + self.term_reverse[p] = termdef + self.terminals.append(termdef) + + return Terminal(term_name, filter_out=isinstance(p, PatternStr)) + +class _ReplaceSymbols(Transformer_InPlace): + " Helper for ApplyTemplates " + + def __init__(self): + self.names = {} + + def value(self, c): + if len(c) == 1 and isinstance(c[0], Token) and c[0].value in self.names: + return self.names[c[0].value] + return self.__default__('value', c, None) + + def template_usage(self, c): + if c[0] in self.names: + return self.__default__('template_usage', [self.names[c[0]].name] + c[1:], None) + return self.__default__('template_usage', c, None) + +class ApplyTemplates(Transformer_InPlace): + " Apply the templates, creating new rules that represent the used templates " + + def __init__(self, rule_defs): + self.rule_defs = rule_defs + self.replacer = _ReplaceSymbols() + self.created_templates = set() + + def template_usage(self, c): + name = c[0] + args = c[1:] + result_name = "%s{%s}" % (name, ",".join(a.name for a in args)) + if result_name not in self.created_templates: + self.created_templates.add(result_name) + (_n, params, tree, options) ,= (t for t in self.rule_defs if t[0] == name) + assert len(params) == len(args), args + result_tree = deepcopy(tree) + self.replacer.names = dict(zip(params, args)) + self.replacer.transform(result_tree) + self.rule_defs.append((result_name, [], result_tree, deepcopy(options))) + return NonTerminal(result_name) + + +def _rfind(s, choices): + return max(s.rfind(c) for c in choices) + + + + +def _literal_to_pattern(literal): + v = literal.value + flag_start = _rfind(v, '/"')+1 + assert flag_start > 0 + flags = v[flag_start:] + assert all(f in _RE_FLAGS for f in flags), flags + + v = v[:flag_start] + assert v[0] == v[-1] and v[0] in '"/' + x = v[1:-1] + + s = eval_escaping(x) + + if literal.type == 'STRING': + s = s.replace('\\\\', '\\') + + return { 'STRING': PatternStr, + 'REGEXP': PatternRE }[literal.type](s, flags) + + +@inline_args +class PrepareLiterals(Transformer_InPlace): + def literal(self, literal): + return ST('pattern', [_literal_to_pattern(literal)]) + + def range(self, start, end): + assert start.type == end.type == 'STRING' + start = start.value[1:-1] + end = end.value[1:-1] + assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1, (start, end, len(eval_escaping(start)), len(eval_escaping(end))) + regexp = '[%s-%s]' % (start, end) + return ST('pattern', [PatternRE(regexp)]) + + +class TerminalTreeToPattern(Transformer): + def pattern(self, ps): + p ,= ps + return p + + def expansion(self, items): + assert items + if len(items) == 1: + return items[0] + if len({i.flags for i in items}) > 1: + raise GrammarError("Lark doesn't support joining terminals with conflicting flags!") + return PatternRE(''.join(i.to_regexp() for i in items), items[0].flags if items else ()) + + def expansions(self, exps): + if len(exps) == 1: + return exps[0] + if len({i.flags for i in exps}) > 1: + raise GrammarError("Lark doesn't support joining terminals with conflicting flags!") + return PatternRE('(?:%s)' % ('|'.join(i.to_regexp() for i in exps)), exps[0].flags) + + def expr(self, args): + inner, op = args[:2] + if op == '~': + if len(args) == 3: + op = "{%d}" % int(args[2]) + else: + mn, mx = map(int, args[2:]) + if mx < mn: + raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx)) + op = "{%d,%d}" % (mn, mx) + else: + assert len(args) == 2 + return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags) + + def maybe(self, expr): + return self.expr(expr + ['?']) + + def alias(self, t): + raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)") + + def value(self, v): + return v[0] + +class PrepareSymbols(Transformer_InPlace): + def value(self, v): + v ,= v + if isinstance(v, Tree): + return v + elif v.type == 'RULE': + return NonTerminal(Str(v.value)) + elif v.type == 'TERMINAL': + return Terminal(Str(v.value), filter_out=v.startswith('_')) + assert False + +def _choice_of_rules(rules): + return ST('expansions', [ST('expansion', [Token('RULE', name)]) for name in rules]) + +def nr_deepcopy_tree(t): + "Deepcopy tree `t` without recursion" + return Transformer_NonRecursive(False).transform(t) + +class Grammar: + def __init__(self, rule_defs, term_defs, ignore): + self.term_defs = term_defs + self.rule_defs = rule_defs + self.ignore = ignore + + def compile(self, start): + # We change the trees in-place (to support huge grammars) + # So deepcopy allows calling compile more than once. + term_defs = deepcopy(list(self.term_defs)) + rule_defs = [(n,p,nr_deepcopy_tree(t),o) for n,p,t,o in self.rule_defs] + + # =================== + # Compile Terminals + # =================== + + # Convert terminal-trees to strings/regexps + + for name, (term_tree, priority) in term_defs: + if term_tree is None: # Terminal added through %declare + continue + expansions = list(term_tree.find_data('expansion')) + if len(expansions) == 1 and not expansions[0].children: + raise GrammarError("Terminals cannot be empty (%s)" % name) + + transformer = PrepareLiterals() * TerminalTreeToPattern() + terminals = [TerminalDef(name, transformer.transform( term_tree ), priority) + for name, (term_tree, priority) in term_defs if term_tree] + + # ================= + # Compile Rules + # ================= + + # 1. Pre-process terminals + transformer = PrepareLiterals() * PrepareSymbols() * PrepareAnonTerminals(terminals) # Adds to terminals + + # 2. Inline Templates + + transformer *= ApplyTemplates(rule_defs) + + # 3. Convert EBNF to BNF (and apply step 1 & 2) + ebnf_to_bnf = EBNF_to_BNF() + rules = [] + i = 0 + while i < len(rule_defs): # We have to do it like this because rule_defs might grow due to templates + name, params, rule_tree, options = rule_defs[i] + i += 1 + if len(params) != 0: # Dont transform templates + continue + ebnf_to_bnf.rule_options = RuleOptions(keep_all_tokens=True) if options.keep_all_tokens else None + ebnf_to_bnf.prefix = name + tree = transformer.transform(rule_tree) + res = ebnf_to_bnf.transform(tree) + rules.append((name, res, options)) + rules += ebnf_to_bnf.new_rules + + assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision" + + # 4. Compile tree to Rule objects + rule_tree_to_text = RuleTreeToText() + + simplify_rule = SimplifyRule_Visitor() + compiled_rules = [] + for rule_content in rules: + name, tree, options = rule_content + simplify_rule.visit(tree) + expansions = rule_tree_to_text.transform(tree) + + for i, (expansion, alias) in enumerate(expansions): + if alias and name.startswith('_'): + raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)" % (name, alias)) + + empty_indices = [x==_EMPTY for x in expansion] + if any(empty_indices): + exp_options = copy(options) or RuleOptions() + exp_options.empty_indices = empty_indices + expansion = [x for x in expansion if x!=_EMPTY] + else: + exp_options = options + + assert all(isinstance(x, Symbol) for x in expansion), expansion + rule = Rule(NonTerminal(name), expansion, i, alias, exp_options) + compiled_rules.append(rule) + + # Remove duplicates of empty rules, throw error for non-empty duplicates + if len(set(compiled_rules)) != len(compiled_rules): + duplicates = classify(compiled_rules, lambda x: x) + for dups in duplicates.values(): + if len(dups) > 1: + if dups[0].expansion: + raise GrammarError("Rules defined twice: %s\n\n(Might happen due to colliding expansion of optionals: [] or ?)" + % ''.join('\n * %s' % i for i in dups)) + + # Empty rule; assert all other attributes are equal + assert len({(r.alias, r.order, r.options) for r in dups}) == len(dups) + + # Remove duplicates + compiled_rules = list(set(compiled_rules)) + + + # Filter out unused rules + while True: + c = len(compiled_rules) + used_rules = {s for r in compiled_rules + for s in r.expansion + if isinstance(s, NonTerminal) + and s != r.origin} + used_rules |= {NonTerminal(s) for s in start} + compiled_rules = [r for r in compiled_rules if r.origin in used_rules] + if len(compiled_rules) == c: + break + + # Filter out unused terminals + used_terms = {t.name for r in compiled_rules + for t in r.expansion + if isinstance(t, Terminal)} + terminals = [t for t in terminals if t.name in used_terms or t.name in self.ignore] + + return terminals, compiled_rules, self.ignore + + + +_imported_grammars = {} +def import_grammar(grammar_path, re_, base_paths=[]): + if grammar_path not in _imported_grammars: + import_paths = base_paths + IMPORT_PATHS + for import_path in import_paths: + with suppress(IOError): + joined_path = os.path.join(import_path, grammar_path) + with open(joined_path, encoding='utf8') as f: + text = f.read() + grammar = load_grammar(text, joined_path, re_) + _imported_grammars[grammar_path] = grammar + break + else: + open(grammar_path, encoding='utf8') + assert False + + return _imported_grammars[grammar_path] + +def import_from_grammar_into_namespace(grammar, namespace, aliases): + """Returns all rules and terminals of grammar, prepended + with a 'namespace' prefix, except for those which are aliased. + """ + + imported_terms = dict(grammar.term_defs) + imported_rules = {n:(n,p,deepcopy(t),o) for n,p,t,o in grammar.rule_defs} + + term_defs = [] + rule_defs = [] + + def rule_dependencies(symbol): + if symbol.type != 'RULE': + return [] + try: + _, params, tree,_ = imported_rules[symbol] + except KeyError: + raise GrammarError("Missing symbol '%s' in grammar %s" % (symbol, namespace)) + return _find_used_symbols(tree) - set(params) + + + + def get_namespace_name(name, params): + if params is not None: + try: + return params[name] + except KeyError: + pass + try: + return aliases[name].value + except KeyError: + if name[0] == '_': + return '_%s__%s' % (namespace, name[1:]) + return '%s__%s' % (namespace, name) + + to_import = list(bfs(aliases, rule_dependencies)) + for symbol in to_import: + if symbol.type == 'TERMINAL': + term_defs.append([get_namespace_name(symbol, None), imported_terms[symbol]]) + else: + assert symbol.type == 'RULE' + _, params, tree, options = imported_rules[symbol] + params_map = {p: ('%s__%s' if p[0]!='_' else '_%s__%s' ) % (namespace, p) for p in params} + for t in tree.iter_subtrees(): + for i, c in enumerate(t.children): + if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'): + t.children[i] = Token(c.type, get_namespace_name(c, params_map)) + params = [params_map[p] for p in params] # We can not rely on ordered dictionaries + rule_defs.append((get_namespace_name(symbol, params_map), params, tree, options)) + + + return term_defs, rule_defs + + + +def resolve_term_references(term_defs): + # TODO Solve with transitive closure (maybe) + + term_dict = {k:t for k, (t,_p) in term_defs} + assert len(term_dict) == len(term_defs), "Same name defined twice?" + + while True: + changed = False + for name, (token_tree, _p) in term_defs: + if token_tree is None: # Terminal added through %declare + continue + for exp in token_tree.find_data('value'): + item ,= exp.children + if isinstance(item, Token): + if item.type == 'RULE': + raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name)) + if item.type == 'TERMINAL': + term_value = term_dict[item] + assert term_value is not None + exp.children[0] = term_value + changed = True + if not changed: + break + + for name, term in term_dict.items(): + if term: # Not just declared + for child in term.children: + ids = [id(x) for x in child.iter_subtrees()] + if id(term) in ids: + raise GrammarError("Recursion in terminal '%s' (recursion is only allowed in rules, not terminals)" % name) + + +def options_from_rule(name, params, *x): + if len(x) > 1: + priority, expansions = x + priority = int(priority) + else: + expansions ,= x + priority = None + params = [t.value for t in params.children] if params is not None else [] # For the grammar parser + + keep_all_tokens = name.startswith('!') + name = name.lstrip('!') + expand1 = name.startswith('?') + name = name.lstrip('?') + + return name, params, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority, + template_source=(name if params else None)) + + +def symbols_from_strcase(expansion): + return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion] + +@inline_args +class PrepareGrammar(Transformer_InPlace): + def terminal(self, name): + return name + def nonterminal(self, name): + return name + + +def _find_used_symbols(tree): + assert tree.data == 'expansions' + return {t for x in tree.find_data('expansion') + for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))} + +class GrammarLoader: + def __init__(self, re_): + self.re = re_ + terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()] + + rules = [options_from_rule(name, None, x) for name, x in RULES.items()] + rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), i, None, o) for r, _p, xs, o in rules for i, x in enumerate(xs)] + callback = ParseTreeBuilder(rules, ST).create_callback() + lexer_conf = LexerConf(terminals, ['WS', 'COMMENT']) + + parser_conf = ParserConf(rules, callback, ['start']) + self.parser = LALR_TraditionalLexer(lexer_conf, parser_conf, re_) + + self.canonize_tree = CanonizeTree() + + def load_grammar(self, grammar_text, grammar_name=''): + "Parse grammar_text, verify, and create Grammar object. Display nice messages on error." + + try: + tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') ) + except UnexpectedCharacters as e: + context = e.get_context(grammar_text) + raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" % + (e.line, e.column, grammar_name, context)) + except UnexpectedToken as e: + context = e.get_context(grammar_text) + error = e.match_examples(self.parser.parse, { + 'Unclosed parenthesis': ['a: (\n'], + 'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'], + 'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'], + 'Alias expects lowercase name': ['a: -> "a"\n'], + 'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'], + 'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'], + 'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'], + '%import expects a name': ['%import "a"\n'], + '%ignore expects a value': ['%ignore %import\n'], + }) + if error: + raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context)) + elif 'STRING' in e.expected: + raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context)) + raise + + tree = PrepareGrammar().transform(tree) + + # Extract grammar items + defs = classify(tree.children, lambda c: c.data, lambda c: c.children) + term_defs = defs.pop('term', []) + rule_defs = defs.pop('rule', []) + statements = defs.pop('statement', []) + assert not defs + + term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs] + term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs] + rule_defs = [options_from_rule(*x) for x in rule_defs] + + # Execute statements + ignore, imports = [], {} + for (stmt,) in statements: + if stmt.data == 'ignore': + t ,= stmt.children + ignore.append(t) + elif stmt.data == 'import': + if len(stmt.children) > 1: + path_node, arg1 = stmt.children + else: + path_node, = stmt.children + arg1 = None + + if isinstance(arg1, Tree): # Multi import + dotted_path = tuple(path_node.children) + names = arg1.children + aliases = dict(zip(names, names)) # Can't have aliased multi import, so all aliases will be the same as names + else: # Single import + dotted_path = tuple(path_node.children[:-1]) + name = path_node.children[-1] # Get name from dotted path + aliases = {name: arg1 or name} # Aliases if exist + + if path_node.data == 'import_lib': # Import from library + base_paths = [] + else: # Relative import + if grammar_name == '': # Import relative to script file path if grammar is coded in script + try: + base_file = os.path.abspath(sys.modules['__main__'].__file__) + except AttributeError: + base_file = None + else: + base_file = grammar_name # Import relative to grammar file path if external grammar file + if base_file: + base_paths = [os.path.split(base_file)[0]] + else: + base_paths = [os.path.abspath(os.path.curdir)] + + try: + import_base_paths, import_aliases = imports[dotted_path] + assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path) + import_aliases.update(aliases) + except KeyError: + imports[dotted_path] = base_paths, aliases + + elif stmt.data == 'declare': + for t in stmt.children: + term_defs.append([t.value, (None, None)]) + else: + assert False, stmt + + # import grammars + for dotted_path, (base_paths, aliases) in imports.items(): + grammar_path = os.path.join(*dotted_path) + EXT + g = import_grammar(grammar_path, self.re, base_paths=base_paths) + new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases) + + term_defs += new_td + rule_defs += new_rd + + # Verify correctness 1 + for name, _ in term_defs: + if name.startswith('__'): + raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name) + + # Handle ignore tokens + # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's + # inability to handle duplicate terminals (two names, one value) + ignore_names = [] + for t in ignore: + if t.data=='expansions' and len(t.children) == 1: + t2 ,= t.children + if t2.data=='expansion' and len(t2.children) == 1: + item ,= t2.children + if item.data == 'value': + item ,= item.children + if isinstance(item, Token) and item.type == 'TERMINAL': + ignore_names.append(item.value) + continue + + name = '__IGNORE_%d'% len(ignore_names) + ignore_names.append(name) + term_defs.append((name, (t, 1))) + + # Verify correctness 2 + terminal_names = set() + for name, _ in term_defs: + if name in terminal_names: + raise GrammarError("Terminal '%s' defined more than once" % name) + terminal_names.add(name) + + if set(ignore_names) > terminal_names: + raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names)) + + resolve_term_references(term_defs) + + rules = rule_defs + + rule_names = {} + for name, params, _x, _o in rules: + if name.startswith('__'): + raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name) + if name in rule_names: + raise GrammarError("Rule '%s' defined more than once" % name) + rule_names[name] = len(params) + + for name, params , expansions, _o in rules: + for i, p in enumerate(params): + if p in rule_names: + raise GrammarError("Template Parameter conflicts with rule %s (in template %s)" % (p, name)) + if p in params[:i]: + raise GrammarError("Duplicate Template Parameter %s (in template %s)" % (p, name)) + for temp in expansions.find_data('template_usage'): + sym = temp.children[0] + args = temp.children[1:] + if sym not in params: + if sym not in rule_names: + raise GrammarError("Template '%s' used but not defined (in rule %s)" % (sym, name)) + if len(args) != rule_names[sym]: + raise GrammarError("Wrong number of template arguments used for %s " + "(expected %s, got %s) (in rule %s)"%(sym, rule_names[sym], len(args), name)) + for sym in _find_used_symbols(expansions): + if sym.type == 'TERMINAL': + if sym not in terminal_names: + raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name)) + else: + if sym not in rule_names and sym not in params: + raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name)) + + + return Grammar(rules, term_defs, ignore_names) + + + +def load_grammar(grammar, source, re_): + return GrammarLoader(re_).load_grammar(grammar, source) diff --git a/poetry/core/_vendor/lark/parse_tree_builder.py b/poetry/core/_vendor/lark/parse_tree_builder.py new file mode 100644 index 000000000..5a7c5d703 --- /dev/null +++ b/poetry/core/_vendor/lark/parse_tree_builder.py @@ -0,0 +1,277 @@ +from .exceptions import GrammarError +from .lexer import Token +from .tree import Tree +from .visitors import InlineTransformer # XXX Deprecated +from .visitors import Transformer_InPlace +from .visitors import _vargs_meta, _vargs_meta_inline + +###{standalone +from functools import partial, wraps +from itertools import repeat, product + + +class ExpandSingleChild: + def __init__(self, node_builder): + self.node_builder = node_builder + + def __call__(self, children): + if len(children) == 1: + return children[0] + else: + return self.node_builder(children) + +class PropagatePositions: + def __init__(self, node_builder): + self.node_builder = node_builder + + def __call__(self, children): + res = self.node_builder(children) + + # local reference to Tree.meta reduces number of presence checks + if isinstance(res, Tree): + res_meta = res.meta + for c in children: + if isinstance(c, Tree): + child_meta = c.meta + if not child_meta.empty: + res_meta.line = child_meta.line + res_meta.column = child_meta.column + res_meta.start_pos = child_meta.start_pos + res_meta.empty = False + break + elif isinstance(c, Token): + res_meta.line = c.line + res_meta.column = c.column + res_meta.start_pos = c.pos_in_stream + res_meta.empty = False + break + + for c in reversed(children): + if isinstance(c, Tree): + child_meta = c.meta + if not child_meta.empty: + res_meta.end_line = child_meta.end_line + res_meta.end_column = child_meta.end_column + res_meta.end_pos = child_meta.end_pos + res_meta.empty = False + break + elif isinstance(c, Token): + res_meta.end_line = c.end_line + res_meta.end_column = c.end_column + res_meta.end_pos = c.end_pos + res_meta.empty = False + break + + return res + + +class ChildFilter: + def __init__(self, to_include, append_none, node_builder): + self.node_builder = node_builder + self.to_include = to_include + self.append_none = append_none + + def __call__(self, children): + filtered = [] + + for i, to_expand, add_none in self.to_include: + if add_none: + filtered += [None] * add_none + if to_expand: + filtered += children[i].children + else: + filtered.append(children[i]) + + if self.append_none: + filtered += [None] * self.append_none + + return self.node_builder(filtered) + +class ChildFilterLALR(ChildFilter): + "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)" + + def __call__(self, children): + filtered = [] + for i, to_expand, add_none in self.to_include: + if add_none: + filtered += [None] * add_none + if to_expand: + if filtered: + filtered += children[i].children + else: # Optimize for left-recursion + filtered = children[i].children + else: + filtered.append(children[i]) + + if self.append_none: + filtered += [None] * self.append_none + + return self.node_builder(filtered) + +class ChildFilterLALR_NoPlaceholders(ChildFilter): + "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)" + def __init__(self, to_include, node_builder): + self.node_builder = node_builder + self.to_include = to_include + + def __call__(self, children): + filtered = [] + for i, to_expand in self.to_include: + if to_expand: + if filtered: + filtered += children[i].children + else: # Optimize for left-recursion + filtered = children[i].children + else: + filtered.append(children[i]) + return self.node_builder(filtered) + +def _should_expand(sym): + return not sym.is_term and sym.name.startswith('_') + +def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices): + # Prepare empty_indices as: How many Nones to insert at each index? + if _empty_indices: + assert _empty_indices.count(False) == len(expansion) + s = ''.join(str(int(b)) for b in _empty_indices) + empty_indices = [len(ones) for ones in s.split('0')] + assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion)) + else: + empty_indices = [0] * (len(expansion)+1) + + to_include = [] + nones_to_add = 0 + for i, sym in enumerate(expansion): + nones_to_add += empty_indices[i] + if keep_all_tokens or not (sym.is_term and sym.filter_out): + to_include.append((i, _should_expand(sym), nones_to_add)) + nones_to_add = 0 + + nones_to_add += empty_indices[len(expansion)] + + if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include): + if _empty_indices or ambiguous: + return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add) + else: + # LALR without placeholders + return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include]) + +class AmbiguousExpander: + """Deal with the case where we're expanding children ('_rule') into a parent but the children + are ambiguous. i.e. (parent->_ambig->_expand_this_rule). In this case, make the parent itself + ambiguous with as many copies as their are ambiguous children, and then copy the ambiguous children + into the right parents in the right places, essentially shifting the ambiguiuty up the tree.""" + def __init__(self, to_expand, tree_class, node_builder): + self.node_builder = node_builder + self.tree_class = tree_class + self.to_expand = to_expand + + def __call__(self, children): + def _is_ambig_tree(child): + return hasattr(child, 'data') and child.data == '_ambig' + + #### When we're repeatedly expanding ambiguities we can end up with nested ambiguities. + # All children of an _ambig node should be a derivation of that ambig node, hence + # it is safe to assume that if we see an _ambig node nested within an ambig node + # it is safe to simply expand it into the parent _ambig node as an alternative derivation. + ambiguous = [] + for i, child in enumerate(children): + if _is_ambig_tree(child): + if i in self.to_expand: + ambiguous.append(i) + + to_expand = [j for j, grandchild in enumerate(child.children) if _is_ambig_tree(grandchild)] + child.expand_kids_by_index(*to_expand) + + if not ambiguous: + return self.node_builder(children) + + expand = [ iter(child.children) if i in ambiguous else repeat(child) for i, child in enumerate(children) ] + return self.tree_class('_ambig', [self.node_builder(list(f[0])) for f in product(zip(*expand))]) + +def maybe_create_ambiguous_expander(tree_class, expansion, keep_all_tokens): + to_expand = [i for i, sym in enumerate(expansion) + if keep_all_tokens or ((not (sym.is_term and sym.filter_out)) and _should_expand(sym))] + if to_expand: + return partial(AmbiguousExpander, to_expand, tree_class) + +def ptb_inline_args(func): + @wraps(func) + def f(children): + return func(*children) + return f + +def inplace_transformer(func): + @wraps(func) + def f(children): + # function name in a Transformer is a rule name. + tree = Tree(func.__name__, children) + return func(tree) + return f + +def apply_visit_wrapper(func, name, wrapper): + if wrapper is _vargs_meta or wrapper is _vargs_meta_inline: + raise NotImplementedError("Meta args not supported for internal transformer") + @wraps(func) + def f(children): + return wrapper(func, name, children, None) + return f + + +class ParseTreeBuilder: + def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False): + self.tree_class = tree_class + self.propagate_positions = propagate_positions + self.always_keep_all_tokens = keep_all_tokens + self.ambiguous = ambiguous + self.maybe_placeholders = maybe_placeholders + + self.rule_builders = list(self._init_builders(rules)) + + def _init_builders(self, rules): + for rule in rules: + options = rule.options + keep_all_tokens = self.always_keep_all_tokens or options.keep_all_tokens + expand_single_child = options.expand1 + + wrapper_chain = list(filter(None, [ + (expand_single_child and not rule.alias) and ExpandSingleChild, + maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders else None), + self.propagate_positions and PropagatePositions, + self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens), + ])) + + yield rule, wrapper_chain + + + def create_callback(self, transformer=None): + callbacks = {} + + for rule, wrapper_chain in self.rule_builders: + + user_callback_name = rule.alias or rule.options.template_source or rule.origin.name + try: + f = getattr(transformer, user_callback_name) + # XXX InlineTransformer is deprecated! + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + f = apply_visit_wrapper(f, user_callback_name, wrapper) + else: + if isinstance(transformer, InlineTransformer): + f = ptb_inline_args(f) + elif isinstance(transformer, Transformer_InPlace): + f = inplace_transformer(f) + except AttributeError: + f = partial(self.tree_class, user_callback_name) + + for w in wrapper_chain: + f = w(f) + + if rule in callbacks: + raise GrammarError("Rule '%s' already exists" % (rule,)) + + callbacks[rule] = f + + return callbacks + +###} diff --git a/poetry/core/_vendor/lark/parser_frontends.py b/poetry/core/_vendor/lark/parser_frontends.py new file mode 100644 index 000000000..c453ab67a --- /dev/null +++ b/poetry/core/_vendor/lark/parser_frontends.py @@ -0,0 +1,233 @@ +from functools import partial + +from .utils import get_regexp_width, Serialize +from .parsers.grammar_analysis import GrammarAnalyzer +from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token +from .parsers import earley, xearley, cyk +from .parsers.lalr_parser import LALR_Parser +from .grammar import Rule +from .tree import Tree +from .common import LexerConf + +###{standalone + +def get_frontend(parser, lexer): + if parser=='lalr': + if lexer is None: + raise ValueError('The LALR parser requires use of a lexer') + elif lexer == 'standard': + return LALR_TraditionalLexer + elif lexer == 'contextual': + return LALR_ContextualLexer + elif issubclass(lexer, Lexer): + return partial(LALR_CustomLexer, lexer) + else: + raise ValueError('Unknown lexer: %s' % lexer) + elif parser=='earley': + if lexer=='standard': + return Earley + elif lexer=='dynamic': + return XEarley + elif lexer=='dynamic_complete': + return XEarley_CompleteLex + elif lexer=='contextual': + raise ValueError('The Earley parser does not support the contextual parser') + else: + raise ValueError('Unknown lexer: %s' % lexer) + elif parser == 'cyk': + if lexer == 'standard': + return CYK + else: + raise ValueError('CYK parser requires using standard parser.') + else: + raise ValueError('Unknown parser: %s' % parser) + + +class _ParserFrontend(Serialize): + def _parse(self, input, start, *args): + if start is None: + start = self.start + if len(start) > 1: + raise ValueError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start) + start ,= start + return self.parser.parse(input, start, *args) + + +class WithLexer(_ParserFrontend): + lexer = None + parser = None + lexer_conf = None + start = None + + __serialize_fields__ = 'parser', 'lexer_conf', 'start' + __serialize_namespace__ = LexerConf, + + def __init__(self, lexer_conf, parser_conf, re_, options=None): + self.lexer_conf = lexer_conf + self.start = parser_conf.start + self.postlex = lexer_conf.postlex + self.re = re_ + + @classmethod + def deserialize(cls, data, memo, callbacks, postlex, re_): + inst = super(WithLexer, cls).deserialize(data, memo) + inst.re = re_ + inst.postlex = postlex + inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks) + inst.init_lexer() + return inst + + def _serialize(self, data, memo): + data['parser'] = data['parser'].serialize(memo) + + def lex(self, *args): + stream = self.lexer.lex(*args) + return self.postlex.process(stream) if self.postlex else stream + + def parse(self, text, start=None): + token_stream = self.lex(text) + return self._parse(token_stream, start) + + def init_traditional_lexer(self): + self.lexer = TraditionalLexer(self.lexer_conf.tokens, re_=self.re, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, g_regex_flags=self.lexer_conf.g_regex_flags) + +class LALR_WithLexer(WithLexer): + def __init__(self, lexer_conf, parser_conf, re_, options=None): + debug = options.debug if options else False + self.re = re_ + self.parser = LALR_Parser(parser_conf, debug=debug) + WithLexer.__init__(self, lexer_conf, parser_conf, re_, options) + + self.init_lexer() + + def init_lexer(self): + raise NotImplementedError() + +class LALR_TraditionalLexer(LALR_WithLexer): + def init_lexer(self): + self.init_traditional_lexer() + +class LALR_ContextualLexer(LALR_WithLexer): + def init_lexer(self): + states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()} + always_accept = self.postlex.always_accept if self.postlex else () + self.lexer = ContextualLexer(self.lexer_conf.tokens, states, + re_=self.re, + ignore=self.lexer_conf.ignore, + always_accept=always_accept, + user_callbacks=self.lexer_conf.callbacks, + g_regex_flags=self.lexer_conf.g_regex_flags) + + + def parse(self, text, start=None): + parser_state = [None] + def set_parser_state(s): + parser_state[0] = s + + token_stream = self.lex(text, lambda: parser_state[0]) + return self._parse(token_stream, start, set_parser_state) +###} + +class LALR_CustomLexer(LALR_WithLexer): + def __init__(self, lexer_cls, lexer_conf, parser_conf, re_, options=None): + self.lexer = lexer_cls(lexer_conf, re_=re_) + debug = options.debug if options else False + self.parser = LALR_Parser(parser_conf, debug=debug) + WithLexer.__init__(self, lexer_conf, parser_conf, re_, options) + + +def tokenize_text(text): + line = 1 + col_start_pos = 0 + for i, ch in enumerate(text): + if '\n' in ch: + line += ch.count('\n') + col_start_pos = i + ch.rindex('\n') + yield Token('CHAR', ch, line=line, column=i - col_start_pos) + +class Earley(WithLexer): + def __init__(self, lexer_conf, parser_conf, re_, options=None): + WithLexer.__init__(self, lexer_conf, parser_conf, re_, options) + self.init_traditional_lexer() + + resolve_ambiguity = options.ambiguity == 'resolve' + debug = options.debug if options else False + self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity, debug=debug) + + def match(self, term, token): + return term.name == token.type + + +class XEarley(_ParserFrontend): + def __init__(self, lexer_conf, parser_conf, re_, options=None, **kw): + self.re = re_ + + self.token_by_name = {t.name:t for t in lexer_conf.tokens} + self.start = parser_conf.start + + self._prepare_match(lexer_conf) + resolve_ambiguity = options.ambiguity == 'resolve' + debug = options.debug if options else False + self.parser = xearley.Parser(parser_conf, + self.match, + ignore=lexer_conf.ignore, + resolve_ambiguity=resolve_ambiguity, + debug=debug, + **kw + ) + + def match(self, term, text, index=0): + return self.regexps[term.name].match(text, index) + + def _prepare_match(self, lexer_conf): + self.regexps = {} + for t in lexer_conf.tokens: + if t.priority != 1: + raise ValueError("Dynamic Earley doesn't support weights on terminals", t, t.priority) + regexp = t.pattern.to_regexp() + try: + width = get_regexp_width(regexp)[0] + except ValueError: + raise ValueError("Bad regexp in token %s: %s" % (t.name, regexp)) + else: + if width == 0: + raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t) + + self.regexps[t.name] = self.re.compile(regexp, lexer_conf.g_regex_flags) + + def parse(self, text, start): + return self._parse(text, start) + +class XEarley_CompleteLex(XEarley): + def __init__(self, *args, **kw): + XEarley.__init__(self, *args, complete_lex=True, **kw) + + + +class CYK(WithLexer): + + def __init__(self, lexer_conf, parser_conf, re_, options=None): + WithLexer.__init__(self, lexer_conf, parser_conf, re_, options) + self.init_traditional_lexer() + + self._analysis = GrammarAnalyzer(parser_conf) + self.parser = cyk.Parser(parser_conf.rules) + + self.callbacks = parser_conf.callbacks + + def parse(self, text, start): + tokens = list(self.lex(text)) + parse = self._parse(tokens, start) + parse = self._transform(parse) + return parse + + def _transform(self, tree): + subtrees = list(tree.iter_subtrees()) + for subtree in subtrees: + subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children] + + return self._apply_callback(tree) + + def _apply_callback(self, tree): + return self.callbacks[tree.rule](tree.children) + diff --git a/poetry/core/_vendor/lark/parsers/__init__.py b/poetry/core/_vendor/lark/parsers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poetry/core/_vendor/lark/parsers/cyk.py b/poetry/core/_vendor/lark/parsers/cyk.py new file mode 100644 index 000000000..ff0924f24 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/cyk.py @@ -0,0 +1,345 @@ +"""This module implements a CYK parser.""" + +# Author: https://github.com/ehudt (2018) +# +# Adapted by Erez + + +from collections import defaultdict +import itertools + +from ..exceptions import ParseError +from ..lexer import Token +from ..tree import Tree +from ..grammar import Terminal as T, NonTerminal as NT, Symbol + +try: + xrange +except NameError: + xrange = range + +def match(t, s): + assert isinstance(t, T) + return t.name == s.type + + +class Rule(object): + """Context-free grammar rule.""" + + def __init__(self, lhs, rhs, weight, alias): + super(Rule, self).__init__() + assert isinstance(lhs, NT), lhs + assert all(isinstance(x, NT) or isinstance(x, T) for x in rhs), rhs + self.lhs = lhs + self.rhs = rhs + self.weight = weight + self.alias = alias + + def __str__(self): + return '%s -> %s' % (str(self.lhs), ' '.join(str(x) for x in self.rhs)) + + def __repr__(self): + return str(self) + + def __hash__(self): + return hash((self.lhs, tuple(self.rhs))) + + def __eq__(self, other): + return self.lhs == other.lhs and self.rhs == other.rhs + + def __ne__(self, other): + return not (self == other) + + +class Grammar(object): + """Context-free grammar.""" + + def __init__(self, rules): + self.rules = frozenset(rules) + + def __eq__(self, other): + return self.rules == other.rules + + def __str__(self): + return '\n' + '\n'.join(sorted(repr(x) for x in self.rules)) + '\n' + + def __repr__(self): + return str(self) + + +# Parse tree data structures +class RuleNode(object): + """A node in the parse tree, which also contains the full rhs rule.""" + + def __init__(self, rule, children, weight=0): + self.rule = rule + self.children = children + self.weight = weight + + def __repr__(self): + return 'RuleNode(%s, [%s])' % (repr(self.rule.lhs), ', '.join(str(x) for x in self.children)) + + + +class Parser(object): + """Parser wrapper.""" + + def __init__(self, rules): + super(Parser, self).__init__() + self.orig_rules = {rule: rule for rule in rules} + rules = [self._to_rule(rule) for rule in rules] + self.grammar = to_cnf(Grammar(rules)) + + def _to_rule(self, lark_rule): + """Converts a lark rule, (lhs, rhs, callback, options), to a Rule.""" + assert isinstance(lark_rule.origin, NT) + assert all(isinstance(x, Symbol) for x in lark_rule.expansion) + return Rule( + lark_rule.origin, lark_rule.expansion, + weight=lark_rule.options.priority if lark_rule.options.priority else 0, + alias=lark_rule) + + def parse(self, tokenized, start): # pylint: disable=invalid-name + """Parses input, which is a list of tokens.""" + assert start + start = NT(start) + + table, trees = _parse(tokenized, self.grammar) + # Check if the parse succeeded. + if all(r.lhs != start for r in table[(0, len(tokenized) - 1)]): + raise ParseError('Parsing failed.') + parse = trees[(0, len(tokenized) - 1)][start] + return self._to_tree(revert_cnf(parse)) + + def _to_tree(self, rule_node): + """Converts a RuleNode parse tree to a lark Tree.""" + orig_rule = self.orig_rules[rule_node.rule.alias] + children = [] + for child in rule_node.children: + if isinstance(child, RuleNode): + children.append(self._to_tree(child)) + else: + assert isinstance(child.name, Token) + children.append(child.name) + t = Tree(orig_rule.origin, children) + t.rule=orig_rule + return t + + +def print_parse(node, indent=0): + if isinstance(node, RuleNode): + print(' ' * (indent * 2) + str(node.rule.lhs)) + for child in node.children: + print_parse(child, indent + 1) + else: + print(' ' * (indent * 2) + str(node.s)) + + +def _parse(s, g): + """Parses sentence 's' using CNF grammar 'g'.""" + # The CYK table. Indexed with a 2-tuple: (start pos, end pos) + table = defaultdict(set) + # Top-level structure is similar to the CYK table. Each cell is a dict from + # rule name to the best (lightest) tree for that rule. + trees = defaultdict(dict) + # Populate base case with existing terminal production rules + for i, w in enumerate(s): + for terminal, rules in g.terminal_rules.items(): + if match(terminal, w): + for rule in rules: + table[(i, i)].add(rule) + if (rule.lhs not in trees[(i, i)] or + rule.weight < trees[(i, i)][rule.lhs].weight): + trees[(i, i)][rule.lhs] = RuleNode(rule, [T(w)], weight=rule.weight) + + # Iterate over lengths of sub-sentences + for l in xrange(2, len(s) + 1): + # Iterate over sub-sentences with the given length + for i in xrange(len(s) - l + 1): + # Choose partition of the sub-sentence in [1, l) + for p in xrange(i + 1, i + l): + span1 = (i, p - 1) + span2 = (p, i + l - 1) + for r1, r2 in itertools.product(table[span1], table[span2]): + for rule in g.nonterminal_rules.get((r1.lhs, r2.lhs), []): + table[(i, i + l - 1)].add(rule) + r1_tree = trees[span1][r1.lhs] + r2_tree = trees[span2][r2.lhs] + rule_total_weight = rule.weight + r1_tree.weight + r2_tree.weight + if (rule.lhs not in trees[(i, i + l - 1)] + or rule_total_weight < trees[(i, i + l - 1)][rule.lhs].weight): + trees[(i, i + l - 1)][rule.lhs] = RuleNode(rule, [r1_tree, r2_tree], weight=rule_total_weight) + return table, trees + + +# This section implements context-free grammar converter to Chomsky normal form. +# It also implements a conversion of parse trees from its CNF to the original +# grammar. +# Overview: +# Applies the following operations in this order: +# * TERM: Eliminates non-solitary terminals from all rules +# * BIN: Eliminates rules with more than 2 symbols on their right-hand-side. +# * UNIT: Eliminates non-terminal unit rules +# +# The following grammar characteristics aren't featured: +# * Start symbol appears on RHS +# * Empty rules (epsilon rules) + + +class CnfWrapper(object): + """CNF wrapper for grammar. + + Validates that the input grammar is CNF and provides helper data structures. + """ + + def __init__(self, grammar): + super(CnfWrapper, self).__init__() + self.grammar = grammar + self.rules = grammar.rules + self.terminal_rules = defaultdict(list) + self.nonterminal_rules = defaultdict(list) + for r in self.rules: + # Validate that the grammar is CNF and populate auxiliary data structures. + assert isinstance(r.lhs, NT), r + if len(r.rhs) not in [1, 2]: + raise ParseError("CYK doesn't support empty rules") + if len(r.rhs) == 1 and isinstance(r.rhs[0], T): + self.terminal_rules[r.rhs[0]].append(r) + elif len(r.rhs) == 2 and all(isinstance(x, NT) for x in r.rhs): + self.nonterminal_rules[tuple(r.rhs)].append(r) + else: + assert False, r + + def __eq__(self, other): + return self.grammar == other.grammar + + def __repr__(self): + return repr(self.grammar) + + +class UnitSkipRule(Rule): + """A rule that records NTs that were skipped during transformation.""" + + def __init__(self, lhs, rhs, skipped_rules, weight, alias): + super(UnitSkipRule, self).__init__(lhs, rhs, weight, alias) + self.skipped_rules = skipped_rules + + def __eq__(self, other): + return isinstance(other, type(self)) and self.skipped_rules == other.skipped_rules + + __hash__ = Rule.__hash__ + + +def build_unit_skiprule(unit_rule, target_rule): + skipped_rules = [] + if isinstance(unit_rule, UnitSkipRule): + skipped_rules += unit_rule.skipped_rules + skipped_rules.append(target_rule) + if isinstance(target_rule, UnitSkipRule): + skipped_rules += target_rule.skipped_rules + return UnitSkipRule(unit_rule.lhs, target_rule.rhs, skipped_rules, + weight=unit_rule.weight + target_rule.weight, alias=unit_rule.alias) + + +def get_any_nt_unit_rule(g): + """Returns a non-terminal unit rule from 'g', or None if there is none.""" + for rule in g.rules: + if len(rule.rhs) == 1 and isinstance(rule.rhs[0], NT): + return rule + return None + + +def _remove_unit_rule(g, rule): + """Removes 'rule' from 'g' without changing the langugage produced by 'g'.""" + new_rules = [x for x in g.rules if x != rule] + refs = [x for x in g.rules if x.lhs == rule.rhs[0]] + new_rules += [build_unit_skiprule(rule, ref) for ref in refs] + return Grammar(new_rules) + + +def _split(rule): + """Splits a rule whose len(rhs) > 2 into shorter rules.""" + rule_str = str(rule.lhs) + '__' + '_'.join(str(x) for x in rule.rhs) + rule_name = '__SP_%s' % (rule_str) + '_%d' + yield Rule(rule.lhs, [rule.rhs[0], NT(rule_name % 1)], weight=rule.weight, alias=rule.alias) + for i in xrange(1, len(rule.rhs) - 2): + yield Rule(NT(rule_name % i), [rule.rhs[i], NT(rule_name % (i + 1))], weight=0, alias='Split') + yield Rule(NT(rule_name % (len(rule.rhs) - 2)), rule.rhs[-2:], weight=0, alias='Split') + + +def _term(g): + """Applies the TERM rule on 'g' (see top comment).""" + all_t = {x for rule in g.rules for x in rule.rhs if isinstance(x, T)} + t_rules = {t: Rule(NT('__T_%s' % str(t)), [t], weight=0, alias='Term') for t in all_t} + new_rules = [] + for rule in g.rules: + if len(rule.rhs) > 1 and any(isinstance(x, T) for x in rule.rhs): + new_rhs = [t_rules[x].lhs if isinstance(x, T) else x for x in rule.rhs] + new_rules.append(Rule(rule.lhs, new_rhs, weight=rule.weight, alias=rule.alias)) + new_rules.extend(v for k, v in t_rules.items() if k in rule.rhs) + else: + new_rules.append(rule) + return Grammar(new_rules) + + +def _bin(g): + """Applies the BIN rule to 'g' (see top comment).""" + new_rules = [] + for rule in g.rules: + if len(rule.rhs) > 2: + new_rules += _split(rule) + else: + new_rules.append(rule) + return Grammar(new_rules) + + +def _unit(g): + """Applies the UNIT rule to 'g' (see top comment).""" + nt_unit_rule = get_any_nt_unit_rule(g) + while nt_unit_rule: + g = _remove_unit_rule(g, nt_unit_rule) + nt_unit_rule = get_any_nt_unit_rule(g) + return g + + +def to_cnf(g): + """Creates a CNF grammar from a general context-free grammar 'g'.""" + g = _unit(_bin(_term(g))) + return CnfWrapper(g) + + +def unroll_unit_skiprule(lhs, orig_rhs, skipped_rules, children, weight, alias): + if not skipped_rules: + return RuleNode(Rule(lhs, orig_rhs, weight=weight, alias=alias), children, weight=weight) + else: + weight = weight - skipped_rules[0].weight + return RuleNode( + Rule(lhs, [skipped_rules[0].lhs], weight=weight, alias=alias), [ + unroll_unit_skiprule(skipped_rules[0].lhs, orig_rhs, + skipped_rules[1:], children, + skipped_rules[0].weight, skipped_rules[0].alias) + ], weight=weight) + + +def revert_cnf(node): + """Reverts a parse tree (RuleNode) to its original non-CNF form (Node).""" + if isinstance(node, T): + return node + # Reverts TERM rule. + if node.rule.lhs.name.startswith('__T_'): + return node.children[0] + else: + children = [] + for child in map(revert_cnf, node.children): + # Reverts BIN rule. + if isinstance(child, RuleNode) and child.rule.lhs.name.startswith('__SP_'): + children += child.children + else: + children.append(child) + # Reverts UNIT rule. + if isinstance(node.rule, UnitSkipRule): + return unroll_unit_skiprule(node.rule.lhs, node.rule.rhs, + node.rule.skipped_rules, children, + node.rule.weight, node.rule.alias) + else: + return RuleNode(node.rule, children) diff --git a/poetry/core/_vendor/lark/parsers/earley.py b/poetry/core/_vendor/lark/parsers/earley.py new file mode 100644 index 000000000..59e9a06a7 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/earley.py @@ -0,0 +1,328 @@ +"""This module implements an scanerless Earley parser. + +The core Earley algorithm used here is based on Elizabeth Scott's implementation, here: + https://www.sciencedirect.com/science/article/pii/S1571066108001497 + +That is probably the best reference for understanding the algorithm here. + +The Earley parser outputs an SPPF-tree as per that document. The SPPF tree format +is better documented here: + http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/ +""" + +import logging +from collections import deque + +from ..visitors import Transformer_InPlace, v_args +from ..exceptions import UnexpectedEOF, UnexpectedToken +from .grammar_analysis import GrammarAnalyzer +from ..grammar import NonTerminal +from .earley_common import Item, TransitiveItem +from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode, ForestToAmbiguousTreeVisitor + +class Parser: + def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False): + analysis = GrammarAnalyzer(parser_conf) + self.parser_conf = parser_conf + self.resolve_ambiguity = resolve_ambiguity + self.debug = debug + + self.FIRST = analysis.FIRST + self.NULLABLE = analysis.NULLABLE + self.callbacks = parser_conf.callbacks + self.predictions = {} + + ## These could be moved to the grammar analyzer. Pre-computing these is *much* faster than + # the slow 'isupper' in is_terminal. + self.TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if sym.is_term } + self.NON_TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if not sym.is_term } + + self.forest_sum_visitor = None + for rule in parser_conf.rules: + if rule.origin not in self.predictions: + self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)] + + ## Detect if any rules have priorities set. If the user specified priority = "none" then + # the priorities will be stripped from all rules before they reach us, allowing us to + # skip the extra tree walk. We'll also skip this if the user just didn't specify priorities + # on any rules. + if self.forest_sum_visitor is None and rule.options.priority is not None: + self.forest_sum_visitor = ForestSumVisitor + + self.term_matcher = term_matcher + + + def predict_and_complete(self, i, to_scan, columns, transitives): + """The core Earley Predictor and Completer. + + At each stage of the input, we handling any completed items (things + that matched on the last cycle) and use those to predict what should + come next in the input stream. The completions and any predicted + non-terminals are recursively processed until we reach a set of, + which can be added to the scan list for the next scanner cycle.""" + # Held Completions (H in E.Scotts paper). + node_cache = {} + held_completions = {} + + column = columns[i] + # R (items) = Ei (column.items) + items = deque(column) + while items: + item = items.pop() # remove an element, A say, from R + + ### The Earley completer + if item.is_complete: ### (item.s == string) + if item.node is None: + label = (item.s, item.start, i) + item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + item.node.add_family(item.s, item.rule, item.start, None, None) + + # create_leo_transitives(item.rule.origin, item.start) + + ###R Joop Leo right recursion Completer + if item.rule.origin in transitives[item.start]: + transitive = transitives[item.start][item.s] + if transitive.previous in transitives[transitive.column]: + root_transitive = transitives[transitive.column][transitive.previous] + else: + root_transitive = transitive + + new_item = Item(transitive.rule, transitive.ptr, transitive.start) + label = (root_transitive.s, root_transitive.start, i) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + new_item.node.add_path(root_transitive, item.node) + if new_item.expect in self.TERMINALS: + # Add (B :: aC.B, h, y) to Q + to_scan.add(new_item) + elif new_item not in column: + # Add (B :: aC.B, h, y) to Ei and R + column.add(new_item) + items.append(new_item) + ###R Regular Earley completer + else: + # Empty has 0 length. If we complete an empty symbol in a particular + # parse step, we need to be able to use that same empty symbol to complete + # any predictions that result, that themselves require empty. Avoids + # infinite recursion on empty symbols. + # held_completions is 'H' in E.Scott's paper. + is_empty_item = item.start == i + if is_empty_item: + held_completions[item.rule.origin] = item.node + + originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s] + for originator in originators: + new_item = originator.advance() + label = (new_item.s, originator.start, i) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + new_item.node.add_family(new_item.s, new_item.rule, i, originator.node, item.node) + if new_item.expect in self.TERMINALS: + # Add (B :: aC.B, h, y) to Q + to_scan.add(new_item) + elif new_item not in column: + # Add (B :: aC.B, h, y) to Ei and R + column.add(new_item) + items.append(new_item) + + ### The Earley predictor + elif item.expect in self.NON_TERMINALS: ### (item.s == lr0) + new_items = [] + for rule in self.predictions[item.expect]: + new_item = Item(rule, 0, i) + new_items.append(new_item) + + # Process any held completions (H). + if item.expect in held_completions: + new_item = item.advance() + label = (new_item.s, item.start, i) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect]) + new_items.append(new_item) + + for new_item in new_items: + if new_item.expect in self.TERMINALS: + to_scan.add(new_item) + elif new_item not in column: + column.add(new_item) + items.append(new_item) + + def _parse(self, stream, columns, to_scan, start_symbol=None): + def is_quasi_complete(item): + if item.is_complete: + return True + + quasi = item.advance() + while not quasi.is_complete: + if quasi.expect not in self.NULLABLE: + return False + if quasi.rule.origin == start_symbol and quasi.expect == start_symbol: + return False + quasi = quasi.advance() + return True + + def create_leo_transitives(origin, start): + visited = set() + to_create = [] + trule = None + previous = None + + ### Recursively walk backwards through the Earley sets until we find the + # first transitive candidate. If this is done continuously, we shouldn't + # have to walk more than 1 hop. + while True: + if origin in transitives[start]: + previous = trule = transitives[start][origin] + break + + is_empty_rule = not self.FIRST[origin] + if is_empty_rule: + break + + candidates = [ candidate for candidate in columns[start] if candidate.expect is not None and origin == candidate.expect ] + if len(candidates) != 1: + break + originator = next(iter(candidates)) + + if originator is None or originator in visited: + break + + visited.add(originator) + if not is_quasi_complete(originator): + break + + trule = originator.advance() + if originator.start != start: + visited.clear() + + to_create.append((origin, start, originator)) + origin = originator.rule.origin + start = originator.start + + # If a suitable Transitive candidate is not found, bail. + if trule is None: + return + + #### Now walk forwards and create Transitive Items in each set we walked through; and link + # each transitive item to the next set forwards. + while to_create: + origin, start, originator = to_create.pop() + titem = None + if previous is not None: + titem = previous.next_titem = TransitiveItem(origin, trule, originator, previous.column) + else: + titem = TransitiveItem(origin, trule, originator, start) + previous = transitives[start][origin] = titem + + + + def scan(i, token, to_scan): + """The core Earley Scanner. + + This is a custom implementation of the scanner that uses the + Lark lexer to match tokens. The scan list is built by the + Earley predictor, based on the previously completed tokens. + This ensures that at each phase of the parse we have a custom + lexer context, allowing for more complex ambiguities.""" + next_to_scan = set() + next_set = set() + columns.append(next_set) + transitives.append({}) + node_cache = {} + + for item in set(to_scan): + if match(item.expect, token): + new_item = item.advance() + label = (new_item.s, new_item.start, i) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token) + + if new_item.expect in self.TERMINALS: + # add (B ::= Aai+1.B, h, y) to Q' + next_to_scan.add(new_item) + else: + # add (B ::= Aa+1.B, h, y) to Ei+1 + next_set.add(new_item) + + if not next_set and not next_to_scan: + expect = {i.expect.name for i in to_scan} + raise UnexpectedToken(token, expect, considered_rules = set(to_scan)) + + return next_to_scan + + + # Define parser functions + match = self.term_matcher + + # Cache for nodes & tokens created in a particular parse step. + transitives = [{}] + + ## The main Earley loop. + # Run the Prediction/Completion cycle for any Items in the current Earley set. + # Completions will be added to the SPPF tree, and predictions will be recursively + # processed down to terminals/empty nodes to be added to the scanner for the next + # step. + i = 0 + for token in stream: + self.predict_and_complete(i, to_scan, columns, transitives) + + to_scan = scan(i, token, to_scan) + i += 1 + + self.predict_and_complete(i, to_scan, columns, transitives) + + ## Column is now the final column in the parse. + assert i == len(columns)-1 + return to_scan + + def parse(self, stream, start): + assert start, start + start_symbol = NonTerminal(start) + + columns = [set()] + to_scan = set() # The scan buffer. 'Q' in E.Scott's paper. + + ## Predict for the start_symbol. + # Add predicted items to the first Earley set (for the predictor) if they + # result in a non-terminal, or the scanner if they result in a terminal. + for rule in self.predictions[start_symbol]: + item = Item(rule, 0, 0) + if item.expect in self.TERMINALS: + to_scan.add(item) + else: + columns[0].add(item) + + to_scan = self._parse(stream, columns, to_scan, start_symbol) + + # If the parse was successful, the start + # symbol should have been completed in the last step of the Earley cycle, and will be in + # this column. Find the item for the start_symbol, which is the root of the SPPF tree. + solutions = [n.node for n in columns[-1] if n.is_complete and n.node is not None and n.s == start_symbol and n.start == 0] + if self.debug: + from .earley_forest import ForestToPyDotVisitor + try: + debug_walker = ForestToPyDotVisitor() + except ImportError: + logging.warning("Cannot find dependency 'pydot', will not generate sppf debug image") + else: + debug_walker.visit(solutions[0], "sppf.png") + + + if not solutions: + expected_tokens = [t.expect for t in to_scan] + raise UnexpectedEOF(expected_tokens) + elif len(solutions) > 1: + assert False, 'Earley should not generate multiple start symbol items!' + + # Perform our SPPF -> AST conversion using the right ForestVisitor. + forest_tree_visitor_cls = ForestToTreeVisitor if self.resolve_ambiguity else ForestToAmbiguousTreeVisitor + forest_tree_visitor = forest_tree_visitor_cls(self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor()) + + return forest_tree_visitor.visit(solutions[0]) + + +class ApplyCallbacks(Transformer_InPlace): + def __init__(self, postprocess): + self.postprocess = postprocess + + @v_args(meta=True) + def drv(self, children, meta): + return self.postprocess[meta.rule](children) diff --git a/poetry/core/_vendor/lark/parsers/earley_common.py b/poetry/core/_vendor/lark/parsers/earley_common.py new file mode 100644 index 000000000..6bd614bad --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/earley_common.py @@ -0,0 +1,75 @@ +"This module implements an Earley Parser" + +# The parser uses a parse-forest to keep track of derivations and ambiguations. +# When the parse ends successfully, a disambiguation stage resolves all ambiguity +# (right now ambiguity resolution is not developed beyond the needs of lark) +# Afterwards the parse tree is reduced (transformed) according to user callbacks. +# I use the no-recursion version of Transformer, because the tree might be +# deeper than Python's recursion limit (a bit absurd, but that's life) +# +# The algorithm keeps track of each state set, using a corresponding Column instance. +# Column keeps track of new items using NewsList instances. +# +# Author: Erez Shinan (2017) +# Email : erezshin@gmail.com + +from ..grammar import NonTerminal, Terminal + +class Item(object): + "An Earley Item, the atom of the algorithm." + + __slots__ = ('s', 'rule', 'ptr', 'start', 'is_complete', 'expect', 'previous', 'node', '_hash') + def __init__(self, rule, ptr, start): + self.is_complete = len(rule.expansion) == ptr + self.rule = rule # rule + self.ptr = ptr # ptr + self.start = start # j + self.node = None # w + if self.is_complete: + self.s = rule.origin + self.expect = None + self.previous = rule.expansion[ptr - 1] if ptr > 0 and len(rule.expansion) else None + else: + self.s = (rule, ptr) + self.expect = rule.expansion[ptr] + self.previous = rule.expansion[ptr - 1] if ptr > 0 and len(rule.expansion) else None + self._hash = hash((self.s, self.start)) + + def advance(self): + return Item(self.rule, self.ptr + 1, self.start) + + def __eq__(self, other): + return self is other or (self.s == other.s and self.start == other.start) + + def __hash__(self): + return self._hash + + def __repr__(self): + before = ( expansion.name for expansion in self.rule.expansion[:self.ptr] ) + after = ( expansion.name for expansion in self.rule.expansion[self.ptr:] ) + symbol = "{} ::= {}* {}".format(self.rule.origin.name, ' '.join(before), ' '.join(after)) + return '%s (%d)' % (symbol, self.start) + + +class TransitiveItem(Item): + __slots__ = ('recognized', 'reduction', 'column', 'next_titem') + def __init__(self, recognized, trule, originator, start): + super(TransitiveItem, self).__init__(trule.rule, trule.ptr, trule.start) + self.recognized = recognized + self.reduction = originator + self.column = start + self.next_titem = None + self._hash = hash((self.s, self.start, self.recognized)) + + def __eq__(self, other): + if not isinstance(other, TransitiveItem): + return False + return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.recognized == other.recognized) + + def __hash__(self): + return self._hash + + def __repr__(self): + before = ( expansion.name for expansion in self.rule.expansion[:self.ptr] ) + after = ( expansion.name for expansion in self.rule.expansion[self.ptr:] ) + return '{} : {} -> {}* {} ({}, {})'.format(self.recognized.name, self.rule.origin.name, ' '.join(before), ' '.join(after), self.column, self.start) diff --git a/poetry/core/_vendor/lark/parsers/earley_forest.py b/poetry/core/_vendor/lark/parsers/earley_forest.py new file mode 100644 index 000000000..c8b4f2531 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/earley_forest.py @@ -0,0 +1,430 @@ +""""This module implements an SPPF implementation + +This is used as the primary output mechanism for the Earley parser +in order to store complex ambiguities. + +Full reference and more details is here: +http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/ +""" + +from random import randint +from math import isinf +from collections import deque +from operator import attrgetter +from importlib import import_module + +from ..tree import Tree +from ..exceptions import ParseError + +class ForestNode(object): + pass + +class SymbolNode(ForestNode): + """ + A Symbol Node represents a symbol (or Intermediate LR0). + + Symbol nodes are keyed by the symbol (s). For intermediate nodes + s will be an LR0, stored as a tuple of (rule, ptr). For completed symbol + nodes, s will be a string representing the non-terminal origin (i.e. + the left hand side of the rule). + + The children of a Symbol or Intermediate Node will always be Packed Nodes; + with each Packed Node child representing a single derivation of a production. + + Hence a Symbol Node with a single child is unambiguous. + """ + __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate', '_hash') + def __init__(self, s, start, end): + self.s = s + self.start = start + self.end = end + self._children = set() + self.paths = set() + self.paths_loaded = False + + ### We use inf here as it can be safely negated without resorting to conditionals, + # unlike None or float('NaN'), and sorts appropriately. + self.priority = float('-inf') + self.is_intermediate = isinstance(s, tuple) + self._hash = hash((self.s, self.start, self.end)) + + def add_family(self, lr0, rule, start, left, right): + self._children.add(PackedNode(self, lr0, rule, start, left, right)) + + def add_path(self, transitive, node): + self.paths.add((transitive, node)) + + def load_paths(self): + for transitive, node in self.paths: + if transitive.next_titem is not None: + vn = SymbolNode(transitive.next_titem.s, transitive.next_titem.start, self.end) + vn.add_path(transitive.next_titem, node) + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn) + else: + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node) + self.paths_loaded = True + + @property + def is_ambiguous(self): + return len(self.children) > 1 + + @property + def children(self): + if not self.paths_loaded: self.load_paths() + return sorted(self._children, key=attrgetter('sort_key')) + + def __iter__(self): + return iter(self._children) + + def __eq__(self, other): + if not isinstance(other, SymbolNode): + return False + return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.end is other.end) + + def __hash__(self): + return self._hash + + def __repr__(self): + if self.is_intermediate: + rule = self.s[0] + ptr = self.s[1] + before = ( expansion.name for expansion in rule.expansion[:ptr] ) + after = ( expansion.name for expansion in rule.expansion[ptr:] ) + symbol = "{} ::= {}* {}".format(rule.origin.name, ' '.join(before), ' '.join(after)) + else: + symbol = self.s.name + return "({}, {}, {}, {})".format(symbol, self.start, self.end, self.priority) + +class PackedNode(ForestNode): + """ + A Packed Node represents a single derivation in a symbol node. + """ + __slots__ = ('parent', 's', 'rule', 'start', 'left', 'right', 'priority', '_hash') + def __init__(self, parent, s, rule, start, left, right): + self.parent = parent + self.s = s + self.start = start + self.rule = rule + self.left = left + self.right = right + self.priority = float('-inf') + self._hash = hash((self.left, self.right)) + + @property + def is_empty(self): + return self.left is None and self.right is None + + @property + def sort_key(self): + """ + Used to sort PackedNode children of SymbolNodes. + A SymbolNode has multiple PackedNodes if it matched + ambiguously. Hence, we use the sort order to identify + the order in which ambiguous children should be considered. + """ + return self.is_empty, -self.priority, self.rule.order + + def __iter__(self): + return iter([self.left, self.right]) + + def __eq__(self, other): + if not isinstance(other, PackedNode): + return False + return self is other or (self.left == other.left and self.right == other.right) + + def __hash__(self): + return self._hash + + def __repr__(self): + if isinstance(self.s, tuple): + rule = self.s[0] + ptr = self.s[1] + before = ( expansion.name for expansion in rule.expansion[:ptr] ) + after = ( expansion.name for expansion in rule.expansion[ptr:] ) + symbol = "{} ::= {}* {}".format(rule.origin.name, ' '.join(before), ' '.join(after)) + else: + symbol = self.s.name + return "({}, {}, {}, {})".format(symbol, self.start, self.priority, self.rule.order) + +class ForestVisitor(object): + """ + An abstract base class for building forest visitors. + + Use this as a base when you need to walk the forest. + """ + __slots__ = ['result'] + + def visit_token_node(self, node): pass + def visit_symbol_node_in(self, node): pass + def visit_symbol_node_out(self, node): pass + def visit_packed_node_in(self, node): pass + def visit_packed_node_out(self, node): pass + + def visit(self, root): + self.result = None + # Visiting is a list of IDs of all symbol/intermediate nodes currently in + # the stack. It serves two purposes: to detect when we 'recurse' in and out + # of a symbol/intermediate so that we can process both up and down. Also, + # since the SPPF can have cycles it allows us to detect if we're trying + # to recurse into a node that's already on the stack (infinite recursion). + visiting = set() + + # We do not use recursion here to walk the Forest due to the limited + # stack size in python. Therefore input_stack is essentially our stack. + input_stack = deque([root]) + + # It is much faster to cache these as locals since they are called + # many times in large parses. + vpno = getattr(self, 'visit_packed_node_out') + vpni = getattr(self, 'visit_packed_node_in') + vsno = getattr(self, 'visit_symbol_node_out') + vsni = getattr(self, 'visit_symbol_node_in') + vtn = getattr(self, 'visit_token_node') + while input_stack: + current = next(reversed(input_stack)) + try: + next_node = next(current) + except StopIteration: + input_stack.pop() + continue + except TypeError: + ### If the current object is not an iterator, pass through to Token/SymbolNode + pass + else: + if next_node is None: + continue + + if id(next_node) in visiting: + raise ParseError("Infinite recursion in grammar, in rule '%s'!" % next_node.s.name) + + input_stack.append(next_node) + continue + + if not isinstance(current, ForestNode): + vtn(current) + input_stack.pop() + continue + + current_id = id(current) + if current_id in visiting: + if isinstance(current, PackedNode): vpno(current) + else: vsno(current) + input_stack.pop() + visiting.remove(current_id) + continue + else: + visiting.add(current_id) + if isinstance(current, PackedNode): next_node = vpni(current) + else: next_node = vsni(current) + if next_node is None: + continue + + if id(next_node) in visiting: + raise ParseError("Infinite recursion in grammar!") + + input_stack.append(next_node) + continue + + return self.result + +class ForestSumVisitor(ForestVisitor): + """ + A visitor for prioritizing ambiguous parts of the Forest. + + This visitor is used when support for explicit priorities on + rules is requested (whether normal, or invert). It walks the + forest (or subsets thereof) and cascades properties upwards + from the leaves. + + It would be ideal to do this during parsing, however this would + require processing each Earley item multiple times. That's + a big performance drawback; so running a forest walk is the + lesser of two evils: there can be significantly more Earley + items created during parsing than there are SPPF nodes in the + final tree. + """ + def visit_packed_node_in(self, node): + return iter([node.left, node.right]) + + def visit_symbol_node_in(self, node): + return iter(node.children) + + def visit_packed_node_out(self, node): + priority = node.rule.options.priority if not node.parent.is_intermediate and node.rule.options.priority else 0 + priority += getattr(node.right, 'priority', 0) + priority += getattr(node.left, 'priority', 0) + node.priority = priority + + def visit_symbol_node_out(self, node): + node.priority = max(child.priority for child in node.children) + +class ForestToTreeVisitor(ForestVisitor): + """ + A Forest visitor which converts an SPPF forest to an unambiguous AST. + + The implementation in this visitor walks only the first ambiguous child + of each symbol node. When it finds an ambiguous symbol node it first + calls the forest_sum_visitor implementation to sort the children + into preference order using the algorithms defined there; so the first + child should always be the highest preference. The forest_sum_visitor + implementation should be another ForestVisitor which sorts the children + according to some priority mechanism. + """ + __slots__ = ['forest_sum_visitor', 'callbacks', 'output_stack'] + def __init__(self, callbacks, forest_sum_visitor = None): + assert callbacks + self.forest_sum_visitor = forest_sum_visitor + self.callbacks = callbacks + + def visit(self, root): + self.output_stack = deque() + return super(ForestToTreeVisitor, self).visit(root) + + def visit_token_node(self, node): + self.output_stack[-1].append(node) + + def visit_symbol_node_in(self, node): + if self.forest_sum_visitor and node.is_ambiguous and isinf(node.priority): + self.forest_sum_visitor.visit(node) + return next(iter(node.children)) + + def visit_packed_node_in(self, node): + if not node.parent.is_intermediate: + self.output_stack.append([]) + return iter([node.left, node.right]) + + def visit_packed_node_out(self, node): + if not node.parent.is_intermediate: + result = self.callbacks[node.rule](self.output_stack.pop()) + if self.output_stack: + self.output_stack[-1].append(result) + else: + self.result = result + +class ForestToAmbiguousTreeVisitor(ForestToTreeVisitor): + """ + A Forest visitor which converts an SPPF forest to an ambiguous AST. + + Because of the fundamental disparity between what can be stored in + an SPPF and what can be stored in a Tree; this implementation is not + complete. It correctly deals with ambiguities that occur on symbol nodes only, + and cannot deal with ambiguities that occur on intermediate nodes. + + Usually, most parsers can be rewritten to avoid intermediate node + ambiguities. Also, this implementation could be fixed, however + the code to handle intermediate node ambiguities is messy and + would not be performant. It is much better not to use this and + instead to correctly disambiguate the forest and only store unambiguous + parses in Trees. It is here just to provide some parity with the + old ambiguity='explicit'. + + This is mainly used by the test framework, to make it simpler to write + tests ensuring the SPPF contains the right results. + """ + def __init__(self, callbacks, forest_sum_visitor = ForestSumVisitor): + super(ForestToAmbiguousTreeVisitor, self).__init__(callbacks, forest_sum_visitor) + + def visit_token_node(self, node): + self.output_stack[-1].children.append(node) + + def visit_symbol_node_in(self, node): + if self.forest_sum_visitor and node.is_ambiguous and isinf(node.priority): + self.forest_sum_visitor.visit(node) + if not node.is_intermediate and node.is_ambiguous: + self.output_stack.append(Tree('_ambig', [])) + return iter(node.children) + + def visit_symbol_node_out(self, node): + if not node.is_intermediate and node.is_ambiguous: + result = self.output_stack.pop() + if self.output_stack: + self.output_stack[-1].children.append(result) + else: + self.result = result + + def visit_packed_node_in(self, node): + if not node.parent.is_intermediate: + self.output_stack.append(Tree('drv', [])) + return iter([node.left, node.right]) + + def visit_packed_node_out(self, node): + if not node.parent.is_intermediate: + result = self.callbacks[node.rule](self.output_stack.pop().children) + if self.output_stack: + self.output_stack[-1].children.append(result) + else: + self.result = result + +class ForestToPyDotVisitor(ForestVisitor): + """ + A Forest visitor which writes the SPPF to a PNG. + + The SPPF can get really large, really quickly because + of the amount of meta-data it stores, so this is probably + only useful for trivial trees and learning how the SPPF + is structured. + """ + def __init__(self, rankdir="TB"): + self.pydot = import_module('pydot') + self.graph = self.pydot.Dot(graph_type='digraph', rankdir=rankdir) + + def visit(self, root, filename): + super(ForestToPyDotVisitor, self).visit(root) + self.graph.write_png(filename) + + def visit_token_node(self, node): + graph_node_id = str(id(node)) + graph_node_label = "\"{}\"".format(node.value.replace('"', '\\"')) + graph_node_color = 0x808080 + graph_node_style = "\"filled,rounded\"" + graph_node_shape = "diamond" + graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label) + self.graph.add_node(graph_node) + + def visit_packed_node_in(self, node): + graph_node_id = str(id(node)) + graph_node_label = repr(node) + graph_node_color = 0x808080 + graph_node_style = "filled" + graph_node_shape = "diamond" + graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label) + self.graph.add_node(graph_node) + return iter([node.left, node.right]) + + def visit_packed_node_out(self, node): + graph_node_id = str(id(node)) + graph_node = self.graph.get_node(graph_node_id)[0] + for child in [node.left, node.right]: + if child is not None: + child_graph_node_id = str(id(child)) + child_graph_node = self.graph.get_node(child_graph_node_id)[0] + self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node)) + else: + #### Try and be above the Python object ID range; probably impl. specific, but maybe this is okay. + child_graph_node_id = str(randint(100000000000000000000000000000,123456789012345678901234567890)) + child_graph_node_style = "invis" + child_graph_node = self.pydot.Node(child_graph_node_id, style=child_graph_node_style, label="None") + child_edge_style = "invis" + self.graph.add_node(child_graph_node) + self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node, style=child_edge_style)) + + def visit_symbol_node_in(self, node): + graph_node_id = str(id(node)) + graph_node_label = repr(node) + graph_node_color = 0x808080 + graph_node_style = "\"filled\"" + if node.is_intermediate: + graph_node_shape = "ellipse" + else: + graph_node_shape = "rectangle" + graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label) + self.graph.add_node(graph_node) + return iter(node.children) + + def visit_symbol_node_out(self, node): + graph_node_id = str(id(node)) + graph_node = self.graph.get_node(graph_node_id)[0] + for child in node.children: + child_graph_node_id = str(id(child)) + child_graph_node = self.graph.get_node(child_graph_node_id)[0] + self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node)) diff --git a/poetry/core/_vendor/lark/parsers/grammar_analysis.py b/poetry/core/_vendor/lark/parsers/grammar_analysis.py new file mode 100644 index 000000000..94c32ccc3 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/grammar_analysis.py @@ -0,0 +1,185 @@ +from collections import Counter, defaultdict + +from ..utils import bfs, fzset, classify +from ..exceptions import GrammarError +from ..grammar import Rule, Terminal, NonTerminal + + +class RulePtr(object): + __slots__ = ('rule', 'index') + + def __init__(self, rule, index): + assert isinstance(rule, Rule) + assert index <= len(rule.expansion) + self.rule = rule + self.index = index + + def __repr__(self): + before = [x.name for x in self.rule.expansion[:self.index]] + after = [x.name for x in self.rule.expansion[self.index:]] + return '<%s : %s * %s>' % (self.rule.origin.name, ' '.join(before), ' '.join(after)) + + @property + def next(self): + return self.rule.expansion[self.index] + + def advance(self, sym): + assert self.next == sym + return RulePtr(self.rule, self.index+1) + + @property + def is_satisfied(self): + return self.index == len(self.rule.expansion) + + def __eq__(self, other): + return self.rule == other.rule and self.index == other.index + def __hash__(self): + return hash((self.rule, self.index)) + + +# state generation ensures no duplicate LR0ItemSets +class LR0ItemSet(object): + __slots__ = ('kernel', 'closure', 'transitions', 'lookaheads') + + def __init__(self, kernel, closure): + self.kernel = fzset(kernel) + self.closure = fzset(closure) + self.transitions = {} + self.lookaheads = defaultdict(set) + + def __repr__(self): + return '{%s | %s}' % (', '.join([repr(r) for r in self.kernel]), ', '.join([repr(r) for r in self.closure])) + + +def update_set(set1, set2): + if not set2 or set1 > set2: + return False + + copy = set(set1) + set1 |= set2 + return set1 != copy + +def calculate_sets(rules): + """Calculate FOLLOW sets. + + Adapted from: http://lara.epfl.ch/w/cc09:algorithm_for_first_and_follow_sets""" + symbols = {sym for rule in rules for sym in rule.expansion} | {rule.origin for rule in rules} + + # foreach grammar rule X ::= Y(1) ... Y(k) + # if k=0 or {Y(1),...,Y(k)} subset of NULLABLE then + # NULLABLE = NULLABLE union {X} + # for i = 1 to k + # if i=1 or {Y(1),...,Y(i-1)} subset of NULLABLE then + # FIRST(X) = FIRST(X) union FIRST(Y(i)) + # for j = i+1 to k + # if i=k or {Y(i+1),...Y(k)} subset of NULLABLE then + # FOLLOW(Y(i)) = FOLLOW(Y(i)) union FOLLOW(X) + # if i+1=j or {Y(i+1),...,Y(j-1)} subset of NULLABLE then + # FOLLOW(Y(i)) = FOLLOW(Y(i)) union FIRST(Y(j)) + # until none of NULLABLE,FIRST,FOLLOW changed in last iteration + + NULLABLE = set() + FIRST = {} + FOLLOW = {} + for sym in symbols: + FIRST[sym]={sym} if sym.is_term else set() + FOLLOW[sym]=set() + + # Calculate NULLABLE and FIRST + changed = True + while changed: + changed = False + + for rule in rules: + if set(rule.expansion) <= NULLABLE: + if update_set(NULLABLE, {rule.origin}): + changed = True + + for i, sym in enumerate(rule.expansion): + if set(rule.expansion[:i]) <= NULLABLE: + if update_set(FIRST[rule.origin], FIRST[sym]): + changed = True + else: + break + + # Calculate FOLLOW + changed = True + while changed: + changed = False + + for rule in rules: + for i, sym in enumerate(rule.expansion): + if i==len(rule.expansion)-1 or set(rule.expansion[i+1:]) <= NULLABLE: + if update_set(FOLLOW[sym], FOLLOW[rule.origin]): + changed = True + + for j in range(i+1, len(rule.expansion)): + if set(rule.expansion[i+1:j]) <= NULLABLE: + if update_set(FOLLOW[sym], FIRST[rule.expansion[j]]): + changed = True + + return FIRST, FOLLOW, NULLABLE + + +class GrammarAnalyzer(object): + def __init__(self, parser_conf, debug=False): + self.debug = debug + + root_rules = {start: Rule(NonTerminal('$root_' + start), [NonTerminal(start), Terminal('$END')]) + for start in parser_conf.start} + + rules = parser_conf.rules + list(root_rules.values()) + self.rules_by_origin = classify(rules, lambda r: r.origin) + + if len(rules) != len(set(rules)): + duplicates = [item for item, count in Counter(rules).items() if count > 1] + raise GrammarError("Rules defined twice: %s" % ', '.join(str(i) for i in duplicates)) + + for r in rules: + for sym in r.expansion: + if not (sym.is_term or sym in self.rules_by_origin): + raise GrammarError("Using an undefined rule: %s" % sym) # TODO test validation + + self.start_states = {start: self.expand_rule(root_rule.origin) + for start, root_rule in root_rules.items()} + + self.end_states = {start: fzset({RulePtr(root_rule, len(root_rule.expansion))}) + for start, root_rule in root_rules.items()} + + lr0_root_rules = {start: Rule(NonTerminal('$root_' + start), [NonTerminal(start)]) + for start in parser_conf.start} + + lr0_rules = parser_conf.rules + list(lr0_root_rules.values()) + assert(len(lr0_rules) == len(set(lr0_rules))) + + self.lr0_rules_by_origin = classify(lr0_rules, lambda r: r.origin) + + # cache RulePtr(r, 0) in r (no duplicate RulePtr objects) + self.lr0_start_states = {start: LR0ItemSet([RulePtr(root_rule, 0)], self.expand_rule(root_rule.origin, self.lr0_rules_by_origin)) + for start, root_rule in lr0_root_rules.items()} + + self.FIRST, self.FOLLOW, self.NULLABLE = calculate_sets(rules) + + def expand_rule(self, source_rule, rules_by_origin=None): + "Returns all init_ptrs accessible by rule (recursive)" + + if rules_by_origin is None: + rules_by_origin = self.rules_by_origin + + init_ptrs = set() + def _expand_rule(rule): + assert not rule.is_term, rule + + for r in rules_by_origin[rule]: + init_ptr = RulePtr(r, 0) + init_ptrs.add(init_ptr) + + if r.expansion: # if not empty rule + new_r = init_ptr.next + if not new_r.is_term: + yield new_r + + for _ in bfs([source_rule], _expand_rule): + pass + + return fzset(init_ptrs) diff --git a/poetry/core/_vendor/lark/parsers/lalr_analysis.py b/poetry/core/_vendor/lark/parsers/lalr_analysis.py new file mode 100644 index 000000000..8890c3cd2 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/lalr_analysis.py @@ -0,0 +1,288 @@ +"""This module builds a LALR(1) transition-table for lalr_parser.py + +For now, shift/reduce conflicts are automatically resolved as shifts. +""" + +# Author: Erez Shinan (2017) +# Email : erezshin@gmail.com + +import logging +from collections import defaultdict, deque + +from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator +from ..exceptions import GrammarError + +from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet +from ..grammar import Rule + +###{standalone + +class Action: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + def __repr__(self): + return str(self) + +Shift = Action('Shift') +Reduce = Action('Reduce') + + +class ParseTable: + def __init__(self, states, start_states, end_states): + self.states = states + self.start_states = start_states + self.end_states = end_states + + def serialize(self, memo): + tokens = Enumerator() + rules = Enumerator() + + states = { + state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg)) + for token, (action, arg) in actions.items()} + for state, actions in self.states.items() + } + + return { + 'tokens': tokens.reversed(), + 'states': states, + 'start_states': self.start_states, + 'end_states': self.end_states, + } + + @classmethod + def deserialize(cls, data, memo): + tokens = data['tokens'] + states = { + state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg)) + for token, (action, arg) in actions.items()} + for state, actions in data['states'].items() + } + return cls(states, data['start_states'], data['end_states']) + + +class IntParseTable(ParseTable): + + @classmethod + def from_ParseTable(cls, parse_table): + enum = list(parse_table.states) + state_to_idx = {s:i for i,s in enumerate(enum)} + int_states = {} + + for s, la in parse_table.states.items(): + la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v + for k,v in la.items()} + int_states[ state_to_idx[s] ] = la + + + start_states = {start:state_to_idx[s] for start, s in parse_table.start_states.items()} + end_states = {start:state_to_idx[s] for start, s in parse_table.end_states.items()} + return cls(int_states, start_states, end_states) + +###} + + +# digraph and traverse, see The Theory and Practice of Compiler Writing + +# computes F(x) = G(x) union (union { G(y) | x R y }) +# X: nodes +# R: relation (function mapping node -> list of nodes that satisfy the relation) +# G: set valued function +def digraph(X, R, G): + F = {} + S = [] + N = {} + for x in X: + N[x] = 0 + for x in X: + # this is always true for the first iteration, but N[x] may be updated in traverse below + if N[x] == 0: + traverse(x, S, N, X, R, G, F) + return F + +# x: single node +# S: stack +# N: weights +# X: nodes +# R: relation (see above) +# G: set valued function +# F: set valued function we are computing (map of input -> output) +def traverse(x, S, N, X, R, G, F): + S.append(x) + d = len(S) + N[x] = d + F[x] = G[x] + for y in R[x]: + if N[y] == 0: + traverse(y, S, N, X, R, G, F) + n_x = N[x] + assert(n_x > 0) + n_y = N[y] + assert(n_y != 0) + if (n_y > 0) and (n_y < n_x): + N[x] = n_y + F[x].update(F[y]) + if N[x] == d: + f_x = F[x] + while True: + z = S.pop() + N[z] = -1 + F[z] = f_x + if z == x: + break + + +class LALR_Analyzer(GrammarAnalyzer): + def __init__(self, parser_conf, debug=False): + GrammarAnalyzer.__init__(self, parser_conf, debug) + self.nonterminal_transitions = [] + self.directly_reads = defaultdict(set) + self.reads = defaultdict(set) + self.includes = defaultdict(set) + self.lookback = defaultdict(set) + + + def compute_lr0_states(self): + self.lr0_states = set() + # map of kernels to LR0ItemSets + cache = {} + + def step(state): + _, unsat = classify_bool(state.closure, lambda rp: rp.is_satisfied) + + d = classify(unsat, lambda rp: rp.next) + for sym, rps in d.items(): + kernel = fzset({rp.advance(sym) for rp in rps}) + new_state = cache.get(kernel, None) + if new_state is None: + closure = set(kernel) + for rp in kernel: + if not rp.is_satisfied and not rp.next.is_term: + closure |= self.expand_rule(rp.next, self.lr0_rules_by_origin) + new_state = LR0ItemSet(kernel, closure) + cache[kernel] = new_state + + state.transitions[sym] = new_state + yield new_state + + self.lr0_states.add(state) + + for _ in bfs(self.lr0_start_states.values(), step): + pass + + def compute_reads_relations(self): + # handle start state + for root in self.lr0_start_states.values(): + assert(len(root.kernel) == 1) + for rp in root.kernel: + assert(rp.index == 0) + self.directly_reads[(root, rp.next)] = set([ Terminal('$END') ]) + + for state in self.lr0_states: + seen = set() + for rp in state.closure: + if rp.is_satisfied: + continue + s = rp.next + # if s is a not a nonterminal + if s not in self.lr0_rules_by_origin: + continue + if s in seen: + continue + seen.add(s) + nt = (state, s) + self.nonterminal_transitions.append(nt) + dr = self.directly_reads[nt] + r = self.reads[nt] + next_state = state.transitions[s] + for rp2 in next_state.closure: + if rp2.is_satisfied: + continue + s2 = rp2.next + # if s2 is a terminal + if s2 not in self.lr0_rules_by_origin: + dr.add(s2) + if s2 in self.NULLABLE: + r.add((next_state, s2)) + + def compute_includes_lookback(self): + for nt in self.nonterminal_transitions: + state, nonterminal = nt + includes = [] + lookback = self.lookback[nt] + for rp in state.closure: + if rp.rule.origin != nonterminal: + continue + # traverse the states for rp(.rule) + state2 = state + for i in range(rp.index, len(rp.rule.expansion)): + s = rp.rule.expansion[i] + nt2 = (state2, s) + state2 = state2.transitions[s] + if nt2 not in self.reads: + continue + for j in range(i + 1, len(rp.rule.expansion)): + if not rp.rule.expansion[j] in self.NULLABLE: + break + else: + includes.append(nt2) + # state2 is at the final state for rp.rule + if rp.index == 0: + for rp2 in state2.closure: + if (rp2.rule == rp.rule) and rp2.is_satisfied: + lookback.add((state2, rp2.rule)) + for nt2 in includes: + self.includes[nt2].add(nt) + + def compute_lookaheads(self): + read_sets = digraph(self.nonterminal_transitions, self.reads, self.directly_reads) + follow_sets = digraph(self.nonterminal_transitions, self.includes, read_sets) + + for nt, lookbacks in self.lookback.items(): + for state, rule in lookbacks: + for s in follow_sets[nt]: + state.lookaheads[s].add(rule) + + def compute_lalr1_states(self): + m = {} + for state in self.lr0_states: + actions = {} + for la, next_state in state.transitions.items(): + actions[la] = (Shift, next_state.closure) + for la, rules in state.lookaheads.items(): + if len(rules) > 1: + raise GrammarError('Reduce/Reduce collision in %s between the following rules: %s' % (la, ''.join([ '\n\t\t- ' + str(r) for r in rules ]))) + if la in actions: + if self.debug: + logging.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name) + logging.warning(' * %s', list(rules)[0]) + else: + actions[la] = (Reduce, list(rules)[0]) + m[state] = { k.name: v for k, v in actions.items() } + + states = { k.closure: v for k, v in m.items() } + + # compute end states + end_states = {} + for state in states: + for rp in state: + for start in self.lr0_start_states: + if rp.rule.origin.name == ('$root_' + start) and rp.is_satisfied: + assert(not start in end_states) + end_states[start] = state + + _parse_table = ParseTable(states, { start: state.closure for start, state in self.lr0_start_states.items() }, end_states) + + if self.debug: + self.parse_table = _parse_table + else: + self.parse_table = IntParseTable.from_ParseTable(_parse_table) + + def compute_lalr(self): + self.compute_lr0_states() + self.compute_reads_relations() + self.compute_includes_lookback() + self.compute_lookaheads() + self.compute_lalr1_states() \ No newline at end of file diff --git a/poetry/core/_vendor/lark/parsers/lalr_parser.py b/poetry/core/_vendor/lark/parsers/lalr_parser.py new file mode 100644 index 000000000..f26cbc5b0 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/lalr_parser.py @@ -0,0 +1,119 @@ +"""This module implements a LALR(1) Parser +""" +# Author: Erez Shinan (2017) +# Email : erezshin@gmail.com +from ..exceptions import UnexpectedToken +from ..lexer import Token +from ..utils import Enumerator, Serialize + +from .lalr_analysis import LALR_Analyzer, Shift, Reduce, IntParseTable +from .lalr_puppet import ParserPuppet + +###{standalone + +class LALR_Parser(object): + def __init__(self, parser_conf, debug=False): + assert all(r.options.priority is None for r in parser_conf.rules), "LALR doesn't yet support prioritization" + analysis = LALR_Analyzer(parser_conf, debug=debug) + analysis.compute_lalr() + callbacks = parser_conf.callbacks + + self._parse_table = analysis.parse_table + self.parser_conf = parser_conf + self.parser = _Parser(analysis.parse_table, callbacks, debug) + + @classmethod + def deserialize(cls, data, memo, callbacks): + inst = cls.__new__(cls) + inst._parse_table = IntParseTable.deserialize(data, memo) + inst.parser = _Parser(inst._parse_table, callbacks) + return inst + + def serialize(self, memo): + return self._parse_table.serialize(memo) + + def parse(self, *args): + return self.parser.parse(*args) + + +class _Parser: + def __init__(self, parse_table, callbacks, debug=False): + self.parse_table = parse_table + self.callbacks = callbacks + self.debug = debug + + def parse(self, seq, start, set_state=None, value_stack=None, state_stack=None): + token = None + stream = iter(seq) + states = self.parse_table.states + start_state = self.parse_table.start_states[start] + end_state = self.parse_table.end_states[start] + + state_stack = state_stack or [start_state] + value_stack = value_stack or [] + + if set_state: set_state(start_state) + + def get_action(token): + state = state_stack[-1] + try: + return states[state][token.type] + except KeyError: + expected = [s for s in states[state].keys() if s.isupper()] + try: + puppet = ParserPuppet(self, state_stack, value_stack, start, stream, set_state) + except NameError: + puppet = None + raise UnexpectedToken(token, expected, state=state, puppet=puppet) + + def reduce(rule): + size = len(rule.expansion) + if size: + s = value_stack[-size:] + del state_stack[-size:] + del value_stack[-size:] + else: + s = [] + + value = self.callbacks[rule](s) + + _action, new_state = states[state_stack[-1]][rule.origin.name] + assert _action is Shift + state_stack.append(new_state) + value_stack.append(value) + + # Main LALR-parser loop + try: + for token in stream: + while True: + action, arg = get_action(token) + assert arg != end_state + + if action is Shift: + state_stack.append(arg) + value_stack.append(token) + if set_state: set_state(arg) + break # next token + else: + reduce(arg) + except Exception as e: + if self.debug: + print("") + print("STATE STACK DUMP") + print("----------------") + for i, s in enumerate(state_stack): + print('%d)' % i , s) + print("") + + raise + + token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1) + while True: + _action, arg = get_action(token) + assert(_action is Reduce) + reduce(arg) + if state_stack[-1] == end_state: + return value_stack[-1] + +###} + diff --git a/poetry/core/_vendor/lark/parsers/lalr_puppet.py b/poetry/core/_vendor/lark/parsers/lalr_puppet.py new file mode 100644 index 000000000..968783cc4 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/lalr_puppet.py @@ -0,0 +1,79 @@ +# This module provide a LALR puppet, which is used to debugging and error handling + +from copy import deepcopy + +from .lalr_analysis import Shift, Reduce + +class ParserPuppet: + def __init__(self, parser, state_stack, value_stack, start, stream, set_state): + self.parser = parser + self._state_stack = state_stack + self._value_stack = value_stack + self._start = start + self._stream = stream + self._set_state = set_state + + self.result = None + + def feed_token(self, token): + """Advance the parser state, as if it just recieved `token` from the lexer + + """ + end_state = self.parser.parse_table.end_states[self._start] + state_stack = self._state_stack + value_stack = self._value_stack + + state = state_stack[-1] + action, arg = self.parser.parse_table.states[state][token.type] + assert arg != end_state + + while action is Reduce: + rule = arg + size = len(rule.expansion) + if size: + s = value_stack[-size:] + del state_stack[-size:] + del value_stack[-size:] + else: + s = [] + + value = self.parser.callbacks[rule](s) + + _action, new_state = self.parser.parse_table.states[state_stack[-1]][rule.origin.name] + assert _action is Shift + state_stack.append(new_state) + value_stack.append(value) + + if state_stack[-1] == end_state: + self.result = value_stack[-1] + return self.result + + state = state_stack[-1] + action, arg = self.parser.parse_table.states[state][token.type] + assert arg != end_state + + assert action is Shift + state_stack.append(arg) + value_stack.append(token) + + def copy(self): + return type(self)( + self.parser, + list(self._state_stack), + deepcopy(self._value_stack), + self._start, + self._stream, + self._set_state, + ) + + def pretty(): + print("Puppet choices:") + for k, v in self.choices.items(): + print('\t-', k, '->', v) + print('stack size:', len(self._state_stack)) + + def choices(self): + return self.parser.parse_table.states[self._state_stack[-1]] + + def resume_parse(self): + return self.parser.parse(self._stream, self._start, self._set_state, self._value_stack, self._state_stack) diff --git a/poetry/core/_vendor/lark/parsers/xearley.py b/poetry/core/_vendor/lark/parsers/xearley.py new file mode 100644 index 000000000..855625a96 --- /dev/null +++ b/poetry/core/_vendor/lark/parsers/xearley.py @@ -0,0 +1,151 @@ +"""This module implements an experimental Earley parser with a dynamic lexer + +The core Earley algorithm used here is based on Elizabeth Scott's implementation, here: + https://www.sciencedirect.com/science/article/pii/S1571066108001497 + +That is probably the best reference for understanding the algorithm here. + +The Earley parser outputs an SPPF-tree as per that document. The SPPF tree format +is better documented here: + http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/ + +Instead of running a lexer beforehand, or using a costy char-by-char method, this parser +uses regular expressions by necessity, achieving high-performance while maintaining all of +Earley's power in parsing any CFG. +""" + +from collections import defaultdict + +from ..exceptions import UnexpectedCharacters +from ..lexer import Token +from ..grammar import Terminal +from .earley import Parser as BaseParser +from .earley_forest import SymbolNode + + +class Parser(BaseParser): + def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, ignore = (), complete_lex = False, debug=False): + BaseParser.__init__(self, parser_conf, term_matcher, resolve_ambiguity, debug) + self.ignore = [Terminal(t) for t in ignore] + self.complete_lex = complete_lex + + def _parse(self, stream, columns, to_scan, start_symbol=None): + + def scan(i, to_scan): + """The core Earley Scanner. + + This is a custom implementation of the scanner that uses the + Lark lexer to match tokens. The scan list is built by the + Earley predictor, based on the previously completed tokens. + This ensures that at each phase of the parse we have a custom + lexer context, allowing for more complex ambiguities.""" + + node_cache = {} + + # 1) Loop the expectations and ask the lexer to match. + # Since regexp is forward looking on the input stream, and we only + # want to process tokens when we hit the point in the stream at which + # they complete, we push all tokens into a buffer (delayed_matches), to + # be held possibly for a later parse step when we reach the point in the + # input stream at which they complete. + for item in set(to_scan): + m = match(item.expect, stream, i) + if m: + t = Token(item.expect.name, m.group(0), i, text_line, text_column) + delayed_matches[m.end()].append( (item, i, t) ) + + if self.complete_lex: + s = m.group(0) + for j in range(1, len(s)): + m = match(item.expect, s[:-j]) + if m: + t = Token(item.expect.name, m.group(0), i, text_line, text_column) + delayed_matches[i+m.end()].append( (item, i, t) ) + + # Remove any items that successfully matched in this pass from the to_scan buffer. + # This ensures we don't carry over tokens that already matched, if we're ignoring below. + to_scan.remove(item) + + # 3) Process any ignores. This is typically used for e.g. whitespace. + # We carry over any unmatched items from the to_scan buffer to be matched again after + # the ignore. This should allow us to use ignored symbols in non-terminals to implement + # e.g. mandatory spacing. + for x in self.ignore: + m = match(x, stream, i) + if m: + # Carry over any items still in the scan buffer, to past the end of the ignored items. + delayed_matches[m.end()].extend([(item, i, None) for item in to_scan ]) + + # If we're ignoring up to the end of the file, # carry over the start symbol if it already completed. + delayed_matches[m.end()].extend([(item, i, None) for item in columns[i] if item.is_complete and item.s == start_symbol]) + + next_to_scan = set() + next_set = set() + columns.append(next_set) + transitives.append({}) + + ## 4) Process Tokens from delayed_matches. + # This is the core of the Earley scanner. Create an SPPF node for each Token, + # and create the symbol node in the SPPF tree. Advance the item that completed, + # and add the resulting new item to either the Earley set (for processing by the + # completer/predictor) or the to_scan buffer for the next parse step. + for item, start, token in delayed_matches[i+1]: + if token is not None: + token.end_line = text_line + token.end_column = text_column + 1 + token.end_pos = i + 1 + + new_item = item.advance() + label = (new_item.s, new_item.start, i) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label)) + new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token) + else: + new_item = item + + if new_item.expect in self.TERMINALS: + # add (B ::= Aai+1.B, h, y) to Q' + next_to_scan.add(new_item) + else: + # add (B ::= Aa+1.B, h, y) to Ei+1 + next_set.add(new_item) + + del delayed_matches[i+1] # No longer needed, so unburden memory + + if not next_set and not delayed_matches and not next_to_scan: + raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect.name for item in to_scan}, set(to_scan)) + + return next_to_scan + + + delayed_matches = defaultdict(list) + match = self.term_matcher + + # Cache for nodes & tokens created in a particular parse step. + transitives = [{}] + + text_line = 1 + text_column = 1 + + ## The main Earley loop. + # Run the Prediction/Completion cycle for any Items in the current Earley set. + # Completions will be added to the SPPF tree, and predictions will be recursively + # processed down to terminals/empty nodes to be added to the scanner for the next + # step. + i = 0 + for token in stream: + self.predict_and_complete(i, to_scan, columns, transitives) + + to_scan = scan(i, to_scan) + + if token == '\n': + text_line += 1 + text_column = 1 + else: + text_column += 1 + i += 1 + + self.predict_and_complete(i, to_scan, columns, transitives) + + ## Column is now the final column in the parse. + assert i == len(columns)-1 + return to_scan \ No newline at end of file diff --git a/poetry/core/_vendor/lark/reconstruct.py b/poetry/core/_vendor/lark/reconstruct.py new file mode 100644 index 000000000..1e3adc77c --- /dev/null +++ b/poetry/core/_vendor/lark/reconstruct.py @@ -0,0 +1,164 @@ +from collections import defaultdict + +from .tree import Tree +from .visitors import Transformer_InPlace +from .common import ParserConf +from .lexer import Token, PatternStr +from .parsers import earley +from .grammar import Rule, Terminal, NonTerminal + + + +def is_discarded_terminal(t): + return t.is_term and t.filter_out + +def is_iter_empty(i): + try: + _ = next(i) + return False + except StopIteration: + return True + + +class WriteTokensTransformer(Transformer_InPlace): + "Inserts discarded tokens into their correct place, according to the rules of grammar" + + def __init__(self, tokens, term_subs): + self.tokens = tokens + self.term_subs = term_subs + + def __default__(self, data, children, meta): + if not getattr(meta, 'match_tree', False): + return Tree(data, children) + + iter_args = iter(children) + to_write = [] + for sym in meta.orig_expansion: + if is_discarded_terminal(sym): + try: + v = self.term_subs[sym.name](sym) + except KeyError: + t = self.tokens[sym.name] + if not isinstance(t.pattern, PatternStr): + raise NotImplementedError("Reconstructing regexps not supported yet: %s" % t) + + v = t.pattern.value + to_write.append(v) + else: + x = next(iter_args) + if isinstance(x, list): + to_write += x + else: + if isinstance(x, Token): + assert Terminal(x.type) == sym, x + else: + assert NonTerminal(x.data) == sym, (sym, x) + to_write.append(x) + + assert is_iter_empty(iter_args) + return to_write + + +class MatchTree(Tree): + pass + +class MakeMatchTree: + def __init__(self, name, expansion): + self.name = name + self.expansion = expansion + + def __call__(self, args): + t = MatchTree(self.name, args) + t.meta.match_tree = True + t.meta.orig_expansion = self.expansion + return t + +def best_from_group(seq, group_key, cmp_key): + d = {} + for item in seq: + key = group_key(item) + if key in d: + v1 = cmp_key(item) + v2 = cmp_key(d[key]) + if v2 > v1: + d[key] = item + else: + d[key] = item + return list(d.values()) + +class Reconstructor: + def __init__(self, parser, term_subs={}): + # XXX TODO calling compile twice returns different results! + assert parser.options.maybe_placeholders == False + tokens, rules, _grammar_extra = parser.grammar.compile(parser.options.start) + + self.write_tokens = WriteTokensTransformer({t.name:t for t in tokens}, term_subs) + self.rules = list(self._build_recons_rules(rules)) + self.rules.reverse() + + # Choose the best rule from each group of {rule => [rule.alias]}, since we only really need one derivation. + self.rules = best_from_group(self.rules, lambda r: r, lambda r: -len(r.expansion)) + + self.rules.sort(key=lambda r: len(r.expansion)) + callbacks = {rule: rule.alias for rule in self.rules} # TODO pass callbacks through dict, instead of alias? + self.parser = earley.Parser(ParserConf(self.rules, callbacks, parser.options.start), + self._match, resolve_ambiguity=True) + + def _build_recons_rules(self, rules): + expand1s = {r.origin for r in rules if r.options.expand1} + + aliases = defaultdict(list) + for r in rules: + if r.alias: + aliases[r.origin].append( r.alias ) + + rule_names = {r.origin for r in rules} + nonterminals = {sym for sym in rule_names + if sym.name.startswith('_') or sym in expand1s or sym in aliases } + + for r in rules: + recons_exp = [sym if sym in nonterminals else Terminal(sym.name) + for sym in r.expansion if not is_discarded_terminal(sym)] + + # Skip self-recursive constructs + if recons_exp == [r.origin]: + continue + + sym = NonTerminal(r.alias) if r.alias else r.origin + + yield Rule(sym, recons_exp, alias=MakeMatchTree(sym.name, r.expansion)) + + for origin, rule_aliases in aliases.items(): + for alias in rule_aliases: + yield Rule(origin, [Terminal(alias)], alias=MakeMatchTree(origin.name, [NonTerminal(alias)])) + yield Rule(origin, [Terminal(origin.name)], alias=MakeMatchTree(origin.name, [origin])) + + def _match(self, term, token): + if isinstance(token, Tree): + return Terminal(token.data) == term + elif isinstance(token, Token): + return term == Terminal(token.type) + assert False + + def _reconstruct(self, tree): + # TODO: ambiguity? + unreduced_tree = self.parser.parse(tree.children, tree.data) # find a full derivation + assert unreduced_tree.data == tree.data + res = self.write_tokens.transform(unreduced_tree) + for item in res: + if isinstance(item, Tree): + for x in self._reconstruct(item): + yield x + else: + yield item + + def reconstruct(self, tree): + x = self._reconstruct(tree) + y = [] + prev_item = '' + for item in x: + if prev_item and item and prev_item[-1].isalnum() and item[0].isalnum(): + y.append(' ') + y.append(item) + prev_item = item + return ''.join(y) diff --git a/poetry/core/_vendor/lark/reconstruct2.py b/poetry/core/_vendor/lark/reconstruct2.py new file mode 100644 index 000000000..c7300a062 --- /dev/null +++ b/poetry/core/_vendor/lark/reconstruct2.py @@ -0,0 +1,155 @@ +from collections import defaultdict + +from .tree import Tree +from .visitors import Transformer_InPlace +from .common import ParserConf +from .lexer import Token, PatternStr +from .parsers import earley +from .grammar import Rule, Terminal, NonTerminal + + + +def is_discarded_terminal(t): + return t.is_term and t.filter_out + +def is_iter_empty(i): + try: + _ = next(i) + return False + except StopIteration: + return True + +class WriteTokensTransformer(Transformer_InPlace): + def __init__(self, tokens): + self.tokens = tokens + + def __default__(self, data, children, meta): + # if not isinstance(t, MatchTree): + # return t + if not getattr(meta, 'match_tree', False): + return Tree(data, children) + + iter_args = iter(children) + print('@@@', children, meta.orig_expansion) + to_write = [] + for sym in meta.orig_expansion: + if is_discarded_terminal(sym): + t = self.tokens[sym.name] + value = t.pattern.value + if not isinstance(t.pattern, PatternStr): + if t.name == "_NEWLINE": + value = "\n" + else: + raise NotImplementedError("Reconstructing regexps not supported yet: %s" % t) + to_write.append(value) + else: + x = next(iter_args) + if isinstance(x, list): + to_write += x + else: + if isinstance(x, Token): + assert Terminal(x.type) == sym, x + else: + assert NonTerminal(x.data) == sym, (sym, x) + to_write.append(x) + + assert is_iter_empty(iter_args) + return to_write + + +class MatchTree(Tree): + pass + +class MakeMatchTree: + def __init__(self, name, expansion): + self.name = name + self.expansion = expansion + + def __call__(self, args): + t = MatchTree(self.name, args) + t.meta.match_tree = True + t.meta.orig_expansion = self.expansion + return t + +from lark.load_grammar import SimplifyRule_Visitor, RuleTreeToText +class Reconstructor: + def __init__(self, parser): + # XXX TODO calling compile twice returns different results! + assert parser.options.maybe_placeholders == False + tokens, rules, _grammar_extra = parser.grammar.compile(parser.options.start) + + self.write_tokens = WriteTokensTransformer({t.name:t for t in tokens}) + self.rules = list(set(list(self._build_recons_rules(rules)))) + callbacks = {rule: rule.alias for rule in self.rules} # TODO pass callbacks through dict, instead of alias? + for r in self.rules: + print("*", r) + self.parser = earley.Parser(ParserConf(self.rules, callbacks, parser.options.start), + self._match, resolve_ambiguity=True) + + def _build_recons_rules(self, rules): + expand1s = {r.origin for r in rules if r.options.expand1} + + aliases = defaultdict(list) + for r in rules: + if r.alias: + aliases[r.origin].append( r.alias ) + + rule_names = {r.origin for r in rules} + nonterminals = {sym for sym in rule_names + if sym.name.startswith('_') or sym in expand1s or sym in aliases } + + for r in rules: + _recons_exp = [] + for sym in r.expansion: + if not is_discarded_terminal(sym): + if sym in nonterminals: + if sym in expand1s: + v = Tree('expansions', [sym, Terminal(sym.name.upper())]) + else: + v = sym + else: + v = Terminal(sym.name.upper()) + _recons_exp.append(v) + + simplify_rule = SimplifyRule_Visitor() + rule_tree_to_text = RuleTreeToText() + tree = Tree('expansions', [Tree('expansion', _recons_exp)]) + simplify_rule.visit(tree) + expansions = rule_tree_to_text.transform(tree) + + for recons_exp, alias in expansions: + + # Skip self-recursive constructs + if recons_exp == [r.origin]: + continue + + sym = NonTerminal(r.alias) if r.alias else r.origin + + yield Rule(sym, recons_exp, alias=MakeMatchTree(sym.name, r.expansion)) + + for origin, rule_aliases in aliases.items(): + for alias in rule_aliases: + yield Rule(origin, [Terminal(alias.upper())], alias=MakeMatchTree(origin.name, [NonTerminal(alias)])) + yield Rule(origin, [Terminal(origin.name.upper())], alias=MakeMatchTree(origin.name, [origin])) + + def _match(self, term, token): + if isinstance(token, Tree): + return Terminal(token.data.upper()) == term + elif isinstance(token, Token): + return term == Terminal(token.type.upper()) + assert False + + def _reconstruct(self, tree): + # TODO: ambiguity? + unreduced_tree = self.parser.parse(tree.children, tree.data) # find a full derivation + assert unreduced_tree.data == tree.data + res = self.write_tokens.transform(unreduced_tree) + for item in res: + if isinstance(item, Tree): + for x in self._reconstruct(item): + yield x + else: + yield item + + def reconstruct(self, tree): + return ''.join(self._reconstruct(tree)) diff --git a/poetry/core/_vendor/lark/tools/__init__.py b/poetry/core/_vendor/lark/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poetry/core/_vendor/lark/tools/nearley.py b/poetry/core/_vendor/lark/tools/nearley.py new file mode 100644 index 000000000..0b04fb55f --- /dev/null +++ b/poetry/core/_vendor/lark/tools/nearley.py @@ -0,0 +1,190 @@ +"Converts between Lark and Nearley grammars. Work in progress!" + +import os.path +import sys +import codecs + + +from lark import Lark, InlineTransformer + +nearley_grammar = r""" + start: (ruledef|directive)+ + + directive: "@" NAME (STRING|NAME) + | "@" JS -> js_code + ruledef: NAME "->" expansions + | NAME REGEXP "->" expansions -> macro + expansions: expansion ("|" expansion)* + + expansion: expr+ js + + ?expr: item (":" /[+*?]/)? + + ?item: rule|string|regexp|null + | "(" expansions ")" + + rule: NAME + string: STRING + regexp: REGEXP + null: "null" + JS: /{%.*?%}/s + js: JS? + + NAME: /[a-zA-Z_$]\w*/ + COMMENT: /#[^\n]*/ + REGEXP: /\[.*?\]/ + + %import common.ESCAPED_STRING -> STRING + %import common.WS + %ignore WS + %ignore COMMENT + + """ + +nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard') + +def _get_rulename(name): + name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name) + return 'n_' + name.replace('$', '__DOLLAR__').lower() + +class NearleyToLark(InlineTransformer): + def __init__(self): + self._count = 0 + self.extra_rules = {} + self.extra_rules_rev = {} + self.alias_js_code = {} + + def _new_function(self, code): + name = 'alias_%d' % self._count + self._count += 1 + + self.alias_js_code[name] = code + return name + + def _extra_rule(self, rule): + if rule in self.extra_rules_rev: + return self.extra_rules_rev[rule] + + name = 'xrule_%d' % len(self.extra_rules) + assert name not in self.extra_rules + self.extra_rules[name] = rule + self.extra_rules_rev[rule] = name + return name + + def rule(self, name): + return _get_rulename(name) + + def ruledef(self, name, exps): + return '!%s: %s' % (_get_rulename(name), exps) + + def expr(self, item, op): + rule = '(%s)%s' % (item, op) + return self._extra_rule(rule) + + def regexp(self, r): + return '/%s/' % r + + def null(self): + return '' + + def string(self, s): + return self._extra_rule(s) + + def expansion(self, *x): + x, js = x[:-1], x[-1] + if js.children: + js_code ,= js.children + js_code = js_code[2:-2] + alias = '-> ' + self._new_function(js_code) + else: + alias = '' + return ' '.join(x) + alias + + def expansions(self, *x): + return '%s' % ('\n |'.join(x)) + + def start(self, *rules): + return '\n'.join(filter(None, rules)) + +def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes): + rule_defs = [] + + tree = nearley_grammar_parser.parse(g) + for statement in tree.children: + if statement.data == 'directive': + directive, arg = statement.children + if directive in ('builtin', 'include'): + folder = builtin_path if directive == 'builtin' else folder_path + path = os.path.join(folder, arg[1:-1]) + if path not in includes: + includes.add(path) + with codecs.open(path, encoding='utf8') as f: + text = f.read() + rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code, os.path.abspath(os.path.dirname(path)), includes) + else: + assert False, directive + elif statement.data == 'js_code': + code ,= statement.children + code = code[2:-2] + js_code.append(code) + elif statement.data == 'macro': + pass # TODO Add support for macros! + elif statement.data == 'ruledef': + rule_defs.append( n2l.transform(statement) ) + else: + raise Exception("Unknown statement: %s" % statement) + + return rule_defs + + +def create_code_for_nearley_grammar(g, start, builtin_path, folder_path): + import js2py + + emit_code = [] + def emit(x=None): + if x: + emit_code.append(x) + emit_code.append('\n') + + js_code = ['function id(x) {return x[0];}'] + n2l = NearleyToLark() + rule_defs = _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, set()) + lark_g = '\n'.join(rule_defs) + lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items()) + + emit('from lark import Lark, Transformer') + emit() + emit('grammar = ' + repr(lark_g)) + emit() + + for alias, code in n2l.alias_js_code.items(): + js_code.append('%s = (%s);' % (alias, code)) + + emit(js2py.translate_js('\n'.join(js_code))) + emit('class TransformNearley(Transformer):') + for alias in n2l.alias_js_code: + emit(" %s = var.get('%s').to_python()" % (alias, alias)) + emit(" __default__ = lambda self, n, c, m: c if c else None") + + emit() + emit('parser = Lark(grammar, start="n_%s", maybe_placeholders=False)' % start) + emit('def parse(text):') + emit(' return TransformNearley().transform(parser.parse(text))') + + return ''.join(emit_code) + +def main(fn, start, nearley_lib): + with codecs.open(fn, encoding='utf8') as f: + grammar = f.read() + return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn))) + + +if __name__ == '__main__': + if len(sys.argv) < 4: + print("Reads Nearley grammar (with js functions) outputs an equivalent lark parser.") + print("Usage: %s " % sys.argv[0]) + sys.exit(1) + + fn, start, nearley_lib = sys.argv[1:] + + print(main(fn, start, nearley_lib)) diff --git a/poetry/core/_vendor/lark/tools/serialize.py b/poetry/core/_vendor/lark/tools/serialize.py new file mode 100644 index 000000000..fb69d35ac --- /dev/null +++ b/poetry/core/_vendor/lark/tools/serialize.py @@ -0,0 +1,39 @@ +import codecs +import sys +import json + +from lark import Lark +from lark.grammar import RuleOptions, Rule +from lark.lexer import TerminalDef + +import argparse + +argparser = argparse.ArgumentParser(prog='python -m lark.tools.serialize') #description='''Lark Serialization Tool -- Stores Lark's internal state & LALR analysis as a convenient JSON file''') + +argparser.add_argument('grammar_file', type=argparse.FileType('r'), help='A valid .lark file') +argparser.add_argument('-o', '--out', type=argparse.FileType('w'), default=sys.stdout, help='json file path to create (default=stdout)') +argparser.add_argument('-s', '--start', default='start', help='start symbol (default="start")', nargs='+') +argparser.add_argument('-l', '--lexer', default='standard', choices=['standard', 'contextual'], help='lexer type (default="standard")') + + +def serialize(infile, outfile, lexer, start): + lark_inst = Lark(infile, parser="lalr", lexer=lexer, start=start) # TODO contextual + + data, memo = lark_inst.memo_serialize([TerminalDef, Rule]) + outfile.write('{\n') + outfile.write(' "data": %s,\n' % json.dumps(data)) + outfile.write(' "memo": %s\n' % json.dumps(memo)) + outfile.write('}\n') + + +def main(): + if len(sys.argv) == 1 or '-h' in sys.argv or '--help' in sys.argv: + print("Lark Serialization Tool - Stores Lark's internal state & LALR analysis as a JSON file") + print("") + argparser.print_help() + else: + args = argparser.parse_args() + serialize(args.grammar_file, args.out, args.lexer, args.start) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/poetry/core/_vendor/lark/tools/standalone.py b/poetry/core/_vendor/lark/tools/standalone.py new file mode 100644 index 000000000..72042cdaa --- /dev/null +++ b/poetry/core/_vendor/lark/tools/standalone.py @@ -0,0 +1,127 @@ +###{standalone +# +# +# Lark Stand-alone Generator Tool +# ---------------------------------- +# Generates a stand-alone LALR(1) parser with a standard lexer +# +# Git: https://github.com/erezsh/lark +# Author: Erez Shinan (erezshin@gmail.com) +# +# +# >>> LICENSE +# +# This tool and its generated code use a separate license from Lark, +# and are subject to the terms of the Mozilla Public License, v. 2.0. +# If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# If you wish to purchase a commercial license for this tool and its +# generated code, you may contact me via email or otherwise. +# +# If MPL2 is incompatible with your free or open-source project, +# contact me and we'll work it out. +# +# + +import os +from io import open +###} + +import codecs +import sys +import os +from pprint import pprint +from os import path +from collections import defaultdict + +import lark +from lark import Lark +from lark.parsers.lalr_analysis import Reduce + + +from lark.grammar import RuleOptions, Rule +from lark.lexer import TerminalDef + +_dir = path.dirname(__file__) +_larkdir = path.join(_dir, path.pardir) + + +EXTRACT_STANDALONE_FILES = [ + 'tools/standalone.py', + 'exceptions.py', + 'utils.py', + 'tree.py', + 'visitors.py', + 'indenter.py', + 'grammar.py', + 'lexer.py', + 'common.py', + 'parse_tree_builder.py', + 'parsers/lalr_parser.py', + 'parsers/lalr_analysis.py', + 'parser_frontends.py', + 'lark.py', +] + +def extract_sections(lines): + section = None + text = [] + sections = defaultdict(list) + for l in lines: + if l.startswith('###'): + if l[3] == '{': + section = l[4:].strip() + elif l[3] == '}': + sections[section] += text + section = None + text = [] + else: + raise ValueError(l) + elif section: + text.append(l) + + return {name:''.join(text) for name, text in sections.items()} + + +def main(fobj, start): + lark_inst = Lark(fobj, parser="lalr", lexer="contextual", start=start) + + print('# The file was automatically generated by Lark v%s' % lark.__version__) + + for pyfile in EXTRACT_STANDALONE_FILES: + with open(os.path.join(_larkdir, pyfile)) as f: + print (extract_sections(f)['standalone']) + + data, m = lark_inst.memo_serialize([TerminalDef, Rule]) + print( 'DATA = (' ) + # pprint(data, width=160) + print(data) + print(')') + print( 'MEMO = (') + print(m) + print(')') + + + print('Shift = 0') + print('Reduce = 1') + print("def Lark_StandAlone(transformer=None, postlex=None):") + print(" return Lark._load_from_dict(DATA, MEMO, transformer=transformer, postlex=postlex)") + + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Lark Stand-alone Generator Tool") + print("Usage: python -m lark.tools.standalone []") + sys.exit(1) + + if len(sys.argv) == 3: + fn, start = sys.argv[1:] + elif len(sys.argv) == 2: + fn, start = sys.argv[1], 'start' + else: + assert False, sys.argv + + with codecs.open(fn, encoding='utf8') as f: + main(f, start) diff --git a/poetry/core/_vendor/lark/tree.py b/poetry/core/_vendor/lark/tree.py new file mode 100644 index 000000000..f9767e43b --- /dev/null +++ b/poetry/core/_vendor/lark/tree.py @@ -0,0 +1,175 @@ +try: + from future_builtins import filter +except ImportError: + pass + +from copy import deepcopy +from collections import OrderedDict + + +###{standalone +class Meta: + def __init__(self): + self.empty = True + +class Tree(object): + def __init__(self, data, children, meta=None): + self.data = data + self.children = children + self._meta = meta + + @property + def meta(self): + if self._meta is None: + self._meta = Meta() + return self._meta + + def __repr__(self): + return 'Tree(%s, %s)' % (self.data, self.children) + + def _pretty_label(self): + return self.data + + def _pretty(self, level, indent_str): + if len(self.children) == 1 and not isinstance(self.children[0], Tree): + return [ indent_str*level, self._pretty_label(), '\t', '%s' % (self.children[0],), '\n'] + + l = [ indent_str*level, self._pretty_label(), '\n' ] + for n in self.children: + if isinstance(n, Tree): + l += n._pretty(level+1, indent_str) + else: + l += [ indent_str*(level+1), '%s' % (n,), '\n' ] + + return l + + def pretty(self, indent_str=' '): + return ''.join(self._pretty(0, indent_str)) + + def __eq__(self, other): + try: + return self.data == other.data and self.children == other.children + except AttributeError: + return False + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash((self.data, tuple(self.children))) + + def iter_subtrees(self): + queue = [self] + subtrees = OrderedDict() + for subtree in queue: + subtrees[id(subtree)] = subtree + queue += [c for c in reversed(subtree.children) + if isinstance(c, Tree) and id(c) not in subtrees] + + del queue + return reversed(list(subtrees.values())) + + def find_pred(self, pred): + "Find all nodes where pred(tree) == True" + return filter(pred, self.iter_subtrees()) + + def find_data(self, data): + "Find all nodes where tree.data == data" + return self.find_pred(lambda t: t.data == data) + +###} + + def expand_kids_by_index(self, *indices): + "Expand (inline) children at the given indices" + for i in sorted(indices, reverse=True): # reverse so that changing tail won't affect indices + kid = self.children[i] + self.children[i:i+1] = kid.children + + def scan_values(self, pred): + for c in self.children: + if isinstance(c, Tree): + for t in c.scan_values(pred): + yield t + else: + if pred(c): + yield c + + def iter_subtrees_topdown(self): + stack = [self] + while stack: + node = stack.pop() + if not isinstance(node, Tree): + continue + yield node + for n in reversed(node.children): + stack.append(n) + + def __deepcopy__(self, memo): + return type(self)(self.data, deepcopy(self.children, memo), meta=self._meta) + + def copy(self): + return type(self)(self.data, self.children) + + def set(self, data, children): + self.data = data + self.children = children + + # XXX Deprecated! Here for backwards compatibility <0.6.0 + @property + def line(self): + return self.meta.line + @property + def column(self): + return self.meta.column + @property + def end_line(self): + return self.meta.end_line + @property + def end_column(self): + return self.meta.end_column + + +class SlottedTree(Tree): + __slots__ = 'data', 'children', 'rule', '_meta' + + +def pydot__tree_to_png(tree, filename, rankdir="LR", **kwargs): + """Creates a colorful image that represents the tree (data+children, without meta) + + Possible values for `rankdir` are "TB", "LR", "BT", "RL", corresponding to + directed graphs drawn from top to bottom, from left to right, from bottom to + top, and from right to left, respectively. + + `kwargs` can be any graph attribute (e. g. `dpi=200`). For a list of + possible attributes, see https://www.graphviz.org/doc/info/attrs.html. + """ + + import pydot + graph = pydot.Dot(graph_type='digraph', rankdir=rankdir, **kwargs) + + i = [0] + + def new_leaf(leaf): + node = pydot.Node(i[0], label=repr(leaf)) + i[0] += 1 + graph.add_node(node) + return node + + def _to_pydot(subtree): + color = hash(subtree.data) & 0xffffff + color |= 0x808080 + + subnodes = [_to_pydot(child) if isinstance(child, Tree) else new_leaf(child) + for child in subtree.children] + node = pydot.Node(i[0], style="filled", fillcolor="#%x"%color, label=subtree.data) + i[0] += 1 + graph.add_node(node) + + for subnode in subnodes: + graph.add_edge(pydot.Edge(node, subnode)) + + return node + + _to_pydot(tree) + graph.write_png(filename) + diff --git a/poetry/core/_vendor/lark/utils.py b/poetry/core/_vendor/lark/utils.py new file mode 100644 index 000000000..36f50d1e4 --- /dev/null +++ b/poetry/core/_vendor/lark/utils.py @@ -0,0 +1,308 @@ +import sys +import os +from functools import reduce +from ast import literal_eval +from collections import deque + +class fzset(frozenset): + def __repr__(self): + return '{%s}' % ', '.join(map(repr, self)) + + +def classify_bool(seq, pred): + true_elems = [] + false_elems = [] + + for elem in seq: + if pred(elem): + true_elems.append(elem) + else: + false_elems.append(elem) + + return true_elems, false_elems + + + +def bfs(initial, expand): + open_q = deque(list(initial)) + visited = set(open_q) + while open_q: + node = open_q.popleft() + yield node + for next_node in expand(node): + if next_node not in visited: + visited.add(next_node) + open_q.append(next_node) + + + + +def _serialize(value, memo): + if isinstance(value, Serialize): + return value.serialize(memo) + elif isinstance(value, list): + return [_serialize(elem, memo) for elem in value] + elif isinstance(value, frozenset): + return list(value) # TODO reversible? + elif isinstance(value, dict): + return {key:_serialize(elem, memo) for key, elem in value.items()} + return value + +###{standalone +def classify(seq, key=None, value=None): + d = {} + for item in seq: + k = key(item) if (key is not None) else item + v = value(item) if (value is not None) else item + if k in d: + d[k].append(v) + else: + d[k] = [v] + return d + + +def _deserialize(data, namespace, memo): + if isinstance(data, dict): + if '__type__' in data: # Object + class_ = namespace[data['__type__']] + return class_.deserialize(data, memo) + elif '@' in data: + return memo[data['@']] + return {key:_deserialize(value, namespace, memo) for key, value in data.items()} + elif isinstance(data, list): + return [_deserialize(value, namespace, memo) for value in data] + return data + + +class Serialize(object): + def memo_serialize(self, types_to_memoize): + memo = SerializeMemoizer(types_to_memoize) + return self.serialize(memo), memo.serialize() + + def serialize(self, memo=None): + if memo and memo.in_types(self): + return {'@': memo.memoized.get(self)} + + fields = getattr(self, '__serialize_fields__') + res = {f: _serialize(getattr(self, f), memo) for f in fields} + res['__type__'] = type(self).__name__ + postprocess = getattr(self, '_serialize', None) + if postprocess: + postprocess(res, memo) + return res + + @classmethod + def deserialize(cls, data, memo): + namespace = getattr(cls, '__serialize_namespace__', {}) + namespace = {c.__name__:c for c in namespace} + + fields = getattr(cls, '__serialize_fields__') + + if '@' in data: + return memo[data['@']] + + inst = cls.__new__(cls) + for f in fields: + try: + setattr(inst, f, _deserialize(data[f], namespace, memo)) + except KeyError as e: + raise KeyError("Cannot find key for class", cls, e) + postprocess = getattr(inst, '_deserialize', None) + if postprocess: + postprocess() + return inst + + +class SerializeMemoizer(Serialize): + __serialize_fields__ = 'memoized', + + def __init__(self, types_to_memoize): + self.types_to_memoize = tuple(types_to_memoize) + self.memoized = Enumerator() + + def in_types(self, value): + return isinstance(value, self.types_to_memoize) + + def serialize(self): + return _serialize(self.memoized.reversed(), None) + + @classmethod + def deserialize(cls, data, namespace, memo): + return _deserialize(data, namespace, memo) + + + +try: + STRING_TYPE = basestring +except NameError: # Python 3 + STRING_TYPE = str + + +import types +from functools import wraps, partial +from contextlib import contextmanager + +Str = type(u'') +try: + classtype = types.ClassType # Python2 +except AttributeError: + classtype = type # Python3 + +def smart_decorator(f, create_decorator): + if isinstance(f, types.FunctionType): + return wraps(f)(create_decorator(f, True)) + + elif isinstance(f, (classtype, type, types.BuiltinFunctionType)): + return wraps(f)(create_decorator(f, False)) + + elif isinstance(f, types.MethodType): + return wraps(f)(create_decorator(f.__func__, True)) + + elif isinstance(f, partial): + # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445 + return wraps(f.func)(create_decorator(lambda *args, **kw: f(*args[1:], **kw), True)) + + else: + return create_decorator(f.__func__.__call__, True) + +try: + import regex +except ImportError: + regex = None + +import sys, re +Py36 = (sys.version_info[:2] >= (3, 6)) + +import sre_parse +import sre_constants +categ_pattern = re.compile(r'\\p{[A-Za-z_]+}') +def get_regexp_width(expr): + if regex: + # Since `sre_parse` cannot deal with Unicode categories of the form `\p{Mn}`, we replace these with + # a simple letter, which makes no difference as we are only trying to get the possible lengths of the regex + # match here below. + regexp_final = re.sub(categ_pattern, 'A', expr) + else: + if re.search(categ_pattern, expr): + raise ImportError('`regex` module must be installed in order to use Unicode categories.', expr) + regexp_final = expr + try: + return [int(x) for x in sre_parse.parse(regexp_final).getwidth()] + except sre_constants.error: + raise ValueError(expr) + +###} + + +def dedup_list(l): + """Given a list (l) will removing duplicates from the list, + preserving the original order of the list. Assumes that + the list entries are hashable.""" + dedup = set() + return [ x for x in l if not (x in dedup or dedup.add(x))] + + + + +try: + from contextlib import suppress # Python 3 +except ImportError: + @contextmanager + def suppress(*excs): + '''Catch and dismiss the provided exception + + >>> x = 'hello' + >>> with suppress(IndexError): + ... x = x[10] + >>> x + 'hello' + ''' + try: + yield + except excs: + pass + + + + +try: + compare = cmp +except NameError: + def compare(a, b): + if a == b: + return 0 + elif a > b: + return 1 + return -1 + + + +class Enumerator(Serialize): + def __init__(self): + self.enums = {} + + def get(self, item): + if item not in self.enums: + self.enums[item] = len(self.enums) + return self.enums[item] + + def __len__(self): + return len(self.enums) + + def reversed(self): + r = {v: k for k, v in self.enums.items()} + assert len(r) == len(self.enums) + return r + + +def eval_escaping(s): + w = '' + i = iter(s) + for n in i: + w += n + if n == '\\': + try: + n2 = next(i) + except StopIteration: + raise ValueError("Literal ended unexpectedly (bad escaping): `%r`" % s) + if n2 == '\\': + w += '\\\\' + elif n2 not in 'uxnftr': + w += '\\' + w += n2 + w = w.replace('\\"', '"').replace("'", "\\'") + + to_eval = "u'''%s'''" % w + try: + s = literal_eval(to_eval) + except SyntaxError as e: + raise ValueError(s, e) + + return s + + +def combine_alternatives(lists): + """ + Accepts a list of alternatives, and enumerates all their possible concatinations. + + Examples: + >>> combine_alternatives([range(2), [4,5]]) + [[0, 4], [0, 5], [1, 4], [1, 5]] + + >>> combine_alternatives(["abc", "xy", '$']) + [['a', 'x', '$'], ['a', 'y', '$'], ['b', 'x', '$'], ['b', 'y', '$'], ['c', 'x', '$'], ['c', 'y', '$']] + + >>> combine_alternatives([]) + [[]] + """ + if not lists: + return [[]] + assert all(l for l in lists), lists + init = [[x] for x in lists[0]] + return reduce(lambda a,b: [i+[j] for i in a for j in b], lists[1:], init) + + + +class FS: + open = open + exists = os.path.exists \ No newline at end of file diff --git a/poetry/core/_vendor/lark/visitors.py b/poetry/core/_vendor/lark/visitors.py new file mode 100644 index 000000000..c9f0e2dd3 --- /dev/null +++ b/poetry/core/_vendor/lark/visitors.py @@ -0,0 +1,399 @@ +from functools import wraps + +from .utils import smart_decorator, combine_alternatives +from .tree import Tree +from .exceptions import VisitError, GrammarError +from .lexer import Token + +###{standalone +from inspect import getmembers, getmro + +class Discard(Exception): + pass + +# Transformers + +class _Decoratable: + @classmethod + def _apply_decorator(cls, decorator, **kwargs): + mro = getmro(cls) + assert mro[0] is cls + libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} + for name, value in getmembers(cls): + + # Make sure the function isn't inherited (unless it's overwritten) + if name.startswith('_') or (name in libmembers and name not in cls.__dict__): + continue + if not callable(value): + continue + + # Skip if v_args already applied (at the function level) + if hasattr(cls.__dict__[name], 'vargs_applied') or hasattr(value, 'vargs_applied'): + continue + + static = isinstance(cls.__dict__[name], (staticmethod, classmethod)) + setattr(cls, name, decorator(value, static=static, **kwargs)) + return cls + + def __class_getitem__(cls, _): + return cls + + +class Transformer(_Decoratable): + """Visits the tree recursively, starting with the leaves and finally the root (bottom-up) + + Calls its methods (provided by user via inheritance) according to tree.data + The returned value replaces the old one in the structure. + + Can be used to implement map or reduce. + """ + __visit_tokens__ = True # For backwards compatibility + + def __init__(self, visit_tokens=True): + self.__visit_tokens__ = visit_tokens + + def _call_userfunc(self, tree, new_children=None): + # Assumes tree is already transformed + children = new_children if new_children is not None else tree.children + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree.data, children, tree.meta) + else: + try: + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + return f.visit_wrapper(f, tree.data, children, tree.meta) + else: + return f(children) + except (GrammarError, Discard): + raise + except Exception as e: + raise VisitError(tree.data, tree, e) + + def _call_userfunc_token(self, token): + try: + f = getattr(self, token.type) + except AttributeError: + return self.__default_token__(token) + else: + try: + return f(token) + except (GrammarError, Discard): + raise + except Exception as e: + raise VisitError(token.type, token, e) + + + def _transform_children(self, children): + for c in children: + try: + if isinstance(c, Tree): + yield self._transform_tree(c) + elif self.__visit_tokens__ and isinstance(c, Token): + yield self._call_userfunc_token(c) + else: + yield c + except Discard: + pass + + def _transform_tree(self, tree): + children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree, children) + + def transform(self, tree): + return self._transform_tree(tree) + + def __mul__(self, other): + return TransformerChain(self, other) + + def __default__(self, data, children, meta): + "Default operation on tree (for override)" + return Tree(data, children, meta) + + def __default_token__(self, token): + "Default operation on token (for override)" + return token + + + +class InlineTransformer(Transformer): # XXX Deprecated + def _call_userfunc(self, tree, new_children=None): + # Assumes tree is already transformed + children = new_children if new_children is not None else tree.children + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree.data, children, tree.meta) + else: + return f(*children) + + +class TransformerChain(object): + def __init__(self, *transformers): + self.transformers = transformers + + def transform(self, tree): + for t in self.transformers: + tree = t.transform(tree) + return tree + + def __mul__(self, other): + return TransformerChain(*self.transformers + (other,)) + + +class Transformer_InPlace(Transformer): + "Non-recursive. Changes the tree in-place instead of returning new instances" + def _transform_tree(self, tree): # Cancel recursion + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform_tree(tree) + + +class Transformer_NonRecursive(Transformer): + "Non-recursive. Doesn't change the original tree." + + def transform(self, tree): + # Tree to postfix + rev_postfix = [] + q = [tree] + while q: + t = q.pop() + rev_postfix.append( t ) + if isinstance(t, Tree): + q += t.children + + # Postfix to tree + stack = [] + for x in reversed(rev_postfix): + if isinstance(x, Tree): + size = len(x.children) + if size: + args = stack[-size:] + del stack[-size:] + else: + args = [] + stack.append(self._call_userfunc(x, args)) + else: + stack.append(x) + + t ,= stack # We should have only one tree remaining + return t + + + +class Transformer_InPlaceRecursive(Transformer): + "Recursive. Changes the tree in-place instead of returning new instances" + def _transform_tree(self, tree): + tree.children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree) + + + +# Visitors + +class VisitorBase: + def _call_userfunc(self, tree): + return getattr(self, tree.data, self.__default__)(tree) + + def __default__(self, tree): + "Default operation on tree (for override)" + return tree + + def __class_getitem__(cls, _): + return cls + + +class Visitor(VisitorBase): + """Bottom-up visitor, non-recursive + + Visits the tree, starting with the leaves and finally the root (bottom-up) + Calls its methods (provided by user via inheritance) according to tree.data + """ + + def visit(self, tree): + for subtree in tree.iter_subtrees(): + self._call_userfunc(subtree) + return tree + + def visit_topdown(self,tree): + for subtree in tree.iter_subtrees_topdown(): + self._call_userfunc(subtree) + return tree + +class Visitor_Recursive(VisitorBase): + """Bottom-up visitor, recursive + + Visits the tree, starting with the leaves and finally the root (bottom-up) + Calls its methods (provided by user via inheritance) according to tree.data + """ + + def visit(self, tree): + for child in tree.children: + if isinstance(child, Tree): + self.visit(child) + + self._call_userfunc(tree) + return tree + + def visit_topdown(self,tree): + self._call_userfunc(tree) + + for child in tree.children: + if isinstance(child, Tree): + self.visit_topdown(child) + + return tree + + + +def visit_children_decor(func): + "See Interpreter" + @wraps(func) + def inner(cls, tree): + values = cls.visit_children(tree) + return func(cls, values) + return inner + + +class Interpreter(_Decoratable): + """Top-down visitor, recursive + + Visits the tree, starting with the root and finally the leaves (top-down) + Calls its methods (provided by user via inheritance) according to tree.data + + Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches. + The user has to explicitly call visit_children, or use the @visit_children_decor + """ + + def visit(self, tree): + f = getattr(self, tree.data) + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + return f.visit_wrapper(f, tree.data, tree.children, tree.meta) + else: + return f(tree) + + def visit_children(self, tree): + return [self.visit(child) if isinstance(child, Tree) else child + for child in tree.children] + + def __getattr__(self, name): + return self.__default__ + + def __default__(self, tree): + return self.visit_children(tree) + + + + +# Decorators + +def _apply_decorator(obj, decorator, **kwargs): + try: + _apply = obj._apply_decorator + except AttributeError: + return decorator(obj, **kwargs) + else: + return _apply(decorator, **kwargs) + + + +def _inline_args__func(func): + @wraps(func) + def create_decorator(_f, with_self): + if with_self: + def f(self, children): + return _f(self, *children) + else: + def f(self, children): + return _f(*children) + return f + + return smart_decorator(func, create_decorator) + + +def inline_args(obj): # XXX Deprecated + return _apply_decorator(obj, _inline_args__func) + + + +def _visitor_args_func_dec(func, visit_wrapper=None, static=False): + def create_decorator(_f, with_self): + if with_self: + def f(self, *args, **kwargs): + return _f(self, *args, **kwargs) + else: + def f(self, *args, **kwargs): + return _f(*args, **kwargs) + return f + + if static: + f = wraps(func)(create_decorator(func, False)) + else: + f = smart_decorator(func, create_decorator) + f.vargs_applied = True + f.visit_wrapper = visit_wrapper + return f + + +def _vargs_inline(f, data, children, meta): + return f(*children) +def _vargs_meta_inline(f, data, children, meta): + return f(meta, *children) +def _vargs_meta(f, data, children, meta): + return f(children, meta) # TODO swap these for consistency? Backwards incompatible! +def _vargs_tree(f, data, children, meta): + return f(Tree(data, children, meta)) + +def v_args(inline=False, meta=False, tree=False, wrapper=None): + "A convenience decorator factory, for modifying the behavior of user-supplied visitor methods" + if tree and (meta or inline): + raise ValueError("Visitor functions cannot combine 'tree' with 'meta' or 'inline'.") + + func = None + if meta: + if inline: + func = _vargs_meta_inline + else: + func = _vargs_meta + elif inline: + func = _vargs_inline + elif tree: + func = _vargs_tree + + if wrapper is not None: + if func is not None: + raise ValueError("Cannot use 'wrapper' along with 'tree', 'meta' or 'inline'.") + func = wrapper + + def _visitor_args_dec(obj): + return _apply_decorator(obj, _visitor_args_func_dec, visit_wrapper=func) + return _visitor_args_dec + + +###} + + +#--- Visitor Utilities --- + +class CollapseAmbiguities(Transformer): + """ + Transforms a tree that contains any number of _ambig nodes into a list of trees, + each one containing an unambiguous tree. + + The length of the resulting list is the product of the length of all _ambig nodes. + + Warning: This may quickly explode for highly ambiguous trees. + + """ + def _ambig(self, options): + return sum(options, []) + def __default__(self, data, children_lists, meta): + return [Tree(data, children, meta) for children in combine_alternatives(children_lists)] + def __default_token__(self, t): + return [t] diff --git a/poetry/core/_vendor/vendor.txt b/poetry/core/_vendor/vendor.txt index 2cb58f5e9..83d314302 100644 --- a/poetry/core/_vendor/vendor.txt +++ b/poetry/core/_vendor/vendor.txt @@ -2,6 +2,7 @@ attrs==19.3.0 jsonschema==3.2.0 packaging==20.3 pyparsing==2.4.6 +lark-parser==0.9.0 pyrsistent==0.16.0 six==1.14.0 tomlkit==0.5.11 \ No newline at end of file diff --git a/poetry/core/semver/__init__.py b/poetry/core/semver/__init__.py index 39ab433d6..e782e3df8 100644 --- a/poetry/core/semver/__init__.py +++ b/poetry/core/semver/__init__.py @@ -1,6 +1,7 @@ import re from .empty_constraint import EmptyConstraint +from .exceptions import ParseConstraintError from .patterns import BASIC_CONSTRAINT from .patterns import CARET_CONSTRAINT from .patterns import TILDE_CONSTRAINT @@ -159,4 +160,6 @@ def parse_single_constraint(constraint): # type: (str) -> VersionConstraint else: return version - raise ValueError("Could not parse version constraint: {}".format(constraint)) + raise ParseConstraintError( + "Could not parse version constraint: {}".format(constraint) + ) diff --git a/poetry/core/semver/exceptions.py b/poetry/core/semver/exceptions.py index f0eab0db7..b24323997 100644 --- a/poetry/core/semver/exceptions.py +++ b/poetry/core/semver/exceptions.py @@ -1,2 +1,6 @@ class ParseVersionError(ValueError): pass + + +class ParseConstraintError(ValueError): + pass diff --git a/poetry/core/version/grammars/__init__.py b/poetry/core/version/grammars/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/poetry/core/version/grammars/markers.lark b/poetry/core/version/grammars/markers.lark new file mode 100644 index 000000000..189ab02a5 --- /dev/null +++ b/poetry/core/version/grammars/markers.lark @@ -0,0 +1,37 @@ +start: marker + +marker: _atom (BOOL_OP _atom)* +_atom: item | (L_PAREN marker R_PAREN) +item: (MARKER_NAME MARKER_OP _marker_value) | (_marker_value MARKER_OP MARKER_NAME) +_marker_value: SINGLE_QUOTED_STRING | ESCAPED_STRING + +MARKER_NAME: "implementation_version" + | "platform_python_implementation" + | "implementation_name" + | "python_full_version" + | "platform_release" + | "platform_version" + | "platform_machine" + | "platform_system" + | "python_version" + | "sys_platform" + | "sys_platform" + | "os_name" + | "os.name" + | "sys.platform" + | "platform.version" + | "platform.machine" + | "platform.python_implementation" + | "python_implementation" + | "extra" +MARKER_OP: "===" | "==" | ">=" | "<=" | ">" | "<" | "!=" | "~=" | "not in" | "in" +SINGLE_QUOTED_STRING: /'([^'])*'/ +QUOTED_STRING: /"([^"])*"/ +MARKER_VALUE: /(.+?)/ +BOOL_OP: "and" | "or" +L_PAREN: "(" +R_PAREN: ")" + +%import common.WS_INLINE +%import common.ESCAPED_STRING +%ignore WS_INLINE diff --git a/poetry/core/version/grammars/pep508.lark b/poetry/core/version/grammars/pep508.lark new file mode 100644 index 000000000..0057fb282 --- /dev/null +++ b/poetry/core/version/grammars/pep508.lark @@ -0,0 +1,29 @@ +start: _requirement + +_requirement: _full_name (_MARKER_SEPARATOR marker_spec)? +_full_name: NAME _extras? (version_specification | _url)? +_extras: _L_BRACKET _extra? _R_BRACKET +_extra: EXTRA (_COMMA EXTRA)* +version_specification: (_version_many | _L_PAREN _version_many _R_PAREN) +_version_many: _single_version (_COMMA _single_version)* +_single_version: LEGACY_VERSION_CONSTRAINT +_url: _AT URI +marker_spec: marker + +NAME: /[a-zA-Z][a-zA-Z0-9-_.]*/ +FULL_NAME: NAME +EXTRA: NAME +VERSION_CONSTRAINT: /(~=|==|!=|<=|>=|<|>|===)((?:(?<====)\s*[^\s]*)|(?:(?<===|!=)\s*v?(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)*(?:[-_.]?(a|b|c|rc|alpha|beta|pre|preview)[-_.]?[0-9]*)?(?:(?:-[0-9]+)|(?:[-_.]?(post|rev|r)[-_.]?[0-9]*))?(?:(?:[-_.]?dev[-_.]?[0-9]*)?(?:\+[a-z0-9]+(?:[-_.][a-z0-9]+)*)? # local|\.\*)?)|(?:(?<=~=)\s*v?(?:[0-9]+!)?[0-9]+(?:\.[0-9]+)+(?:[-_.]?(a|b|c|rc|alpha|beta|pre|preview)[-_.]?[0-9]*)?(?:(?:-[0-9]+)|(?:[-_.]?(post|rev|r)[-_.]?[0-9]*))?(?:[-_.]?dev[-_.]?[0-9]*)?)|(?:(?=|<|>)\s*[^,;\s)]*/i +URI: /[^ ]+/ +_MARKER_SEPARATOR: ";" +_L_PAREN: "(" +_R_PAREN: ")" +_L_BRACKET: "[" +_R_BRACKET: "]" +_COMMA: "," +_AT: "@" + +%import .markers.marker +%import common.WS_INLINE +%ignore WS_INLINE diff --git a/poetry/core/version/markers.py b/poetry/core/version/markers.py index 9cb4a00c7..3c2f2de19 100644 --- a/poetry/core/version/markers.py +++ b/poetry/core/version/markers.py @@ -1,3 +1,4 @@ +import os import re from typing import Any @@ -5,14 +6,9 @@ from typing import Iterator from typing import List -from pyparsing import Forward -from pyparsing import Group -from pyparsing import Literal as L # noqa -from pyparsing import ParseResults -from pyparsing import QuotedString -from pyparsing import ZeroOrMore -from pyparsing import stringEnd -from pyparsing import stringStart +from lark import Lark +from lark import Token +from lark import Tree class InvalidMarker(ValueError): @@ -34,55 +30,6 @@ class UndefinedEnvironmentName(ValueError): """ -class Node(object): - def __init__(self, value): - self.value = value - - def __str__(self): - return str(self.value) - - def __repr__(self): - return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) - - def serialize(self): - raise NotImplementedError - - -class Variable(Node): - def serialize(self): - return str(self) - - -class Value(Node): - def serialize(self): - return '"{0}"'.format(self) - - -class Op(Node): - def serialize(self): - return str(self) - - -VARIABLE = ( - L("implementation_version") - | L("platform_python_implementation") - | L("implementation_name") - | L("python_full_version") - | L("platform_release") - | L("platform_version") - | L("platform_machine") - | L("platform_system") - | L("python_version") - | L("sys_platform") - | L("os_name") - | L("os.name") - | L("sys.platform") # PEP-345 - | L("platform.version") # PEP-345 - | L("platform.machine") # PEP-345 - | L("platform.python_implementation") # PEP-345 - | L("python_implementation") # PEP-345 - | L("extra") # undocumented setuptools legacy -) ALIASES = { "os.name": "os_name", "sys.platform": "sys_platform", @@ -91,70 +38,10 @@ def serialize(self): "platform.python_implementation": "platform_python_implementation", "python_implementation": "platform_python_implementation", } -VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) - -VERSION_CMP = ( - L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") +_parser = Lark.open( + os.path.join(os.path.dirname(__file__), "grammars", "markers.lark"), parser="lalr" ) -MARKER_OP = VERSION_CMP | L("not in") | L("in") -MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) - -MARKER_VALUE = QuotedString("'") | QuotedString('"') -MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) - -BOOLOP = L("and") | L("or") - -MARKER_VAR = VARIABLE | MARKER_VALUE - -MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) -MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) - -LPAREN = L("(").suppress() -RPAREN = L(")").suppress() - -MARKER_EXPR = Forward() -MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) -MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) - -MARKER = stringStart + MARKER_EXPR + stringEnd - - -_undefined = object() - - -def _coerce_parse_result(results): - if isinstance(results, ParseResults): - return [_coerce_parse_result(i) for i in results] - else: - return results - - -def _format_marker(marker, first=True): - assert isinstance(marker, (list, tuple, str)) - - # Sometimes we have a structure like [[...]] which is a single item list - # where the single item is itself it's own list. In that case we want skip - # the rest of this function so that we don't get extraneous () on the - # outside. - if ( - isinstance(marker, list) - and len(marker) == 1 - and isinstance(marker[0], (list, tuple)) - ): - return _format_marker(marker[0]) - - if isinstance(marker, list): - inner = (_format_marker(m, first=False) for m in marker) - if first: - return " ".join(inner) - else: - return "(" + " ".join(inner) + ")" - elif isinstance(marker, tuple): - return " ".join([m.serialize() for m in marker]) - else: - return marker - class BaseMarker(object): def intersect(self, other): # type: (BaseMarker) -> BaseMarker @@ -291,7 +178,7 @@ def __init__(self, name, constraint): ) from poetry.core.semver import parse_constraint - self._name = name + self._name = ALIASES.get(name, name) self._constraint_string = str(constraint) # Extract operator and value @@ -461,9 +348,7 @@ def __hash__(self): return hash((self._name, self._constraint_string)) def __str__(self): - return _format_marker( - (Variable(self._name), Op(self._operator), Value(self._value)) - ) + return '{} {} "{}"'.format(self._name, self._operator, self._value) def _flatten_markers( @@ -669,16 +554,6 @@ def of(cls, *markers): # type: (BaseMarker) -> MarkerUnion if any(m.is_any() for m in markers): return AnyMarker() - to_delete_indices = set() - for i, marker in enumerate(markers): - for j, m in enumerate(markers): - if m.invert() == marker: - to_delete_indices.add(i) - to_delete_indices.add(j) - - for idx in reversed(sorted(to_delete_indices)): - del markers[idx] - if not markers: return AnyMarker() @@ -805,32 +680,37 @@ def parse_marker(marker): if not marker or marker == "*": return AnyMarker() - markers = _coerce_parse_result(MARKER.parseString(marker)) + parsed = _parser.parse(marker) + + markers = _compact_markers(parsed.children) - return _compact_markers(markers) + return markers -def _compact_markers(markers): +def _compact_markers(tree_elements, tree_prefix=""): # type: (Tree, str) -> BaseMarker groups = [MultiMarker()] + for token in tree_elements: + if isinstance(token, Token): + if token.type == "{}BOOL_OP".format(tree_prefix) and token.value == "or": + groups.append(MultiMarker()) - for marker in markers: - if isinstance(marker, list): - groups[-1] = MultiMarker.of(groups[-1], _compact_markers(marker)) - elif isinstance(marker, tuple): - lhs, op, rhs = marker - - if isinstance(lhs, Variable): - name = lhs.value - value = rhs.value - else: - value = lhs.value - name = rhs.value + continue + if token.data == "marker": + groups[-1] = MultiMarker.of( + groups[-1], _compact_markers(token.children, tree_prefix=tree_prefix) + ) + elif token.data == "{}item".format(tree_prefix): + name, op, value = token.children + if value.type == "{}MARKER_NAME".format(tree_prefix): + name, value, = value, name + + value = value[1:-1] groups[-1] = MultiMarker.of( groups[-1], SingleMarker(name, "{}{}".format(op, value)) ) - else: - if marker == "or": + elif token.data == "{}BOOL_OP".format(tree_prefix): + if token.children[0] == "or": groups.append(MultiMarker()) for i, group in enumerate(reversed(groups)): diff --git a/poetry/core/version/requirements.py b/poetry/core/version/requirements.py index 60b8f7546..3ba352825 100644 --- a/poetry/core/version/requirements.py +++ b/poetry/core/version/requirements.py @@ -5,23 +5,15 @@ from __future__ import division from __future__ import print_function -import re -import string +import os +from lark import Lark +from lark import UnexpectedCharacters +from lark import UnexpectedToken from poetry.core.semver import parse_constraint -from pyparsing import Combine -from pyparsing import Literal as L # noqa -from pyparsing import Optional -from pyparsing import ParseException -from pyparsing import Regex -from pyparsing import Word -from pyparsing import ZeroOrMore -from pyparsing import originalTextFor -from pyparsing import stringEnd -from pyparsing import stringStart +from poetry.core.semver.exceptions import ParseConstraintError -from .markers import MARKER_EXPR -from .markers import parse_marker +from .markers import _compact_markers try: @@ -30,172 +22,20 @@ import urlparse -LEGACY_REGEX = r""" - (?P(==|!=|<=|>=|<|>)) - \s* - (?P - [^,;\s)]* # Since this is a "legacy" specifier, and the version - # string can be just about anything, we match everything - # except for whitespace, a semi-colon for marker support, - # a closing paren since versions can be enclosed in - # them, and a comma since it's a version separator. - ) - """ - - -REGEX = r""" - (?P(~=|==|!=|<=|>=|<|>|===)) - (?P - (?: - # The identity operators allow for an escape hatch that will - # do an exact string match of the version you wish to install. - # This will not be parsed by PEP 440 and we cannot determine - # any semantic meaning from it. This operator is discouraged - # but included entirely as an escape hatch. - (?<====) # Only match for the identity operator - \s* - [^\s]* # We just match everything, except for whitespace - # since we are only testing for strict identity. - ) - | - (?: - # The (non)equality operators allow for wild card and local - # versions to be specified so we have to define these two - # operators separately to enable that. - (?<===|!=) # Only match for equals and not equals - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)* # release - (?: # pre release - [-_\.]? - (a|b|c|rc|alpha|beta|pre|preview) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - - # You cannot use a wild card and a dev or local version - # together so group them with a | and make them optional. - (?: - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local - | - \.\* # Wild card syntax of .* - )? - ) - | - (?: - # The compatible operator requires at least two digits in the - # release segment. - (?<=~=) # Only match for the compatible operator - - \s* - v? - (?:[0-9]+!)? # epoch - [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) - (?: # pre release - [-_\.]? - (a|b|c|rc|alpha|beta|pre|preview) - [-_\.]? - [0-9]* - )? - (?: # post release - (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) - )? - (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release - ) - | - (?: - # All other operators only allow a sub set of what the - # (non)equality operators do. Specifically they do not allow - # local versions to be specified nor do they allow the prefix - # matching wild cards. - (?=3.5, <3.8"} pathlib2 = {version = "^2.3.5", python = "~2.7"} typing = {version = "^3.7.4.1", python = "~2.7"} @@ -45,7 +46,7 @@ pytest = "^4.6" pytest-cov = "^2.8.1" pytest-mock = "^2.0.0" tox = "^3.0" -cleo = "^0.7.6" +cleo = "^0.8.1" tomlkit = "^0.5.8" vendoring = {version = "^0.2.2", python = "~3.8"} isort = "^4.3.21" diff --git a/stanza b/stanza index b6a186deb..a0469c38e 100755 --- a/stanza +++ b/stanza @@ -6,6 +6,7 @@ import zipfile from pathlib import Path from typing import Dict +from typing import List from cleo import Application from cleo import Command @@ -15,7 +16,10 @@ from vendoring.configuration import load_configuration from vendoring.tasks.cleanup import cleanup_existing_vendored from vendoring.tasks.license import find_and_extract_license from vendoring.tasks.license import license_fallback -from vendoring.tasks.vendor import vendor_libraries +from vendoring.tasks.vendor import apply_patches +from vendoring.tasks.vendor import detect_vendored_libs +from vendoring.tasks.vendor import download_libraries +from vendoring.tasks.vendor import remove_unnecessary_items from vendoring.utils import remove_all from vendoring.utils import run @@ -71,6 +75,24 @@ def fetch_licenses(config: Configuration) -> None: remove_all([tmp_dir]) +def vendor_libraries(config: Configuration) -> List[str]: + destination = config.destination + + # Download the relevant libraries. + download_libraries(config.requirements, destination) + + # Cleanup unnecessary directories/files created. + remove_unnecessary_items(destination, config.drop_paths) + + # Detect what got downloaded. + vendored_libs = detect_vendored_libs(destination, config.protected_files) + + # Apply user provided patches. + apply_patches(config.patches_dir, working_directory=config.base_directory) + + return vendored_libs + + def download_sources(location: Path, requirements: Path) -> None: cmd = [ "pip", @@ -130,8 +152,10 @@ class VendorUpdateCommand(Command): if line.startswith("wheels/"): line = "vendors/" + line - if line.startswith("enum34"): - line = "vendors/wheels/enum34-1.1.10-py2.py3-none-any.whl" + if line.startswith( + ("enum34", "functools32", "pathlib2", "typing", "scandir", "typing") + ): + continue lines.append(line.strip()) diff --git a/tests/version/test_markers.py b/tests/version/test_markers.py index f6c6f9520..b1deaa968 100644 --- a/tests/version/test_markers.py +++ b/tests/version/test_markers.py @@ -651,11 +651,6 @@ def test_invert(marker, inverse): @pytest.mark.parametrize( "marker, expected", [ - ('python_version >= "3.6" or python_version < "3.6"', "*"), - ( - 'python_version >= "3.6" or implementation_name == "pypy" or python_version < "3.6"', - 'implementation_name == "pypy"', - ), ( 'python_version >= "3.6" or python_version < "3.7" or python_version < "3.6"', 'python_version >= "3.6" or python_version < "3.7"', diff --git a/tests/version/test_requirements.py b/tests/version/test_requirements.py new file mode 100644 index 000000000..e6202b44a --- /dev/null +++ b/tests/version/test_requirements.py @@ -0,0 +1,104 @@ +import re + +import pytest + +from poetry.core.semver import parse_constraint +from poetry.core.version.requirements import InvalidRequirement +from poetry.core.version.requirements import Requirement + + +def assert_requirement(req, name, url=None, extras=None, constraint="*", marker=None): + if extras is None: + extras = [] + + assert name == req.name + assert url == req.url + assert sorted(extras) == sorted(req.extras) + assert parse_constraint(constraint) == req.constraint + + if marker: + assert marker == str(req.marker) + + +@pytest.mark.parametrize( + ["string", "expected"], + [ + ("A", {"name": "A"}), + ("aa", {"name": "aa"}), + ("name", {"name": "name"}), + ("foo-bar.quux_baz", {"name": "foo-bar.quux_baz"}), + ("name>=3", {"name": "name", "constraint": ">=3"}), + ("name==1.0.org1", {"name": "name", "constraint": "==1.0.org1"}), + ( + "name>=1.x.y;python_version=='2.6'", + { + "name": "name", + "constraint": ">=1.x.y", + "marker": 'python_version == "2.6"', + }, + ), + ("name (==4)", {"name": "name", "constraint": "==4"}), + ("name>=2,<3", {"name": "name", "constraint": ">=2,<3"}), + ("name >=2, <3", {"name": "name", "constraint": ">=2,<3"}), + # Extras + ("foobar [quux,bar]", {"name": "foobar", "extras": ["quux", "bar"]}), + ("foo[]", {"name": "foo"}), + # Url + ("foo @ http://example.com", {"name": "foo", "url": "http://example.com"}), + ( + 'foo @ http://example.com ; os_name=="a"', + {"name": "foo", "url": "http://example.com", "marker": 'os_name == "a"'}, + ), + ( + "name @ file:///absolute/path", + {"name": "name", "url": "file:///absolute/path"}, + ), + ("name @ file://.", {"name": "name", "url": "file://."},), + ( + "name [fred,bar] @ http://foo.com ; python_version=='2.7'", + { + "name": "name", + "url": "http://foo.com", + "extras": ["fred", "bar"], + "marker": 'python_version == "2.7"', + }, + ), + ( + "foo @ https://example.com/name;v=1.1/?query=foo&bar=baz#blah ; python_version=='3.4'", + { + "name": "foo", + "url": "https://example.com/name;v=1.1/?query=foo&bar=baz#blah", + "marker": 'python_version == "3.4"', + }, + ), + ( + 'foo (>=1.2.3) ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "3.5"', + { + "name": "foo", + "constraint": ">=1.2.3", + "marker": 'python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "3.5"', + }, + ), + ], +) +def test_requirement(string, expected): + req = Requirement(string) + + assert_requirement(req, **expected) + + +@pytest.mark.parametrize( + ["string", "exception"], + [ + ("foo!", "Unexpected character at column 4\n\nfoo!\n ^\n"), + ("foo (>=bar)", 'invalid version constraint ">=bar"'), + ("name @ file:.", "invalid URL"), + ("name @ file:/.", "invalid URL"), + ], +) +def test_invalid_requirement(string, exception): + with pytest.raises( + InvalidRequirement, + match=re.escape("The requirement is invalid: {}".format(exception)), + ): + Requirement(string) diff --git a/vendors/poetry.lock b/vendors/poetry.lock index dc8d77f27..83b345b6c 100644 --- a/vendors/poetry.lock +++ b/vendors/poetry.lock @@ -12,73 +12,20 @@ dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.int docs = ["sphinx", "zope.interface"] tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -[[package]] -category = "main" -description = "Updated configparser from Python 3.7 for Python 2.6+." -marker = "python_version < \"3\"" -name = "configparser" -optional = false -python-versions = ">=2.6" -version = "4.0.2" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2)", "pytest-flake8", "pytest-black-multipy"] - -[[package]] -category = "main" -description = "Backports and enhancements for the contextlib module" -marker = "python_version < \"3.4\"" -name = "contextlib2" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.6.0.post1" - -[[package]] -category = "main" -description = "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4" -marker = "python_version >= \"2.7\" and python_version < \"2.8\"" -name = "enum34" -optional = false -python-versions = "*" -version = "1.1.10" - -[[package]] -category = "main" -description = "Backport of the functools module from Python 3.2.3 for use on 2.7 and PyPy." -marker = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version < \"3\"" -name = "functools32" -optional = false -python-versions = "*" -version = "3.2.3-2" - [[package]] category = "main" description = "Read metadata from Python packages" -marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.0" +version = "1.7.0" [package.dependencies] zipp = ">=0.5" -[package.dependencies.configparser] -python = "<3" -version = ">=3.5" - -[package.dependencies.contextlib2] -python = "<3" -version = "*" - -[package.dependencies.pathlib2] -python = "<3" -version = "*" - [package.extras] docs = ["sphinx", "rst.linker"] -testing = ["packaging", "importlib-resources"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] [[package]] category = "main" @@ -91,16 +38,8 @@ version = "3.2.0" [package.dependencies] attrs = ">=17.4.0" pyrsistent = ">=0.14.0" -setuptools = "*" six = ">=1.11.0" - -[package.dependencies.functools32] -python = "<3" -version = "*" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = "*" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] @@ -108,32 +47,27 @@ format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator [[package]] category = "main" -description = "Core utilities for Python packages" -name = "packaging" +description = "a modern parsing library" +name = "lark-parser" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "20.3" +python-versions = "*" +version = "0.9.0" -[package.dependencies] -pyparsing = ">=2.0.2" -six = "*" +[package.extras] +regex = ["regex"] [[package]] category = "main" -description = "Object-oriented filesystem paths" -marker = "python_version < \"3\"" -name = "pathlib2" +description = "Core utilities for Python packages" +name = "packaging" optional = false -python-versions = "*" -version = "2.3.5" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.4" [package.dependencies] +pyparsing = ">=2.0.2" six = "*" -[package.dependencies.scandir] -python = "<3.5" -version = "*" - [[package]] category = "main" description = "Python parsing module" @@ -153,22 +87,13 @@ version = "0.16.0" [package.dependencies] six = "*" -[[package]] -category = "main" -description = "scandir, a better directory iterator and faster os.walk()" -marker = "python_version < \"3\"" -name = "scandir" -optional = false -python-versions = "*" -version = "1.10.0" - [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" +version = "1.15.0" [[package]] category = "main" @@ -178,87 +103,42 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.5.11" -[package.dependencies] -[package.dependencies.enum34] -python = ">=2.7,<2.8" -version = ">=1.1,<2.0" - -[package.dependencies.functools32] -python = ">=2.7,<2.8" -version = ">=3.2.3,<4.0.0" - -[package.dependencies.typing] -python = ">=2.7,<2.8 || >=3.4,<3.5" -version = ">=3.6,<4.0" - -[[package]] -category = "main" -description = "Type Hints for Python" -marker = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\"" -name = "typing" -optional = false -python-versions = "*" -version = "3.7.4.1" - [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false -python-versions = ">=2.7" -version = "1.2.0" - -[package.dependencies] -[package.dependencies.contextlib2] -python = "<3.4" -version = "*" +python-versions = ">=3.6" +version = "3.1.0" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] +testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "544ee175948bc259dadaa2826179e94f1b4b2655ede5a90bc7753c1cbd98afae" -python-versions = "~2.7 || ^3.5" +content-hash = "26d70290fa26bccd817aa7a67977d20d76d8dc34f80479787fd21c7692bd898e" +lock-version = "1.0" +python-versions = "^3.6" [metadata.files] attrs = [ {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, ] -configparser = [ - {file = "configparser-4.0.2-py2.py3-none-any.whl", hash = "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c"}, - {file = "configparser-4.0.2.tar.gz", hash = "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df"}, -] -contextlib2 = [ - {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, - {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, -] -enum34 = [ - {file = "enum34-1.1.10-py2-none-any.whl", hash = "sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53"}, - {file = "enum34-1.1.10-py3-none-any.whl", hash = "sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328"}, - {file = "enum34-1.1.10.tar.gz", hash = "sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248"}, -] -functools32 = [ - {file = "functools32-3.2.3-2.tar.gz", hash = "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d"}, - {file = "functools32-3.2.3-2.zip", hash = "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0"}, -] importlib-metadata = [ - {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"}, - {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, + {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, + {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, ] jsonschema = [ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, ] -packaging = [ - {file = "packaging-20.3-py2.py3-none-any.whl", hash = "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752"}, - {file = "packaging-20.3.tar.gz", hash = "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3"}, +lark-parser = [ + {file = "lark-parser-0.9.0.tar.gz", hash = "sha256:9e7589365d6b6de1cca40b0eaec31104a3fb96a37a11a9dfd5098e95b50aa6cd"}, ] -pathlib2 = [ - {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"}, - {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"}, +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, @@ -267,33 +147,15 @@ pyparsing = [ pyrsistent = [ {file = "pyrsistent-0.16.0.tar.gz", hash = "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3"}, ] -scandir = [ - {file = "scandir-1.10.0-cp27-cp27m-win32.whl", hash = "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188"}, - {file = "scandir-1.10.0-cp27-cp27m-win_amd64.whl", hash = "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"}, - {file = "scandir-1.10.0-cp34-cp34m-win32.whl", hash = "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f"}, - {file = "scandir-1.10.0-cp34-cp34m-win_amd64.whl", hash = "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e"}, - {file = "scandir-1.10.0-cp35-cp35m-win32.whl", hash = "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f"}, - {file = "scandir-1.10.0-cp35-cp35m-win_amd64.whl", hash = "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32"}, - {file = "scandir-1.10.0-cp36-cp36m-win32.whl", hash = "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022"}, - {file = "scandir-1.10.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4"}, - {file = "scandir-1.10.0-cp37-cp37m-win32.whl", hash = "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173"}, - {file = "scandir-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d"}, - {file = "scandir-1.10.0.tar.gz", hash = "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae"}, -] six = [ - {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, - {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] tomlkit = [ {file = "tomlkit-0.5.11-py2.py3-none-any.whl", hash = "sha256:4e1bd6c9197d984528f9ff0cc9db667c317d8881288db50db20eeeb0f6b0380b"}, {file = "tomlkit-0.5.11.tar.gz", hash = "sha256:f044eda25647882e5ef22b43a1688fb6ab12af2fc50e8456cdfc751c873101cf"}, ] -typing = [ - {file = "typing-3.7.4.1-py2-none-any.whl", hash = "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36"}, - {file = "typing-3.7.4.1-py3-none-any.whl", hash = "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714"}, - {file = "typing-3.7.4.1.tar.gz", hash = "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23"}, -] zipp = [ - {file = "zipp-1.2.0-py2.py3-none-any.whl", hash = "sha256:e0d9e63797e483a30d27e09fffd308c59a700d365ec34e93cc100844168bf921"}, - {file = "zipp-1.2.0.tar.gz", hash = "sha256:c70410551488251b0fee67b460fb9a536af8d6f9f008ad10ac51f615b6a521b1"}, + {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"}, + {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"}, ] diff --git a/vendors/pyproject.toml b/vendors/pyproject.toml index 029407cb2..098847758 100644 --- a/vendors/pyproject.toml +++ b/vendors/pyproject.toml @@ -19,9 +19,9 @@ classifiers = [ ] [tool.poetry.dependencies] -python = "~2.7 || ^3.5" +python = "^3.6" jsonschema = "^3.2.0" +lark-parser = "^0.9.0" packaging = "^20.1" -pyparsing = "^2.4.6" tomlkit = "^0.5.11"