-
Notifications
You must be signed in to change notification settings - Fork 116
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Major changes to interpolations and resolvers (#445)
* Add a grammar to parse interpolations * Add support for nested interpolations * Deprecate register_resolver() and introduce new_register_resolver() (that allows resolvers to use non-string arguments, and to decide whether or not to use the cache) * The `env` resolver now parses environment variables in a way that is consistent with the new interpolation grammar Fixes #100 #230 #266 #318
1 parent
2cf4e7c
commit 499de6b
Showing
54 changed files
with
2,751 additions
and
436 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,47 +1,41 @@ | ||
version: 2.1 | ||
|
||
jobs: | ||
py36_linux: | ||
docker: | ||
- image: circleci/python:3.6 | ||
steps: | ||
- checkout | ||
- run: echo 'export NOX_PYTHON_VERSIONS=3.6' >> $BASH_ENV | ||
- run: sudo pip install nox | ||
- run: nox --add-timestamp | ||
|
||
py37_linux: | ||
docker: | ||
- image: circleci/python:3.7 | ||
commands: | ||
linux: | ||
description: "Commands run on Linux" | ||
parameters: | ||
py_version: | ||
type: string | ||
steps: | ||
- checkout | ||
- run: echo 'export NOX_PYTHON_VERSIONS=3.7' >> $BASH_ENV | ||
- run: sudo pip install nox | ||
- run: nox --add-timestamp | ||
- run: | ||
name: "Preparing environment" | ||
command: | | ||
sudo apt-get update | ||
sudo apt-get install -y openjdk-11-jre | ||
sudo pip install nox | ||
- run: | ||
name: "Testing OmegaConf" | ||
command: | | ||
export NOX_PYTHON_VERSIONS=<< parameters.py_version >> | ||
nox --add-timestamp | ||
py38_linux: | ||
docker: | ||
- image: circleci/python:3.8 | ||
steps: | ||
- checkout | ||
- run: echo 'export NOX_PYTHON_VERSIONS=3.8' >> $BASH_ENV | ||
- run: sudo pip install nox | ||
- run: nox --add-timestamp | ||
|
||
py39_linux: | ||
jobs: | ||
test_linux: | ||
parameters: | ||
py_version: | ||
type: string | ||
docker: | ||
- image: circleci/python:3.9 | ||
- image: circleci/python:<< parameters.py_version >> | ||
steps: | ||
- checkout | ||
- run: echo 'export NOX_PYTHON_VERSIONS=3.9' >> $BASH_ENV | ||
- run: sudo pip install nox | ||
- run: nox --add-timestamp | ||
- linux: | ||
py_version: << parameters.py_version >> | ||
|
||
workflows: | ||
version: 2 | ||
build: | ||
jobs: | ||
- py36_linux | ||
- py37_linux | ||
- py38_linux | ||
- py39_linux | ||
- test_linux: | ||
matrix: | ||
parameters: | ||
py_version: ["3.6", "3.7", "3.8", "3.9"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
[flake8] | ||
exclude = .git,.nox,.tox | ||
exclude = .git,.nox,.tox,omegaconf/grammar/gen | ||
max-line-length = 119 | ||
select = E,F,W,C | ||
ignore=W503,E203 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
*.jar binary |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Order of imports is important (see warning otherwise when running tests) | ||
import setuptools # isort:skip # noqa | ||
import distutils # isort:skip # noqa |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,195 @@ | ||
import codecs | ||
import distutils.log | ||
import errno | ||
import os | ||
import re | ||
import shutil | ||
import subprocess | ||
import sys | ||
from pathlib import Path | ||
from typing import List, Optional | ||
|
||
from setuptools import Command | ||
from setuptools.command import build_py, develop, sdist # type: ignore | ||
|
||
|
||
class ANTLRCommand(Command): # type: ignore # pragma: no cover | ||
"""Generate parsers using ANTLR.""" | ||
|
||
description = "Run ANTLR" | ||
user_options: List[str] = [] | ||
|
||
def run(self) -> None: | ||
"""Run command.""" | ||
build_dir = Path(__file__).parent.absolute() | ||
project_root = build_dir.parent | ||
for grammar in [ | ||
"OmegaConfGrammarLexer.g4", | ||
"OmegaConfGrammarParser.g4", | ||
]: | ||
command = [ | ||
"java", | ||
"-jar", | ||
str(build_dir / "bin" / "antlr-4.8-complete.jar"), | ||
"-Dlanguage=Python3", | ||
"-o", | ||
str(project_root / "omegaconf" / "grammar" / "gen"), | ||
"-Xexact-output-dir", | ||
"-visitor", | ||
str(project_root / "omegaconf" / "grammar" / grammar), | ||
] | ||
|
||
self.announce( | ||
f"Generating parser for Python3: {command}", | ||
level=distutils.log.INFO, | ||
) | ||
|
||
subprocess.check_call(command) | ||
|
||
def initialize_options(self) -> None: | ||
pass | ||
|
||
def finalize_options(self) -> None: | ||
pass | ||
|
||
|
||
class BuildPyCommand(build_py.build_py): # type: ignore # pragma: no cover | ||
def run(self) -> None: | ||
if not self.dry_run: | ||
self.run_command("clean") | ||
run_antlr(self) | ||
build_py.build_py.run(self) | ||
|
||
|
||
class CleanCommand(Command): # type: ignore # pragma: no cover | ||
""" | ||
Our custom command to clean out junk files. | ||
""" | ||
|
||
description = "Cleans out generated and junk files we don't want in the repo" | ||
dry_run: bool | ||
user_options: List[str] = [] | ||
|
||
def run(self) -> None: | ||
root = Path(__file__).parent.parent.absolute() | ||
files = find( | ||
root=root, | ||
include_files=["^omegaconf/grammar/gen/.*"], | ||
include_dirs=[ | ||
"^omegaconf\\.egg-info$", | ||
"\\.eggs$", | ||
"^\\.mypy_cache$", | ||
"^\\.nox$", | ||
"^\\.pytest_cache$", | ||
".*/__pycache__$", | ||
"^__pycache__$", | ||
"^build$", | ||
"^dist$", | ||
], | ||
scan_exclude=["^.git$", "^.nox/.*$"], | ||
excludes=[".*\\.gitignore$", ".*/__init__.py"], | ||
) | ||
|
||
if self.dry_run: | ||
print("Dry run! Would clean up the following files and dirs:") | ||
print("\n".join(sorted(map(str, files)))) | ||
else: | ||
for f in files: | ||
if f.exists(): | ||
if f.is_dir(): | ||
shutil.rmtree(f, ignore_errors=True) | ||
else: | ||
f.unlink() | ||
|
||
def initialize_options(self) -> None: | ||
pass | ||
|
||
def finalize_options(self) -> None: | ||
pass | ||
|
||
|
||
class DevelopCommand(develop.develop): # type: ignore # pragma: no cover | ||
def run(self) -> None: | ||
if not self.dry_run: | ||
run_antlr(self) | ||
develop.develop.run(self) | ||
|
||
|
||
class SDistCommand(sdist.sdist): # type: ignore # pragma: no cover | ||
def run(self) -> None: | ||
if not self.dry_run: | ||
self.run_command("clean") | ||
run_antlr(self) | ||
sdist.sdist.run(self) | ||
|
||
|
||
def find( | ||
root: Path, | ||
include_files: List[str], | ||
include_dirs: List[str], | ||
excludes: List[str], | ||
rbase: Optional[Path] = None, | ||
scan_exclude: Optional[List[str]] = None, | ||
) -> List[Path]: | ||
if rbase is None: | ||
rbase = Path() | ||
if scan_exclude is None: | ||
scan_exclude = [] | ||
files = [] | ||
scan_root = root / rbase | ||
for entry in scan_root.iterdir(): | ||
path = rbase / entry.name | ||
if matches(scan_exclude, path): | ||
continue | ||
|
||
if entry.is_dir(): | ||
if matches(include_dirs, path): | ||
if not matches(excludes, path): | ||
files.append(path) | ||
else: | ||
ret = find( | ||
root=root, | ||
include_files=include_files, | ||
include_dirs=include_dirs, | ||
excludes=excludes, | ||
rbase=path, | ||
scan_exclude=scan_exclude, | ||
) | ||
files.extend(ret) | ||
else: | ||
if matches(include_files, path) and not matches(excludes, path): | ||
files.append(path) | ||
|
||
return files | ||
|
||
|
||
def find_version(*file_paths: str) -> str: | ||
root = Path(__file__).parent.parent.absolute() | ||
with codecs.open(root / Path(*file_paths), "r") as fp: # type: ignore | ||
version_file = fp.read() | ||
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) | ||
if version_match: | ||
return version_match.group(1) | ||
raise RuntimeError("Unable to find version string.") # pragma: no cover | ||
|
||
|
||
def matches(patterns: List[str], path: Path) -> bool: | ||
string = str(path).replace(os.sep, "/") # for Windows | ||
for pattern in patterns: | ||
if re.match(pattern, string): | ||
return True | ||
return False | ||
|
||
|
||
def run_antlr(cmd: Command) -> None: # pragma: no cover | ||
try: | ||
cmd.announce("Generating parsers with antlr4", level=distutils.log.INFO) | ||
cmd.run_command("antlr") | ||
except OSError as e: | ||
if e.errno == errno.ENOENT: | ||
msg = f"| Unable to generate parsers: {e} |" | ||
msg = "=" * len(msg) + "\n" + msg + "\n" + "=" * len(msg) | ||
cmd.announce(f"{msg}", level=distutils.log.FATAL) | ||
sys.exit(1) | ||
else: | ||
raise |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
Intentionally left empty for git to keep the directory |
Empty file.
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
Intentionally left empty for git to keep the directory |
Empty file.
Empty file.
Empty file.
Oops, something went wrong.