Skip to content

Commit

Permalink
Major changes to interpolations and resolvers (#445)
Browse files Browse the repository at this point in the history
* Add a grammar to parse interpolations
* Add support for nested interpolations
* Deprecate register_resolver() and introduce new_register_resolver() (that allows resolvers to use non-string arguments, and to decide whether or not to use the cache)
* The `env` resolver now parses environment variables in a way that is consistent with the new interpolation grammar

Fixes #100 #230 #266 #318
odelalleau authored Feb 12, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 2cf4e7c commit 499de6b
Showing 54 changed files with 2,751 additions and 436 deletions.
64 changes: 29 additions & 35 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,47 +1,41 @@
version: 2.1

jobs:
py36_linux:
docker:
- image: circleci/python:3.6
steps:
- checkout
- run: echo 'export NOX_PYTHON_VERSIONS=3.6' >> $BASH_ENV
- run: sudo pip install nox
- run: nox --add-timestamp

py37_linux:
docker:
- image: circleci/python:3.7
commands:
linux:
description: "Commands run on Linux"
parameters:
py_version:
type: string
steps:
- checkout
- run: echo 'export NOX_PYTHON_VERSIONS=3.7' >> $BASH_ENV
- run: sudo pip install nox
- run: nox --add-timestamp
- run:
name: "Preparing environment"
command: |
sudo apt-get update
sudo apt-get install -y openjdk-11-jre
sudo pip install nox
- run:
name: "Testing OmegaConf"
command: |
export NOX_PYTHON_VERSIONS=<< parameters.py_version >>
nox --add-timestamp
py38_linux:
docker:
- image: circleci/python:3.8
steps:
- checkout
- run: echo 'export NOX_PYTHON_VERSIONS=3.8' >> $BASH_ENV
- run: sudo pip install nox
- run: nox --add-timestamp

py39_linux:
jobs:
test_linux:
parameters:
py_version:
type: string
docker:
- image: circleci/python:3.9
- image: circleci/python:<< parameters.py_version >>
steps:
- checkout
- run: echo 'export NOX_PYTHON_VERSIONS=3.9' >> $BASH_ENV
- run: sudo pip install nox
- run: nox --add-timestamp
- linux:
py_version: << parameters.py_version >>

workflows:
version: 2
build:
jobs:
- py36_linux
- py37_linux
- py38_linux
- py39_linux
- test_linux:
matrix:
parameters:
py_version: ["3.6", "3.7", "3.8", "3.9"]
3 changes: 2 additions & 1 deletion .coveragerc
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ omit =
.nox/*
*tests*
docs/*
omegaconf/grammar/gen/*
omegaconf/version.py
.stubs

@@ -15,7 +16,7 @@ exclude_lines =
assert False
@abstractmethod
\.\.\.

if TYPE_CHECKING:

[html]
directory = docs/build/coverage
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[flake8]
exclude = .git,.nox,.tox
exclude = .git,.nox,.tox,omegaconf/grammar/gen
max-line-length = 119
select = E,F,W,C
ignore=W503,E203
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.jar binary
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -16,7 +16,9 @@ TODO
.coverage
.eggs
.mypy_cache
/omegaconf/grammar/gen
/pip-wheel-metadata
/.pyre
.dmypy.json
.python-version
.vscode
1 change: 1 addition & 0 deletions .isort.cfg
Original file line number Diff line number Diff line change
@@ -7,3 +7,4 @@ line_length=88
ensure_newline_before_comments=True
known_third_party=attr,pytest
known_first_party=omegaconf
skip=.eggs,.nox,omegaconf/grammar/gen
21 changes: 21 additions & 0 deletions benchmark/benchmark.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
from pytest import fixture, mark, param

from omegaconf import OmegaConf
from omegaconf._utils import ValueKind, get_value_kind


def build_dict(
@@ -119,3 +120,23 @@ def iterate(seq: Any) -> None:
pass

benchmark(iterate, lst)


@mark.parametrize(
"strict_interpolation_validation",
[True, False],
)
@mark.parametrize(
("value", "expected"),
[
("simple", ValueKind.VALUE),
("${a}", ValueKind.INTERPOLATION),
("${a:b,c,d}", ValueKind.INTERPOLATION),
("${${b}}", ValueKind.INTERPOLATION),
("${a:${b}}", ValueKind.INTERPOLATION),
],
)
def test_get_value_kind(
strict_interpolation_validation: bool, value: Any, expected: Any, benchmark: Any
) -> None:
assert benchmark(get_value_kind, value, strict_interpolation_validation) == expected
3 changes: 3 additions & 0 deletions build_helpers/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Order of imports is important (see warning otherwise when running tests)
import setuptools # isort:skip # noqa
import distutils # isort:skip # noqa
Binary file added build_helpers/bin/antlr-4.8-complete.jar
Binary file not shown.
195 changes: 195 additions & 0 deletions build_helpers/build_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
import codecs
import distutils.log
import errno
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from typing import List, Optional

from setuptools import Command
from setuptools.command import build_py, develop, sdist # type: ignore


class ANTLRCommand(Command): # type: ignore # pragma: no cover
"""Generate parsers using ANTLR."""

description = "Run ANTLR"
user_options: List[str] = []

def run(self) -> None:
"""Run command."""
build_dir = Path(__file__).parent.absolute()
project_root = build_dir.parent
for grammar in [
"OmegaConfGrammarLexer.g4",
"OmegaConfGrammarParser.g4",
]:
command = [
"java",
"-jar",
str(build_dir / "bin" / "antlr-4.8-complete.jar"),
"-Dlanguage=Python3",
"-o",
str(project_root / "omegaconf" / "grammar" / "gen"),
"-Xexact-output-dir",
"-visitor",
str(project_root / "omegaconf" / "grammar" / grammar),
]

self.announce(
f"Generating parser for Python3: {command}",
level=distutils.log.INFO,
)

subprocess.check_call(command)

def initialize_options(self) -> None:
pass

def finalize_options(self) -> None:
pass


class BuildPyCommand(build_py.build_py): # type: ignore # pragma: no cover
def run(self) -> None:
if not self.dry_run:
self.run_command("clean")
run_antlr(self)
build_py.build_py.run(self)


class CleanCommand(Command): # type: ignore # pragma: no cover
"""
Our custom command to clean out junk files.
"""

description = "Cleans out generated and junk files we don't want in the repo"
dry_run: bool
user_options: List[str] = []

def run(self) -> None:
root = Path(__file__).parent.parent.absolute()
files = find(
root=root,
include_files=["^omegaconf/grammar/gen/.*"],
include_dirs=[
"^omegaconf\\.egg-info$",
"\\.eggs$",
"^\\.mypy_cache$",
"^\\.nox$",
"^\\.pytest_cache$",
".*/__pycache__$",
"^__pycache__$",
"^build$",
"^dist$",
],
scan_exclude=["^.git$", "^.nox/.*$"],
excludes=[".*\\.gitignore$", ".*/__init__.py"],
)

if self.dry_run:
print("Dry run! Would clean up the following files and dirs:")
print("\n".join(sorted(map(str, files))))
else:
for f in files:
if f.exists():
if f.is_dir():
shutil.rmtree(f, ignore_errors=True)
else:
f.unlink()

def initialize_options(self) -> None:
pass

def finalize_options(self) -> None:
pass


class DevelopCommand(develop.develop): # type: ignore # pragma: no cover
def run(self) -> None:
if not self.dry_run:
run_antlr(self)
develop.develop.run(self)


class SDistCommand(sdist.sdist): # type: ignore # pragma: no cover
def run(self) -> None:
if not self.dry_run:
self.run_command("clean")
run_antlr(self)
sdist.sdist.run(self)


def find(
root: Path,
include_files: List[str],
include_dirs: List[str],
excludes: List[str],
rbase: Optional[Path] = None,
scan_exclude: Optional[List[str]] = None,
) -> List[Path]:
if rbase is None:
rbase = Path()
if scan_exclude is None:
scan_exclude = []
files = []
scan_root = root / rbase
for entry in scan_root.iterdir():
path = rbase / entry.name
if matches(scan_exclude, path):
continue

if entry.is_dir():
if matches(include_dirs, path):
if not matches(excludes, path):
files.append(path)
else:
ret = find(
root=root,
include_files=include_files,
include_dirs=include_dirs,
excludes=excludes,
rbase=path,
scan_exclude=scan_exclude,
)
files.extend(ret)
else:
if matches(include_files, path) and not matches(excludes, path):
files.append(path)

return files


def find_version(*file_paths: str) -> str:
root = Path(__file__).parent.parent.absolute()
with codecs.open(root / Path(*file_paths), "r") as fp: # type: ignore
version_file = fp.read()
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
if version_match:
return version_match.group(1)
raise RuntimeError("Unable to find version string.") # pragma: no cover


def matches(patterns: List[str], path: Path) -> bool:
string = str(path).replace(os.sep, "/") # for Windows
for pattern in patterns:
if re.match(pattern, string):
return True
return False


def run_antlr(cmd: Command) -> None: # pragma: no cover
try:
cmd.announce("Generating parsers with antlr4", level=distutils.log.INFO)
cmd.run_command("antlr")
except OSError as e:
if e.errno == errno.ENOENT:
msg = f"| Unable to generate parsers: {e} |"
msg = "=" * len(msg) + "\n" + msg + "\n" + "=" * len(msg)
cmd.announce(f"{msg}", level=distutils.log.FATAL)
sys.exit(1)
else:
raise
1 change: 1 addition & 0 deletions build_helpers/test_files/a/b/bad_dir/.gitkeep
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Intentionally left empty for git to keep the directory
Empty file.
Empty file.
Empty file.
1 change: 1 addition & 0 deletions build_helpers/test_files/c/bad_dir/.gitkeep
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Intentionally left empty for git to keep the directory
Empty file.
Empty file.
Empty file.
Loading

0 comments on commit 499de6b

Please sign in to comment.