From f24d746dee2769b7fe2947bd293afa5d6e252fce Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Fri, 3 Nov 2017 19:15:04 -0700 Subject: [PATCH 1/4] Expose package metadata from wheels in .par files --- WORKSPACE | 8 + runtime/support.py | 80 +++ tests/BUILD | 14 + tests/package_pkg_resources/main.py | 43 ++ .../main_PY2_filelist.txt | 12 + .../main_PY3_filelist.txt | 12 + third_party/README.md | 9 + third_party/pypi__portpicker_1_2_0/BUILD | 10 + third_party/pypi__portpicker_1_2_0/WORKSPACE | 1 + .../portpicker-1.2.0.dist-info/METADATA | 29 + .../portpicker-1.2.0.dist-info/metadata.json | 1 + third_party/pypi__yapf_0_19_0/BUILD | 10 + third_party/pypi__yapf_0_19_0/WORKSPACE | 1 + .../yapf-0.19.0.dist-info/METADATA | 647 ++++++++++++++++++ .../yapf-0.19.0.dist-info/metadata.json | 1 + 15 files changed, 878 insertions(+) create mode 100644 tests/package_pkg_resources/main.py create mode 100644 tests/package_pkg_resources/main_PY2_filelist.txt create mode 100644 tests/package_pkg_resources/main_PY3_filelist.txt create mode 100644 third_party/README.md create mode 100644 third_party/pypi__portpicker_1_2_0/BUILD create mode 100644 third_party/pypi__portpicker_1_2_0/WORKSPACE create mode 100644 third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA create mode 100644 third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json create mode 100644 third_party/pypi__yapf_0_19_0/BUILD create mode 100644 third_party/pypi__yapf_0_19_0/WORKSPACE create mode 100644 third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA create mode 100644 third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json diff --git a/WORKSPACE b/WORKSPACE index 89cbb3c..09df09f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,6 +5,14 @@ local_repository( name = "test_workspace", path = "tests/test_workspace", ) +local_repository( + name = "pypi__portpicker_1_2_0", + path = "third_party/pypi__portpicker_1_2_0", +) +local_repository( + name = "pypi__yapf_0_19_0", + path = "third_party/pypi__yapf_0_19_0", +) # Not actually referenced anywhere, but must be marked as a separate # repository so that things like "bazel test //..." don't get confused diff --git a/runtime/support.py b/runtime/support.py index d9e957a..83991d6 100644 --- a/runtime/support.py +++ b/runtime/support.py @@ -61,6 +61,83 @@ def _find_archive(): return archive_path +def _setup_pkg_resources(): + try: + import pkg_resources + import zipimport + except ImportError: + # Skip setup + return + + class DistInfoMetadata(pkg_resources.EggMetadata): + """Metadata provider for zip files containing .dist-info + + In find_dist_info_in_zip(), we call + metadata.resource_listdir(directory_name). However, it doesn't + work with EggMetadata, because _zipinfo_name() expects the + directory name to end with a /, but metadata._listdir() which + expects the directory to _not_ end with a /. + + Therefore this class exists. + """ + + def _listdir(self, fspath): + """List of resource names in the directory (like ``os.listdir()``) + + Overrides EggMetadata._listdir() + """ + + zipinfo_name = self._zipinfo_name(fspath) + while zipinfo_name.endswith('/'): + zipinfo_name = zipinfo_name[:-1] + result = self._index().get(zipinfo_name, ()) + return list(result) + + + def find_dist_info_in_zip(importer, path_item, only=False): + """Find dist-info style metadata in zip files. + + We ignore the `only` flag because it's not clear what it should + actually do in this case. + """ + metadata = DistInfoMetadata(importer) + for subitem in metadata.resource_listdir('/'): + if subitem.lower().endswith('.dist-info'): + subpath = os.path.join(path_item, subitem) + submeta = pkg_resources.EggMetadata(zipimport.zipimporter(subpath)) + submeta.egg_info = subpath + dist = pkg_resources.Distribution.from_location(path_item, subitem, submeta) + yield dist + return + + + def find_eggs_and_dist_info_in_zip(importer, path_item, only=False): + """Chain together our finder and the standard pkg_resources finder + + For simplicity, and since pkg_resources doesn't provide a public + interface to do so, we hardcode the chaining (find_eggs_in_zip). + """ + # Our finder + for dist in find_dist_info_in_zip(importer, path_item, only): + yield dist + # The standard pkg_resources finder + for dist in pkg_resources.find_eggs_in_zip(importer, path_item, only): + yield dist + return + + + # This overwrites the existing registered finder. + # + # Note that this also doesn't update the default WorkingSet created by + # pkg_resources when it is imported, since there is no public + # interface to do so that doesn't also have a "Don't use this" + # warning. + pkg_resources.register_finder(zipimport.zipimporter, + find_eggs_and_dist_info_in_zip) + + + + def setup(import_roots=None): """Initialize subpar run-time support""" # Add third-party library entries to sys.path @@ -75,3 +152,6 @@ def setup(import_roots=None): new_path = os.path.join(archive_path, import_root) _log('# adding %s to sys.path' % new_path) sys.path.insert(1, new_path) + + # Add hook for package metadata + _setup_pkg_resources() diff --git a/tests/BUILD b/tests/BUILD index 0d35eb7..0d215bc 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -104,6 +104,19 @@ par_binary( srcs_version = "PY2AND3", ) +par_binary( + name = "package_pkg_resources/main", + srcs = [ + "package_pkg_resources/main.py", + ], + data = [ + "@pypi__portpicker_1_2_0//:files", + "@pypi__yapf_0_19_0//:files", + ], + main = "package_pkg_resources/main.py", + srcs_version = "PY2AND3", +) + # Test targets [( # Run test without .par file as a control @@ -154,6 +167,7 @@ par_binary( ), ("indirect_dependency", "//tests:package_c/c", "tests/package_c/c"), ("main_boilerplate", "//tests:package_g/g", "tests/package_g/g"), + ("pkg_resources", "//tests:package_pkg_resources/main", "tests/package_pkg_resources/main"), ("shadow", "//tests:package_shadow/main", "tests/package_shadow/main"), ("version", "//tests:package_f/f", "tests/package_f/f"), ]] diff --git a/tests/package_pkg_resources/main.py b/tests/package_pkg_resources/main.py new file mode 100644 index 0000000..b37ae54 --- /dev/null +++ b/tests/package_pkg_resources/main.py @@ -0,0 +1,43 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration test program for Subpar. + +Test that pkg_resources correctly identifies distribution packages +inside a .par file. +""" + +def main(): + print('In pkg_resources test main()') + try: + import pkg_resources + except ImportError: + print('Skipping test, pkg_resources module is not available') + return + + ws = pkg_resources.WorkingSet() + + # Informational for debugging + distributions = list(ws) + print('Resources found: %s' % distributions) + + # Check for the packages we provided metadata for. There will + # also be metadata for whatever other packages happen to be + # installed in the current Python interpreter. + for spec in ['portpicker==1.2.0', 'yapf==0.19.0']: + dist = ws.find(pkg_resources.Requirement.parse(spec)) + assert dist, (spec, distributions) + +if __name__ == '__main__': + main() diff --git a/tests/package_pkg_resources/main_PY2_filelist.txt b/tests/package_pkg_resources/main_PY2_filelist.txt new file mode 100644 index 0000000..7b93f00 --- /dev/null +++ b/tests/package_pkg_resources/main_PY2_filelist.txt @@ -0,0 +1,12 @@ +__main__.py +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json +subpar/__init__.py +subpar/runtime/__init__.py +subpar/runtime/support.py +subpar/tests/__init__.py +subpar/tests/package_pkg_resources/__init__.py +subpar/tests/package_pkg_resources/main +subpar/tests/package_pkg_resources/main.py diff --git a/tests/package_pkg_resources/main_PY3_filelist.txt b/tests/package_pkg_resources/main_PY3_filelist.txt new file mode 100644 index 0000000..7b93f00 --- /dev/null +++ b/tests/package_pkg_resources/main_PY3_filelist.txt @@ -0,0 +1,12 @@ +__main__.py +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA +pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA +pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json +subpar/__init__.py +subpar/runtime/__init__.py +subpar/runtime/support.py +subpar/tests/__init__.py +subpar/tests/package_pkg_resources/__init__.py +subpar/tests/package_pkg_resources/main +subpar/tests/package_pkg_resources/main.py diff --git a/third_party/README.md b/third_party/README.md new file mode 100644 index 0000000..6cbf2d2 --- /dev/null +++ b/third_party/README.md @@ -0,0 +1,9 @@ +# Wheels + +This directory contains code, metadata files, and wheels, from other projects, +used for testing. The projects were chosen to match the licence and ownership +of this project, i.e. Apache License 2.0, copyright owned by Google, from the +Google organization repository. Do not add any other type of thing here. + +- [python_portpicker](https://github.com/google/python_portpicker) +- [yapf](https://github.com/google/yapf) diff --git a/third_party/pypi__portpicker_1_2_0/BUILD b/third_party/pypi__portpicker_1_2_0/BUILD new file mode 100644 index 0000000..6cff5d5 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/BUILD @@ -0,0 +1,10 @@ +# Test package for Subpar + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "files", + srcs = [], + data = glob(["portpicker-1.2.0.dist-info/**"]), + imports = ["."], +) diff --git a/third_party/pypi__portpicker_1_2_0/WORKSPACE b/third_party/pypi__portpicker_1_2_0/WORKSPACE new file mode 100644 index 0000000..57c31df --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pypi__portpicker_1_2_0") diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA new file mode 100644 index 0000000..8e852c1 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA @@ -0,0 +1,29 @@ +Metadata-Version: 2.0 +Name: portpicker +Version: 1.2.0 +Summary: A library to choose unique available network ports. +Home-page: https://github.com/google/python_portpicker +Author: Google +Author-email: greg@krypto.org +License: Apache 2.0 +Description-Content-Type: UNKNOWN +Platform: POSIX +Classifier: Development Status :: 5 - Production/Stable +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy + +Portpicker provides an API to find and return an available network +port for an application to bind to. Ideally suited for use from +unittests or for test harnesses that launch local servers. + diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json new file mode 100644 index 0000000..6ec9c43 --- /dev/null +++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: Jython", "Programming Language :: Python :: Implementation :: PyPy"], "description_content_type": "UNKNOWN", "extensions": {"python.details": {"contacts": [{"email": "greg@krypto.org", "name": "Google", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/google/python_portpicker"}}}, "generator": "bdist_wheel (0.30.0)", "license": "Apache 2.0", "metadata_version": "2.0", "name": "portpicker", "platform": "POSIX", "summary": "A library to choose unique available network ports.", "version": "1.2.0"} \ No newline at end of file diff --git a/third_party/pypi__yapf_0_19_0/BUILD b/third_party/pypi__yapf_0_19_0/BUILD new file mode 100644 index 0000000..cd12f69 --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/BUILD @@ -0,0 +1,10 @@ +# Test package for Subpar + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "files", + srcs = [], + data = glob(["yapf-0.19.0.dist-info/**"]), + imports = ["."], +) diff --git a/third_party/pypi__yapf_0_19_0/WORKSPACE b/third_party/pypi__yapf_0_19_0/WORKSPACE new file mode 100644 index 0000000..eae31db --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "pypi__yapf_0_19_0") diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA new file mode 100644 index 0000000..03a64f1 --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA @@ -0,0 +1,647 @@ +Metadata-Version: 2.0 +Name: yapf +Version: 0.19.0 +Summary: A formatter for Python code. +Home-page: UNKNOWN +Author: Bill Wendling +Author-email: morbo@google.com +License: Apache License, Version 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Quality Assurance + +==== +YAPF +==== + +.. image:: https://badge.fury.io/py/yapf.svg + :target: http://badge.fury.io/py/yapf + :alt: PyPI version + +.. image:: https://travis-ci.org/google/yapf.svg?branch=master + :target: https://travis-ci.org/google/yapf + :alt: Build status + +.. image:: https://coveralls.io/repos/google/yapf/badge.svg?branch=master + :target: https://coveralls.io/r/google/yapf?branch=master + :alt: Coverage status + + +Introduction +============ + +Most of the current formatters for Python --- e.g., autopep8, and pep8ify --- +are made to remove lint errors from code. This has some obvious limitations. +For instance, code that conforms to the PEP 8 guidelines may not be +reformatted. But it doesn't mean that the code looks good. + +YAPF takes a different approach. It's based off of 'clang-format', developed by +Daniel Jasper. In essence, the algorithm takes the code and reformats it to the +best formatting that conforms to the style guide, even if the original code +didn't violate the style guide. The idea is also similar to the 'gofmt' tool for +the Go programming language: end all holy wars about formatting - if the whole +codebase of a project is simply piped through YAPF whenever modifications are +made, the style remains consistent throughout the project and there's no point +arguing about style in every code review. + +The ultimate goal is that the code YAPF produces is as good as the code that a +programmer would write if they were following the style guide. It takes away +some of the drudgery of maintaining your code. + +.. footer:: + + YAPF is not an official Google product (experimental or otherwise), it is + just code that happens to be owned by Google. + +.. contents:: + + +Installation +============ + +To install YAPF from PyPI: + +.. code-block:: + + $ pip install yapf + +(optional) If you are using Python 2.7 and want to enable multiprocessing: + +.. code-block:: + + $ pip install futures + +YAPF is still considered in "alpha" stage, and the released version may change +often; therefore, the best way to keep up-to-date with the latest development +is to clone this repository. + +Note that if you intend to use YAPF as a command-line tool rather than as a +library, installation is not necessary. YAPF supports being run as a directory +by the Python interpreter. If you cloned/unzipped YAPF into ``DIR``, it's +possible to run: + +.. code-block:: + + $ PYTHONPATH=DIR python DIR/yapf [options] ... + + +Python versions +=============== + +YAPF supports Python 2.7 and 3.4.1+. + +YAPF requires the code it formats to be valid Python for the version YAPF itself +runs under. Therefore, if you format Python 3 code with YAPF, run YAPF itself +under Python 3 (and similarly for Python 2). + + +Usage +===== + +Options:: + + usage: yapf [-h] [-v] [-d | -i] [-r | -l START-END] [-e PATTERN] + [--style STYLE] [--style-help] [--no-local-style] [-p] + [files [files ...]] + + Formatter for Python code. + + positional arguments: + files + + optional arguments: + -h, --help show this help message and exit + -v, --version show version number and exit + -d, --diff print the diff for the fixed source + -i, --in-place make changes to files in place + -r, --recursive run recursively over directories + -l START-END, --lines START-END + range of lines to reformat, one-based + -e PATTERN, --exclude PATTERN + patterns for files to exclude from formatting + --style STYLE specify formatting style: either a style name (for + example "pep8" or "google"), or the name of a file + with style settings. The default is pep8 unless a + .style.yapf or setup.cfg file located in one of the + parent directories of the source file (or current + directory for stdin) + --style-help show style settings and exit + --no-local-style don't search for local style definition (.style.yapf) + -p, --parallel Run yapf in parallel when formatting multiple files. + Requires concurrent.futures in Python 2.X + + +Formatting style +================ + +The formatting style used by YAPF is configurable and there are many "knobs" +that can be used to tune how YAPF does formatting. See the ``style.py`` module +for the full list. + +To control the style, run YAPF with the ``--style`` argument. It accepts one of +the predefined styles (e.g., ``pep8`` or ``google``), a path to a configuration +file that specifies the desired style, or a dictionary of key/value pairs. + +The config file is a simple listing of (case-insensitive) ``key = value`` pairs +with a ``[style]`` heading. For example: + +.. code-block:: + + [style] + based_on_style = pep8 + spaces_before_comment = 4 + split_before_logical_operator = true + +The ``based_on_style`` setting determines which of the predefined styles this +custom style is based on (think of it like subclassing). + +It's also possible to do the same on the command line with a dictionary. For +example: + +.. code-block:: + + --style='{based_on_style: chromium, indent_width: 4}' + +This will take the ``chromium`` base style and modify it to have four space +indentations. + +YAPF will search for the formatting style in the following manner: + +1. Specified on the command line +2. In the `[style]` section of a `.style.yapf` file in either the current + directory or one of its parent directories. +3. In the `[yapf]` section of a `setup.cfg` file in either the current + directory or one of its parent directories. +4. In the `~/.config/yapf/style` file in your home directory. + +If none of those files are found, the default style is used (PEP8). + + +Example +======= + +An example of the type of formatting that YAPF can do, it will take this ugly +code: + +.. code-block:: python + + x = { 'a':37,'b':42, + + 'c':927} + + y = 'hello ''world' + z = 'hello '+'world' + a = 'hello {}'.format('world') + class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y + def f ( a ) : + return 37+-+a[42-x : y**3] + +and reformat it into: + +.. code-block:: python + + x = {'a': 37, 'b': 42, 'c': 927} + + y = 'hello ' 'world' + z = 'hello ' + 'world' + a = 'hello {}'.format('world') + + + class foo(object): + def f(self): + return 37 * -+2 + + def g(self, x, y=42): + return y + + + def f(a): + return 37 + -+a[42 - x:y**3] + + +Example as a module +=================== + +The two main APIs for calling yapf are ``FormatCode`` and ``FormatFile``, these +share several arguments which are described below: + +.. code-block:: python + + >>> from yapf.yapflib.yapf_api import FormatCode # reformat a string of code + + >>> FormatCode("f ( a = 1, b = 2 )") + 'f(a=1, b=2)\n' + +A ``style_config`` argument: Either a style name or a path to a file that contains +formatting style settings. If None is specified, use the default style +as set in ``style.DEFAULT_STYLE_FACTORY``. + +.. code-block:: python + + >>> FormatCode("def g():\n return True", style_config='pep8') + 'def g():\n return True\n' + +A ``lines`` argument: A list of tuples of lines (ints), [start, end], +that we want to format. The lines are 1-based indexed. It can be used by +third-party code (e.g., IDEs) when reformatting a snippet of code rather +than a whole file. + +.. code-block:: python + + >>> FormatCode("def g( ):\n a=1\n b = 2\n return a==b", lines=[(1, 1), (2, 3)]) + 'def g():\n a = 1\n b = 2\n return a==b\n' + +A ``print_diff`` (bool): Instead of returning the reformatted source, return a +diff that turns the formatted source into reformatter source. + +.. code-block:: python + + >>> print(FormatCode("a==b", filename="foo.py", print_diff=True)) + --- foo.py (original) + +++ foo.py (reformatted) + @@ -1 +1 @@ + -a==b + +a == b + +Note: the ``filename`` argument for ``FormatCode`` is what is inserted into +the diff, the default is ````. + +``FormatFile`` returns reformatted code from the passed file along with its encoding: + +.. code-block:: python + + >>> from yapf.yapflib.yapf_api import FormatFile # reformat a file + + >>> print(open("foo.py").read()) # contents of file + a==b + + >>> FormatFile("foo.py") + ('a == b\n', 'utf-8') + +The ``in-place`` argument saves the reformatted code back to the file: + +.. code-block:: python + + >>> FormatFile("foo.py", in_place=True) + (None, 'utf-8') + + >>> print(open("foo.py").read()) # contents of file (now fixed) + a == b + + +Knobs +===== + +``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` + Align closing bracket with visual indentation. + +``ALLOW_MULTILINE_LAMBDAS`` + Allow lambdas to be formatted on more than one line. + +``ALLOW_MULTILINE_DICTIONARY_KEYS`` + Allow dictionary keys to exist on multiple lines. For example: + + .. code-block:: python + + x = { + ('this is the first element of a tuple', + 'this is the second element of a tuple'): + value, + } + +``ALLOW_SPLIT_BEFORE_DICT_VALUE`` + Allow splits before the dictionary value. + +``BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF`` + Insert a blank line before a ``def`` or ``class`` immediately nested within + another ``def`` or ``class``. For example: + + .. code-block:: python + + class Foo: + # <------ this blank line + def method(): + pass + +``BLANK_LINE_BEFORE_CLASS_DOCSTRING`` + Insert a blank line before a class-level docstring. + +``COALESCE_BRACKETS`` + Do not split consecutive brackets. Only relevant when + ``DEDENT_CLOSING_BRACKETS`` is set. For example: + + .. code-block:: python + + call_func_that_takes_a_dict( + { + 'key1': 'value1', + 'key2': 'value2', + } + ) + + would reformat to: + + .. code-block:: python + + call_func_that_takes_a_dict({ + 'key1': 'value1', + 'key2': 'value2', + }) + + +``COLUMN_LIMIT`` + The column limit (or max line-length) + +``CONTINUATION_INDENT_WIDTH`` + Indent width used for line continuations. + +``DEDENT_CLOSING_BRACKETS`` + Put closing brackets on a separate line, dedented, if the bracketed + expression can't fit in a single line. Applies to all kinds of brackets, + including function definitions and calls. For example: + + .. code-block:: python + + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is dedented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is dedented and on a separate line + +``EACH_DICT_ENTRY_ON_SEPARATE_LINE`` + Place each dictionary entry onto its own line. + +``I18N_COMMENT`` + The regex for an internationalization comment. The presence of this comment + stops reformatting of that line, because the comments are required to be + next to the string they translate. + +``I18N_FUNCTION_CALL`` + The internationalization function call names. The presence of this function + stops reformatting on that line, because the string it has cannot be moved + away from the i18n comment. + +``INDENT_DICTIONARY_VALUE`` + Indent the dictionary value if it cannot fit on the same line as the + dictionary key. For example: + + .. code-block:: python + + config = { + 'key1': + 'value1', + 'key2': value1 + + value2, + } + +``INDENT_WIDTH`` + The number of columns to use for indentation. + +``JOIN_MULTIPLE_LINES`` + Join short lines into one line. E.g., single line ``if`` statements. + +``SPACES_AROUND_POWER_OPERATOR`` + Set to ``True`` to prefer using spaces around ``**``. + +``NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS`` + Do not include spaces around selected binary operators. For example: + + .. code-block:: python + + 1 + 2 * 3 - 4 / 5 + + will be formatted as follows when configured with a value ``"*,/"``: + + .. code-block:: python + + 1 + 2*3 - 4/5 + +``SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN`` + Set to ``True`` to prefer spaces around the assignment operator for default + or keyword arguments. + +``SPACES_BEFORE_COMMENT`` + The number of spaces required before a trailing comment. + +``SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET`` + Insert a space between the ending comma and closing bracket of a list, etc. + +``SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED`` + Split before arguments if the argument list is terminated by a comma. + +``SPLIT_BEFORE_BITWISE_OPERATOR`` + Set to ``True`` to prefer splitting before ``&``, ``|`` or ``^`` rather + than after. + +``SPLIT_BEFORE_DICT_SET_GENERATOR`` + Split before a dictionary or set generator (comp_for). For example, note + the split before the ``for``: + + .. code-block:: python + + foo = { + variable: 'Hello world, have a nice day!' + for variable in bar if variable != 42 + } + +``SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN`` + Split after the opening paren which surrounds an expression if it doesn't + fit on a single line. + +``SPLIT_BEFORE_FIRST_ARGUMENT`` + If an argument / parameter list is going to be split, then split before the + first argument. + +``SPLIT_BEFORE_LOGICAL_OPERATOR`` + Set to ``True`` to prefer splitting before ``and`` or ``or`` rather than + after. + +``SPLIT_BEFORE_NAMED_ASSIGNS`` + Split named assignments onto individual lines. + +``SPLIT_PENALTY_AFTER_OPENING_BRACKET`` + The penalty for splitting right after the opening bracket. + +``SPLIT_PENALTY_AFTER_UNARY_OPERATOR`` + The penalty for splitting the line after a unary operator. + +``SPLIT_PENALTY_BEFORE_IF_EXPR`` + The penalty for splitting right before an ``if`` expression. + +``SPLIT_PENALTY_BITWISE_OPERATOR`` + The penalty of splitting the line around the ``&``, ``|``, and ``^`` + operators. + +``SPLIT_PENALTY_EXCESS_CHARACTER`` + The penalty for characters over the column limit. + +``SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT`` + The penalty incurred by adding a line split to the unwrapped line. The more + line splits added the higher the penalty. + +``SPLIT_PENALTY_IMPORT_NAMES`` + The penalty of splitting a list of ``import as`` names. For example: + + .. code-block:: python + + from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, + long_argument_2, + long_argument_3) + + would reformat to something like: + + .. code-block:: python + + from a_very_long_or_indented_module_name_yada_yad import ( + long_argument_1, long_argument_2, long_argument_3) + +``SPLIT_PENALTY_LOGICAL_OPERATOR`` + The penalty of splitting the line around the ``and`` and ``or`` operators. + +``USE_TABS`` + Use the Tab character for indentation. + +(Potentially) Frequently Asked Questions +======================================== + +Why does YAPF destroy my awesome formatting? +-------------------------------------------- + +YAPF tries very hard to get the formatting correct. But for some code, it won't +be as good as hand-formatting. In particular, large data literals may become +horribly disfigured under YAPF. + +The reason for this is many-fold. But in essence YAPF is simply a tool to help +with development. It will format things to coincide with the style guide, but +that may not equate with readability. + +What can be done to alleviate this situation is to indicate regions YAPF should +ignore when reformatting something: + +.. code-block:: python + + # yapf: disable + FOO = { + # ... some very large, complex data literal. + } + + BAR = [ + # ... another large data literal. + ] + # yapf: enable + +You can also disable formatting for a single literal like this: + +.. code-block:: python + + BAZ = { + (1, 2, 3, 4), + (5, 6, 7, 8), + (9, 10, 11, 12), + } # yapf: disable + +To preserve the nice dedented closing brackets, use the +``dedent_closing_brackets`` in your style. Note that in this case all +brackets, including function definitions and calls, are going to use +that style. This provides consistency across the formatted codebase. + +Why Not Improve Existing Tools? +------------------------------- + +We wanted to use clang-format's reformatting algorithm. It's very powerful and +designed to come up with the best formatting possible. Existing tools were +created with different goals in mind, and would require extensive modifications +to convert to using clang-format's algorithm. + +Can I Use YAPF In My Program? +----------------------------- + +Please do! YAPF was designed to be used as a library as well as a command line +tool. This means that a tool or IDE plugin is free to use YAPF. + + +Gory Details +============ + +Algorithm Design +---------------- + +The main data structure in YAPF is the ``UnwrappedLine`` object. It holds a list +of ``FormatToken``\s, that we would want to place on a single line if there were +no column limit. An exception being a comment in the middle of an expression +statement will force the line to be formatted on more than one line. The +formatter works on one ``UnwrappedLine`` object at a time. + +An ``UnwrappedLine`` typically won't affect the formatting of lines before or +after it. There is a part of the algorithm that may join two or more +``UnwrappedLine``\s into one line. For instance, an if-then statement with a +short body can be placed on a single line: + +.. code-block:: python + + if a == 42: continue + +YAPF's formatting algorithm creates a weighted tree that acts as the solution +space for the algorithm. Each node in the tree represents the result of a +formatting decision --- i.e., whether to split or not to split before a token. +Each formatting decision has a cost associated with it. Therefore, the cost is +realized on the edge between two nodes. (In reality, the weighted tree doesn't +have separate edge objects, so the cost resides on the nodes themselves.) + +For example, take the following Python code snippet. For the sake of this +example, assume that line (1) violates the column limit restriction and needs to +be reformatted. + +.. code-block:: python + + def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee): # 1 + pass # 2 + +For line (1), the algorithm will build a tree where each node (a +``FormattingDecisionState`` object) is the state of the line at that token given +the decision to split before the token or not. Note: the ``FormatDecisionState`` +objects are copied by value so each node in the graph is unique and a change in +one doesn't affect other nodes. + +Heuristics are used to determine the costs of splitting or not splitting. +Because a node holds the state of the tree up to a token's insertion, it can +easily determine if a splitting decision will violate one of the style +requirements. For instance, the heuristic is able to apply an extra penalty to +the edge when not splitting between the previous token and the one being added. + +There are some instances where we will never want to split the line, because +doing so will always be detrimental (i.e., it will require a backslash-newline, +which is very rarely desirable). For line (1), we will never want to split the +first three tokens: ``def``, ``xxxxxxxxxxx``, and ``(``. Nor will we want to +split between the ``)`` and the ``:`` at the end. These regions are said to be +"unbreakable." This is reflected in the tree by there not being a "split" +decision (left hand branch) within the unbreakable region. + +Now that we have the tree, we determine what the "best" formatting is by finding +the path through the tree with the lowest cost. + +And that's it! + + diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json new file mode 100644 index 0000000..71e9541 --- /dev/null +++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance"], "extensions": {"python.commands": {"wrap_console": {"yapf": "yapf:run_main"}}, "python.details": {"contacts": [{"email": "morbo@google.com", "name": "Bill Wendling", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}}, "python.exports": {"console_scripts": {"yapf": "yapf:run_main"}}}, "generator": "bdist_wheel (0.29.0)", "license": "Apache License, Version 2.0", "metadata_version": "2.0", "name": "yapf", "summary": "A formatter for Python code.", "version": "0.19.0"} \ No newline at end of file From f299643213381fcc9f9677386311daad979b6835 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Mon, 6 Nov 2017 19:28:55 -0800 Subject: [PATCH 2/4] Update pkg_resources's default distribution metadata object. `pkg_resources` creates a default WorkingSet as soon as it is imported, so even though we add hooks for it to find the .dist-info metadata inside .par files, it is too late. So we manually update the default WorkingSet, trying not to disturb existing entries. --- runtime/support.py | 39 +++++++++++++++++++++-------- runtime/support_test.py | 31 +++++++++++++++++------ tests/package_pkg_resources/main.py | 2 +- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/runtime/support.py b/runtime/support.py index 83991d6..0e89192 100644 --- a/runtime/support.py +++ b/runtime/support.py @@ -33,8 +33,10 @@ """ import os +import pkgutil import sys import warnings +import zipimport def _log(msg): @@ -61,10 +63,21 @@ def _find_archive(): return archive_path -def _setup_pkg_resources(): +def _setup_pkg_resources(pkg_resources_name): + """Setup hooks into the `pkg_resources` module + + This enables the pkg_resources module to find metadata from wheels + that have been included in this .par file. + + The functions and classes here are scoped to this function, since + we might have multitple pkg_resources modules, or none. + """ + try: - import pkg_resources - import zipimport + __import__(pkg_resources_name) + pkg_resources = sys.modules.get(pkg_resources_name) + if pkg_resources is None: + return except ImportError: # Skip setup return @@ -125,17 +138,20 @@ def find_eggs_and_dist_info_in_zip(importer, path_item, only=False): yield dist return - # This overwrites the existing registered finder. - # - # Note that this also doesn't update the default WorkingSet created by - # pkg_resources when it is imported, since there is no public - # interface to do so that doesn't also have a "Don't use this" - # warning. pkg_resources.register_finder(zipimport.zipimporter, find_eggs_and_dist_info_in_zip) - + # Note that the default WorkingSet has already been created, and + # there is no public interface to easily refresh/reload it that + # doesn't also have a "Don't use this" warning. So we manually + # add just the entries we know about to the existing WorkingSet. + for entry in sys.path: + importer = pkgutil.get_importer(entry) + if isinstance(importer, zipimport.zipimporter): + for dist in find_dist_info_in_zip(importer, entry, only=True): + pkg_resources.working_set.add(dist, entry, insert=False, + replace=False) def setup(import_roots=None): @@ -154,4 +170,5 @@ def setup(import_roots=None): sys.path.insert(1, new_path) # Add hook for package metadata - _setup_pkg_resources() + _setup_pkg_resources('pkg_resources') + _setup_pkg_resources('pip._vendor.pkg_resources') diff --git a/runtime/support_test.py b/runtime/support_test.py index ab25bd6..2a75334 100644 --- a/runtime/support_test.py +++ b/runtime/support_test.py @@ -21,7 +21,7 @@ class SupportTest(unittest.TestCase): - def test_log(self): + def test__log(self): old_stderr = sys.stderr try: mock_stderr = io.StringIO() @@ -36,17 +36,34 @@ def test_log(self): finally: sys.stderr = old_stderr - def test_find_archive(self): + def test__find_archive(self): # pylint: disable=protected-access path = support._find_archive() self.assertNotEqual(path, None) def test_setup(self): - support.setup(import_roots=['some_root', 'another_root']) - self.assertTrue(sys.path[1].endswith('subpar/runtime/some_root'), - sys.path) - self.assertTrue(sys.path[2].endswith('subpar/runtime/another_root'), - sys.path) + # `import pip` can cause arbitrary sys.path changes, + # especially if using the Debian `python-pip` package or + # similar. Get that lunacy out of the way before starting + # test + try: + import pip + except ImportError: + pass + + old_sys_path = sys.path + try: + mock_sys_path = list(sys.path) + sys.path = mock_sys_path + support.setup(import_roots=['some_root', 'another_root']) + finally: + sys.path = old_sys_path + self.assertTrue(mock_sys_path[1].endswith('subpar/runtime/some_root'), + mock_sys_path) + self.assertTrue(mock_sys_path[2].endswith('subpar/runtime/another_root'), + mock_sys_path) + self.assertEqual(mock_sys_path[0], sys.path[0]) + self.assertEqual(mock_sys_path[3:], sys.path[1:]) if __name__ == '__main__': diff --git a/tests/package_pkg_resources/main.py b/tests/package_pkg_resources/main.py index b37ae54..2888bc2 100644 --- a/tests/package_pkg_resources/main.py +++ b/tests/package_pkg_resources/main.py @@ -26,7 +26,7 @@ def main(): print('Skipping test, pkg_resources module is not available') return - ws = pkg_resources.WorkingSet() + ws = pkg_resources.working_set # Informational for debugging distributions = list(ws) From 4ecb8e326af26e3846b8c23064d6853b5b8b468d Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 8 Nov 2017 16:52:39 -0800 Subject: [PATCH 3/4] Cut down unnecessary test data. --- .../yapf-0.19.0.dist-info/METADATA | 625 +----------------- 1 file changed, 1 insertion(+), 624 deletions(-) diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA index 03a64f1..067f12a 100644 --- a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA +++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA @@ -21,627 +21,4 @@ Classifier: Programming Language :: Python :: 3.5 Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: Software Development :: Quality Assurance -==== -YAPF -==== - -.. image:: https://badge.fury.io/py/yapf.svg - :target: http://badge.fury.io/py/yapf - :alt: PyPI version - -.. image:: https://travis-ci.org/google/yapf.svg?branch=master - :target: https://travis-ci.org/google/yapf - :alt: Build status - -.. image:: https://coveralls.io/repos/google/yapf/badge.svg?branch=master - :target: https://coveralls.io/r/google/yapf?branch=master - :alt: Coverage status - - -Introduction -============ - -Most of the current formatters for Python --- e.g., autopep8, and pep8ify --- -are made to remove lint errors from code. This has some obvious limitations. -For instance, code that conforms to the PEP 8 guidelines may not be -reformatted. But it doesn't mean that the code looks good. - -YAPF takes a different approach. It's based off of 'clang-format', developed by -Daniel Jasper. In essence, the algorithm takes the code and reformats it to the -best formatting that conforms to the style guide, even if the original code -didn't violate the style guide. The idea is also similar to the 'gofmt' tool for -the Go programming language: end all holy wars about formatting - if the whole -codebase of a project is simply piped through YAPF whenever modifications are -made, the style remains consistent throughout the project and there's no point -arguing about style in every code review. - -The ultimate goal is that the code YAPF produces is as good as the code that a -programmer would write if they were following the style guide. It takes away -some of the drudgery of maintaining your code. - -.. footer:: - - YAPF is not an official Google product (experimental or otherwise), it is - just code that happens to be owned by Google. - -.. contents:: - - -Installation -============ - -To install YAPF from PyPI: - -.. code-block:: - - $ pip install yapf - -(optional) If you are using Python 2.7 and want to enable multiprocessing: - -.. code-block:: - - $ pip install futures - -YAPF is still considered in "alpha" stage, and the released version may change -often; therefore, the best way to keep up-to-date with the latest development -is to clone this repository. - -Note that if you intend to use YAPF as a command-line tool rather than as a -library, installation is not necessary. YAPF supports being run as a directory -by the Python interpreter. If you cloned/unzipped YAPF into ``DIR``, it's -possible to run: - -.. code-block:: - - $ PYTHONPATH=DIR python DIR/yapf [options] ... - - -Python versions -=============== - -YAPF supports Python 2.7 and 3.4.1+. - -YAPF requires the code it formats to be valid Python for the version YAPF itself -runs under. Therefore, if you format Python 3 code with YAPF, run YAPF itself -under Python 3 (and similarly for Python 2). - - -Usage -===== - -Options:: - - usage: yapf [-h] [-v] [-d | -i] [-r | -l START-END] [-e PATTERN] - [--style STYLE] [--style-help] [--no-local-style] [-p] - [files [files ...]] - - Formatter for Python code. - - positional arguments: - files - - optional arguments: - -h, --help show this help message and exit - -v, --version show version number and exit - -d, --diff print the diff for the fixed source - -i, --in-place make changes to files in place - -r, --recursive run recursively over directories - -l START-END, --lines START-END - range of lines to reformat, one-based - -e PATTERN, --exclude PATTERN - patterns for files to exclude from formatting - --style STYLE specify formatting style: either a style name (for - example "pep8" or "google"), or the name of a file - with style settings. The default is pep8 unless a - .style.yapf or setup.cfg file located in one of the - parent directories of the source file (or current - directory for stdin) - --style-help show style settings and exit - --no-local-style don't search for local style definition (.style.yapf) - -p, --parallel Run yapf in parallel when formatting multiple files. - Requires concurrent.futures in Python 2.X - - -Formatting style -================ - -The formatting style used by YAPF is configurable and there are many "knobs" -that can be used to tune how YAPF does formatting. See the ``style.py`` module -for the full list. - -To control the style, run YAPF with the ``--style`` argument. It accepts one of -the predefined styles (e.g., ``pep8`` or ``google``), a path to a configuration -file that specifies the desired style, or a dictionary of key/value pairs. - -The config file is a simple listing of (case-insensitive) ``key = value`` pairs -with a ``[style]`` heading. For example: - -.. code-block:: - - [style] - based_on_style = pep8 - spaces_before_comment = 4 - split_before_logical_operator = true - -The ``based_on_style`` setting determines which of the predefined styles this -custom style is based on (think of it like subclassing). - -It's also possible to do the same on the command line with a dictionary. For -example: - -.. code-block:: - - --style='{based_on_style: chromium, indent_width: 4}' - -This will take the ``chromium`` base style and modify it to have four space -indentations. - -YAPF will search for the formatting style in the following manner: - -1. Specified on the command line -2. In the `[style]` section of a `.style.yapf` file in either the current - directory or one of its parent directories. -3. In the `[yapf]` section of a `setup.cfg` file in either the current - directory or one of its parent directories. -4. In the `~/.config/yapf/style` file in your home directory. - -If none of those files are found, the default style is used (PEP8). - - -Example -======= - -An example of the type of formatting that YAPF can do, it will take this ugly -code: - -.. code-block:: python - - x = { 'a':37,'b':42, - - 'c':927} - - y = 'hello ''world' - z = 'hello '+'world' - a = 'hello {}'.format('world') - class foo ( object ): - def f (self ): - return 37*-+2 - def g(self, x,y=42): - return y - def f ( a ) : - return 37+-+a[42-x : y**3] - -and reformat it into: - -.. code-block:: python - - x = {'a': 37, 'b': 42, 'c': 927} - - y = 'hello ' 'world' - z = 'hello ' + 'world' - a = 'hello {}'.format('world') - - - class foo(object): - def f(self): - return 37 * -+2 - - def g(self, x, y=42): - return y - - - def f(a): - return 37 + -+a[42 - x:y**3] - - -Example as a module -=================== - -The two main APIs for calling yapf are ``FormatCode`` and ``FormatFile``, these -share several arguments which are described below: - -.. code-block:: python - - >>> from yapf.yapflib.yapf_api import FormatCode # reformat a string of code - - >>> FormatCode("f ( a = 1, b = 2 )") - 'f(a=1, b=2)\n' - -A ``style_config`` argument: Either a style name or a path to a file that contains -formatting style settings. If None is specified, use the default style -as set in ``style.DEFAULT_STYLE_FACTORY``. - -.. code-block:: python - - >>> FormatCode("def g():\n return True", style_config='pep8') - 'def g():\n return True\n' - -A ``lines`` argument: A list of tuples of lines (ints), [start, end], -that we want to format. The lines are 1-based indexed. It can be used by -third-party code (e.g., IDEs) when reformatting a snippet of code rather -than a whole file. - -.. code-block:: python - - >>> FormatCode("def g( ):\n a=1\n b = 2\n return a==b", lines=[(1, 1), (2, 3)]) - 'def g():\n a = 1\n b = 2\n return a==b\n' - -A ``print_diff`` (bool): Instead of returning the reformatted source, return a -diff that turns the formatted source into reformatter source. - -.. code-block:: python - - >>> print(FormatCode("a==b", filename="foo.py", print_diff=True)) - --- foo.py (original) - +++ foo.py (reformatted) - @@ -1 +1 @@ - -a==b - +a == b - -Note: the ``filename`` argument for ``FormatCode`` is what is inserted into -the diff, the default is ````. - -``FormatFile`` returns reformatted code from the passed file along with its encoding: - -.. code-block:: python - - >>> from yapf.yapflib.yapf_api import FormatFile # reformat a file - - >>> print(open("foo.py").read()) # contents of file - a==b - - >>> FormatFile("foo.py") - ('a == b\n', 'utf-8') - -The ``in-place`` argument saves the reformatted code back to the file: - -.. code-block:: python - - >>> FormatFile("foo.py", in_place=True) - (None, 'utf-8') - - >>> print(open("foo.py").read()) # contents of file (now fixed) - a == b - - -Knobs -===== - -``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` - Align closing bracket with visual indentation. - -``ALLOW_MULTILINE_LAMBDAS`` - Allow lambdas to be formatted on more than one line. - -``ALLOW_MULTILINE_DICTIONARY_KEYS`` - Allow dictionary keys to exist on multiple lines. For example: - - .. code-block:: python - - x = { - ('this is the first element of a tuple', - 'this is the second element of a tuple'): - value, - } - -``ALLOW_SPLIT_BEFORE_DICT_VALUE`` - Allow splits before the dictionary value. - -``BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF`` - Insert a blank line before a ``def`` or ``class`` immediately nested within - another ``def`` or ``class``. For example: - - .. code-block:: python - - class Foo: - # <------ this blank line - def method(): - pass - -``BLANK_LINE_BEFORE_CLASS_DOCSTRING`` - Insert a blank line before a class-level docstring. - -``COALESCE_BRACKETS`` - Do not split consecutive brackets. Only relevant when - ``DEDENT_CLOSING_BRACKETS`` is set. For example: - - .. code-block:: python - - call_func_that_takes_a_dict( - { - 'key1': 'value1', - 'key2': 'value2', - } - ) - - would reformat to: - - .. code-block:: python - - call_func_that_takes_a_dict({ - 'key1': 'value1', - 'key2': 'value2', - }) - - -``COLUMN_LIMIT`` - The column limit (or max line-length) - -``CONTINUATION_INDENT_WIDTH`` - Indent width used for line continuations. - -``DEDENT_CLOSING_BRACKETS`` - Put closing brackets on a separate line, dedented, if the bracketed - expression can't fit in a single line. Applies to all kinds of brackets, - including function definitions and calls. For example: - - .. code-block:: python - - config = { - 'key1': 'value1', - 'key2': 'value2', - } # <--- this bracket is dedented and on a separate line - - time_series = self.remote_client.query_entity_counters( - entity='dev3246.region1', - key='dns.query_latency_tcp', - transform=Transformation.AVERAGE(window=timedelta(seconds=60)), - start_ts=now()-timedelta(days=3), - end_ts=now(), - ) # <--- this bracket is dedented and on a separate line - -``EACH_DICT_ENTRY_ON_SEPARATE_LINE`` - Place each dictionary entry onto its own line. - -``I18N_COMMENT`` - The regex for an internationalization comment. The presence of this comment - stops reformatting of that line, because the comments are required to be - next to the string they translate. - -``I18N_FUNCTION_CALL`` - The internationalization function call names. The presence of this function - stops reformatting on that line, because the string it has cannot be moved - away from the i18n comment. - -``INDENT_DICTIONARY_VALUE`` - Indent the dictionary value if it cannot fit on the same line as the - dictionary key. For example: - - .. code-block:: python - - config = { - 'key1': - 'value1', - 'key2': value1 + - value2, - } - -``INDENT_WIDTH`` - The number of columns to use for indentation. - -``JOIN_MULTIPLE_LINES`` - Join short lines into one line. E.g., single line ``if`` statements. - -``SPACES_AROUND_POWER_OPERATOR`` - Set to ``True`` to prefer using spaces around ``**``. - -``NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS`` - Do not include spaces around selected binary operators. For example: - - .. code-block:: python - - 1 + 2 * 3 - 4 / 5 - - will be formatted as follows when configured with a value ``"*,/"``: - - .. code-block:: python - - 1 + 2*3 - 4/5 - -``SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN`` - Set to ``True`` to prefer spaces around the assignment operator for default - or keyword arguments. - -``SPACES_BEFORE_COMMENT`` - The number of spaces required before a trailing comment. - -``SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET`` - Insert a space between the ending comma and closing bracket of a list, etc. - -``SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED`` - Split before arguments if the argument list is terminated by a comma. - -``SPLIT_BEFORE_BITWISE_OPERATOR`` - Set to ``True`` to prefer splitting before ``&``, ``|`` or ``^`` rather - than after. - -``SPLIT_BEFORE_DICT_SET_GENERATOR`` - Split before a dictionary or set generator (comp_for). For example, note - the split before the ``for``: - - .. code-block:: python - - foo = { - variable: 'Hello world, have a nice day!' - for variable in bar if variable != 42 - } - -``SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN`` - Split after the opening paren which surrounds an expression if it doesn't - fit on a single line. - -``SPLIT_BEFORE_FIRST_ARGUMENT`` - If an argument / parameter list is going to be split, then split before the - first argument. - -``SPLIT_BEFORE_LOGICAL_OPERATOR`` - Set to ``True`` to prefer splitting before ``and`` or ``or`` rather than - after. - -``SPLIT_BEFORE_NAMED_ASSIGNS`` - Split named assignments onto individual lines. - -``SPLIT_PENALTY_AFTER_OPENING_BRACKET`` - The penalty for splitting right after the opening bracket. - -``SPLIT_PENALTY_AFTER_UNARY_OPERATOR`` - The penalty for splitting the line after a unary operator. - -``SPLIT_PENALTY_BEFORE_IF_EXPR`` - The penalty for splitting right before an ``if`` expression. - -``SPLIT_PENALTY_BITWISE_OPERATOR`` - The penalty of splitting the line around the ``&``, ``|``, and ``^`` - operators. - -``SPLIT_PENALTY_EXCESS_CHARACTER`` - The penalty for characters over the column limit. - -``SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT`` - The penalty incurred by adding a line split to the unwrapped line. The more - line splits added the higher the penalty. - -``SPLIT_PENALTY_IMPORT_NAMES`` - The penalty of splitting a list of ``import as`` names. For example: - - .. code-block:: python - - from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, - long_argument_2, - long_argument_3) - - would reformat to something like: - - .. code-block:: python - - from a_very_long_or_indented_module_name_yada_yad import ( - long_argument_1, long_argument_2, long_argument_3) - -``SPLIT_PENALTY_LOGICAL_OPERATOR`` - The penalty of splitting the line around the ``and`` and ``or`` operators. - -``USE_TABS`` - Use the Tab character for indentation. - -(Potentially) Frequently Asked Questions -======================================== - -Why does YAPF destroy my awesome formatting? --------------------------------------------- - -YAPF tries very hard to get the formatting correct. But for some code, it won't -be as good as hand-formatting. In particular, large data literals may become -horribly disfigured under YAPF. - -The reason for this is many-fold. But in essence YAPF is simply a tool to help -with development. It will format things to coincide with the style guide, but -that may not equate with readability. - -What can be done to alleviate this situation is to indicate regions YAPF should -ignore when reformatting something: - -.. code-block:: python - - # yapf: disable - FOO = { - # ... some very large, complex data literal. - } - - BAR = [ - # ... another large data literal. - ] - # yapf: enable - -You can also disable formatting for a single literal like this: - -.. code-block:: python - - BAZ = { - (1, 2, 3, 4), - (5, 6, 7, 8), - (9, 10, 11, 12), - } # yapf: disable - -To preserve the nice dedented closing brackets, use the -``dedent_closing_brackets`` in your style. Note that in this case all -brackets, including function definitions and calls, are going to use -that style. This provides consistency across the formatted codebase. - -Why Not Improve Existing Tools? -------------------------------- - -We wanted to use clang-format's reformatting algorithm. It's very powerful and -designed to come up with the best formatting possible. Existing tools were -created with different goals in mind, and would require extensive modifications -to convert to using clang-format's algorithm. - -Can I Use YAPF In My Program? ------------------------------ - -Please do! YAPF was designed to be used as a library as well as a command line -tool. This means that a tool or IDE plugin is free to use YAPF. - - -Gory Details -============ - -Algorithm Design ----------------- - -The main data structure in YAPF is the ``UnwrappedLine`` object. It holds a list -of ``FormatToken``\s, that we would want to place on a single line if there were -no column limit. An exception being a comment in the middle of an expression -statement will force the line to be formatted on more than one line. The -formatter works on one ``UnwrappedLine`` object at a time. - -An ``UnwrappedLine`` typically won't affect the formatting of lines before or -after it. There is a part of the algorithm that may join two or more -``UnwrappedLine``\s into one line. For instance, an if-then statement with a -short body can be placed on a single line: - -.. code-block:: python - - if a == 42: continue - -YAPF's formatting algorithm creates a weighted tree that acts as the solution -space for the algorithm. Each node in the tree represents the result of a -formatting decision --- i.e., whether to split or not to split before a token. -Each formatting decision has a cost associated with it. Therefore, the cost is -realized on the edge between two nodes. (In reality, the weighted tree doesn't -have separate edge objects, so the cost resides on the nodes themselves.) - -For example, take the following Python code snippet. For the sake of this -example, assume that line (1) violates the column limit restriction and needs to -be reformatted. - -.. code-block:: python - - def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee): # 1 - pass # 2 - -For line (1), the algorithm will build a tree where each node (a -``FormattingDecisionState`` object) is the state of the line at that token given -the decision to split before the token or not. Note: the ``FormatDecisionState`` -objects are copied by value so each node in the graph is unique and a change in -one doesn't affect other nodes. - -Heuristics are used to determine the costs of splitting or not splitting. -Because a node holds the state of the tree up to a token's insertion, it can -easily determine if a splitting decision will violate one of the style -requirements. For instance, the heuristic is able to apply an extra penalty to -the edge when not splitting between the previous token and the one being added. - -There are some instances where we will never want to split the line, because -doing so will always be detrimental (i.e., it will require a backslash-newline, -which is very rarely desirable). For line (1), we will never want to split the -first three tokens: ``def``, ``xxxxxxxxxxx``, and ``(``. Nor will we want to -split between the ``)`` and the ``:`` at the end. These regions are said to be -"unbreakable." This is reflected in the tree by there not being a "split" -decision (left hand branch) within the unbreakable region. - -Now that we have the tree, we determine what the "best" formatting is by finding -the path through the tree with the lowest cost. - -And that's it! - - +Text from original package has been elided. From 27c9267a22bd232efd4de365a0527e7c275d0cd5 Mon Sep 17 00:00:00 2001 From: Doug Greiman Date: Wed, 8 Nov 2017 17:00:41 -0800 Subject: [PATCH 4/4] Fix lint errors --- runtime/support.py | 8 ++++---- runtime/support_test.py | 7 ++++--- tests/package_pkg_resources/main.py | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/runtime/support.py b/runtime/support.py index 0e89192..0132e39 100644 --- a/runtime/support.py +++ b/runtime/support.py @@ -106,7 +106,6 @@ def _listdir(self, fspath): result = self._index().get(zipinfo_name, ()) return list(result) - def find_dist_info_in_zip(importer, path_item, only=False): """Find dist-info style metadata in zip files. @@ -117,13 +116,14 @@ def find_dist_info_in_zip(importer, path_item, only=False): for subitem in metadata.resource_listdir('/'): if subitem.lower().endswith('.dist-info'): subpath = os.path.join(path_item, subitem) - submeta = pkg_resources.EggMetadata(zipimport.zipimporter(subpath)) + submeta = pkg_resources.EggMetadata( + zipimport.zipimporter(subpath)) submeta.egg_info = subpath - dist = pkg_resources.Distribution.from_location(path_item, subitem, submeta) + dist = pkg_resources.Distribution.from_location( + path_item, subitem, submeta) yield dist return - def find_eggs_and_dist_info_in_zip(importer, path_item, only=False): """Chain together our finder and the standard pkg_resources finder diff --git a/runtime/support_test.py b/runtime/support_test.py index 2a75334..3ec1063 100644 --- a/runtime/support_test.py +++ b/runtime/support_test.py @@ -47,7 +47,7 @@ def test_setup(self): # similar. Get that lunacy out of the way before starting # test try: - import pip + import pip # noqa except ImportError: pass @@ -60,8 +60,9 @@ def test_setup(self): sys.path = old_sys_path self.assertTrue(mock_sys_path[1].endswith('subpar/runtime/some_root'), mock_sys_path) - self.assertTrue(mock_sys_path[2].endswith('subpar/runtime/another_root'), - mock_sys_path) + self.assertTrue( + mock_sys_path[2].endswith('subpar/runtime/another_root'), + mock_sys_path) self.assertEqual(mock_sys_path[0], sys.path[0]) self.assertEqual(mock_sys_path[3:], sys.path[1:]) diff --git a/tests/package_pkg_resources/main.py b/tests/package_pkg_resources/main.py index 2888bc2..dea801d 100644 --- a/tests/package_pkg_resources/main.py +++ b/tests/package_pkg_resources/main.py @@ -18,6 +18,7 @@ inside a .par file. """ + def main(): print('In pkg_resources test main()') try: @@ -39,5 +40,6 @@ def main(): dist = ws.find(pkg_resources.Requirement.parse(spec)) assert dist, (spec, distributions) + if __name__ == '__main__': main()