diff --git a/CHANGES.txt b/CHANGES.txt index 3851329e1..58603e31f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,9 +3,22 @@ Changes All issue numbers are relative to https://github.com/Toblerity/Fiona/issues. -1.01a2 (TBD) +1.10a2 (TBD) ------------ +Deprecations: + +The Python style of rio-filter expressions introduced in version 1.0 are +deprecated. Only the parenthesized list type of expression will be supported by +version 2.0. + +New features: + +The filter, map, and reduce CLI commands from the public domain version 1.1.0 +of fio-planet have been incorporated into Fiona's core set of commands (#1362). +These commands are only available if pyparsing and shapely (each of these are +declared in the "calc" set of extra requirements) are installed. + Bug fixes: - Add a 16-bit integer type "int16" based on OGR's OSFTInt16 integer sub-type diff --git a/Makefile b/Makefile index bce5ab676..41b1f33dd 100644 --- a/Makefile +++ b/Makefile @@ -33,10 +33,11 @@ dockertestimage: docker build --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona:$(GDAL)-py$(PYTHON_VERSION) . dockertest: dockertestimage - docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' + docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable .[all] --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' dockershell: dockertestimage docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /bin/bash' + dockersdist: dockertestimage docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m build --sdist' @@ -50,4 +51,4 @@ dockertestimage-amd64: docker build --platform linux/amd64 --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) . dockertest-amd64: dockertestimage-amd64 - docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' + docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable .[all] --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)' diff --git a/docs/cli.rst b/docs/cli.rst index 97fdfeced..b9c4b48b7 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,12 +10,15 @@ Fiona's new command line interface is a program named "fio". Fiona command line interface. Options: - -v, --verbose Increase verbosity. - -q, --quiet Decrease verbosity. - --version Show the version and exit. - --gdal-version Show the version and exit. - --python-version Show the version and exit. - --help Show this message and exit. + -v, --verbose Increase verbosity. + -q, --quiet Decrease verbosity. + --aws-profile TEXT Select a profile from the AWS credentials file + --aws-no-sign-requests Make requests anonymously + --aws-requester-pays Requester pays data transfer costs + --version Show the version and exit. + --gdal-version Show the version and exit. + --python-version Show the version and exit. + --help Show this message and exit. Commands: bounds Print the extent of GeoJSON objects @@ -25,21 +28,21 @@ Fiona's new command line interface is a program named "fio". distrib Distribute features from a collection. dump Dump a dataset to GeoJSON. env Print information about the fio environment. - filter Filter GeoJSON features by python expression. + filter Evaluate pipeline expressions to filter GeoJSON features. info Print information about a dataset. insp Open a dataset and start an interpreter. load Load GeoJSON to a dataset in another format. ls List layers in a datasource. + map Map a pipeline expression over GeoJSON features. + reduce Reduce a stream of GeoJSON features to one value. rm Remove a datasource or an individual layer. -It is developed using the ``click`` package and is new in 1.1.6. +It is developed using the ``click`` package. bounds ------ -New in 1.4.5. - -Fio-bounds reads LF or RS-delimited GeoJSON texts, either features or +The bounds command reads LF or RS-delimited GeoJSON texts, either features or collections, from stdin and prints their bounds with or without other data to stdout. @@ -62,8 +65,6 @@ Using ``--with-id`` gives you calc ---- -New in 1.7b1 - The calc command creates a new property on GeoJSON features using the specified expression. @@ -81,6 +82,11 @@ will not be overwritten by default (an `Exception` is raised). $ fio cat data.shp | fio calc sumAB "f.properties.A + f.properties.B" +.. note:: + + ``fio calc`` requires installation of the "calc" set of extra requirements + that will be installed by ``pip install fiona[calc]``. + cat --- @@ -102,8 +108,6 @@ concatenated datasets. $ fio info docs/data/test_uk.shp --count 48 -New in 1.4.0. - The cat command provides optional methods to filter data, which are different to the ``fio filter`` tool. A bounding box ``--bbox w,s,e,n`` tests for a spatial intersection with @@ -125,8 +129,6 @@ as the output of ``fio cat`` and writes a GeoJSON feature collection. $ fio info /tmp/collected.json --count 96 -New in 1.4.0. - distrib ------- @@ -140,8 +142,6 @@ and writes a JSON text sequence of GeoJSON feature objects. $ fio cat tests/data/coutwildrnp.shp | fio collect | fio distrib | wc -l 67 -New in 1.4.0. - dump ---- @@ -274,41 +274,284 @@ collection into a feature sequence. filter ------ -The filter command reads GeoJSON features from stdin and writes the feature to -stdout *if* the provided expression evalutates to `True` for that feature. -The python expression is evaluated in a restricted namespace containing 3 functions -(`sum`, `min`, `max`), the `math` module, the shapely `shape` function, -and an object `f` representing the feature to be evaluated. This `f` object allows -access in javascript-style dot notation for convenience. +For each feature read from stdin, filter evaluates a pipeline of one or +more steps described using methods from the Shapely library in Lisp-like +expressions. If the pipeline expression evaluates to True, the feature passes +through the filter. Otherwise the feature does not pass. -If the expression evaluates to a "truthy" value, the feature is printed verbatim. -Otherwise, the feature is excluded from the output. +For example, this pipeline expression .. code-block:: console - $ fio cat data.shp \ - > | fio filter "f.properties.area > 1000.0" \ - > | fio collect > large_polygons.geojson + $ fio cat zip+https://s3.amazonaws.com/fiona-testing/coutwildrnp.zip \ + | fio filter '< (distance g (Point -109.0 38.5)) 100' + +lets through all features that are less than 100 meters from the given point +and filters out all other features. + +*New in version 1.10*: these parenthesized list expressions. -Would create a geojson file with only those features from `data.shp` where the -area was over a given threshold. +The older style Python expressions like -Note this tool is different than ``fio cat --where TEXT ...``, which provides +.. code-block:: + + 'f.properties.area > 1000.0' + +are deprecated and will not be supported in version 2.0. + +Note this tool is different from ``fio cat --where TEXT ...``, which provides SQL WHERE clause filtering of feature attributes. +.. note:: + + ``fio filter`` requires installation of the "calc" set of extra requirements + that will be installed by ``pip install fiona[calc]``. + +map +--- + +For each feature read from stdin, ``fio map`` applies a transformation pipeline and +writes a copy of the feature, containing the modified geometry, to stdout. For +example, polygonal features can be roughly "cleaned" by using a ``buffer g 0`` +pipeline. + +.. code-block:: console + + $ fio cat zip+https://s3.amazonaws.com/fiona-testing/coutwildrnp.zip \ + | fio map 'buffer g 0' + +*New in version 1.10*. + +.. note:: + + ``fio map`` requires installation of the "calc" set of extra requirements + that will be installed by ``pip install fiona[calc]``. + + +reduce +------ + +Given a sequence of GeoJSON features (RS-delimited or not) on stdin this prints +a single value using a provided transformation pipeline. The set of geometries +of the input features in the context of these expressions is named ``c``. + +For example, the pipeline expression + +.. code-block:: console + + $ fio cat zip+https://s3.amazonaws.com/fiona-testing/coutwildrnp.zip \ + | fio reduce 'unary_union c' + +dissolves the geometries of input features. + +*New in version 1.10*. + +.. note:: + + ``fio reduce`` requires installation of the "calc" set of extra requirements + that will be installed by ``pip install fiona[calc]``. + rm -- -The ``fio rm`` command deletes an entire datasource or a single layer in a -multi-layer datasource. If the datasource is composed of multiple files -(e.g. an ESRI Shapefile) all of the files will be removed. + +The rm command deletes an entire datasource or a single layer in a multi-layer +datasource. If the datasource is composed of multiple files (e.g. an ESRI +Shapefile) all of the files will be removed. .. code-block:: console $ fio rm countries.shp $ fio rm --layer forests land_cover.gpkg -New in 1.8.0. +Expressions and functions +------------------------- + +``fio filter``, ``fio map``, and ``fio reduce`` expressions take the form of +parenthesized lists that may contain other expressions. The first item in a +list is the name of a function or method, or an expression that evaluates to a +function. The second item is the function's first argument or the object to +which the method is bound. The remaining list items are the positional and +keyword arguments for the named function or method. The list of functions and +callables available in an expression includes: + +* Python operators such as ``+``, ``/``, and ``<=`` +* Python builtins such as ``dict``, ``list``, and ``map`` +* All public functions from itertools, e.g. ``islice``, and ``repeat`` +* All functions importable from Shapely 2.0, e.g. ``Point``, and ``unary_union`` +* All methods of Shapely geometry classes +* Functions specific to Fiona + +Expressions are evaluated by ``fiona.features.snuggs.eval()``. Let's look at +some examples using that function. + +.. note:: + + The outer parentheses are not optional within ``snuggs.eval()``. + +.. note:: + + ``snuggs.eval()`` does not use Python's builtin ``eval()`` but isn't intended + to be a secure computing environment. Expressions which access the + computer's filesystem and create new processes are possible. + +Builtin Python functions +------------------------ + +``bool()`` + +.. code-block:: python + + >>> snuggs.eval('(bool 0)') + False + +``range()`` + +.. code-block:: python + + >>> snuggs.eval('(range 1 4)') + range(1, 4) + +``list()`` + +.. code-block:: python + + >>> snuggs.eval('(list (range 1 4))') + [1, 2, 3] + +Values can be bound to names for use in expressions. + +.. code-block:: python + + >>> snuggs.eval('(list (range start stop))', start=0, stop=5) + [0, 1, 2, 3, 4] + +Itertools functions +------------------- + +Here's an example of using ``itertools.repeat()``. + +.. code-block:: python + + >>> snuggs.eval('(list (repeat "*" times))', times=6) + ['*', '*', '*', '*', '*', '*'] + +Shapely functions +----------------- + +Here's an expression that evaluates to a Shapely Point instance. + +.. code-block:: python + + >>> snuggs.eval('(Point 0 0)') + + +The expression below evaluates to a MultiPoint instance. + +.. code-block:: python + + >>> snuggs.eval('(union (Point 0 0) (Point 1 1))') + + +Functions specific to fiona +--------------------------- + +The fio CLI introduces four new functions not available in Python's standard +library, or Shapely: ``collect()``, ``dump()``, ``identity()``, and +``vertex_count()``. + +The ``collect()`` function turns a list of geometries into a geometry +collection and ``dump()`` does the inverse, turning a geometry collection into +a sequence of geometries. + +.. code-block:: python + + >>> snuggs.eval('(collect (Point 0 0) (Point 1 1))') + + >>> snuggs.eval('(list (dump (collect (Point 0 0) (Point 1 1))))') + [, ] + +The ``identity()`` function returns its single argument. + +.. code-block:: python + + >>> snuggs.eval('(identity 42)') + 42 + +To count the number of vertices in a geometry, use ``vertex_count()``. + +.. code-block:: python + + >>> snuggs.eval('(vertex_count (Point 0 0))') + 1 + +The ``area()``, ``buffer()``, ``distance()``, ``length()``, ``simplify()``, and +``set_precision()`` functions shadow, or override, functions from the shapely +module. They automatically reproject geometry objects from their natural +coordinate reference system (CRS) of ``OGC:CRS84`` to ``EPSG:6933`` so that the +shapes can be measured or modified using meters as units. + +``buffer()`` dilates (or erodes) a given geometry, with coordinates in decimal +longitude and latitude degrees, by a given distance in meters. + +.. code-block:: python + + >>> snuggs.eval('(buffer (Point 0 0) :distance 100)') + + +The ``area()`` and ``length()`` of this polygon have units of square meter and +meter. + +.. code-block:: python + + >>> snuggs.eval('(area (buffer (Point 0 0) :distance 100))') + 31214.451487413342 + >>> snuggs.eval('(length (buffer (Point 0 0) :distance 100))') + 627.3096977558143 + +The ``distance()`` between two geometries is in meters. + +.. code-block:: python + + >>> snuggs.eval('(distance (Point 0 0) (Point 0.1 0.1))') + 15995.164946207413 + +A geometry can be simplified to a tolerance value in meters using +``simplify()``. There are more examples of this function later in this +document. + +.. code-block:: python + + >>> snuggs.eval('(simplify (buffer (Point 0 0) :distance 100) :tolerance 100)') + + +The ``set_precision()`` function snaps a geometry to a fixed precision grid +with a size in meters. + +.. code-block:: python + + >>> snuggs.eval('(set_precision (Point 0.001 0.001) :grid_size 500)') + + +Feature and geometry context for expressions +-------------------------------------------- + +``fio filter`` and ``fio map`` evaluate expressions in the context of a GeoJSON +feature and its geometry attribute. These are named ``f`` and ``g``. For example, +here is an expression that tests whether the input feature is within 62.5 +kilometers of the given point. + +.. code-block:: lisp + + < (distance g (Point 4 43)) 62.5E3 + +``fio reduce`` evaluates expressions in the context of the sequence of all input +geometries, named ``c``. For example, this expression dissolves input +geometries using Shapely's ``unary_union``. + +.. code-block:: lisp + + unary_union c Coordinate Reference System Transformations ------------------------------------------- @@ -337,4 +580,101 @@ by fio cat. The following, does the same thing, but for ESRI Shapefile output. -New in 1.4.2. +Sizing up and simplifying shapes +-------------------------------- + +The following examples use the program ``jq`` and a 25-feature shapefile. You +can get the data from from `rmnp.zip +`__ or access it in +a streaming fashion as shown in the examples below. + +Counting vertices in a feature collection ++++++++++++++++++++++++++++++++++++++++++ + +The builtin ``vertex_count()`` function, in conjunction with ``fio map``'s +``--raw`` option, prints out the number of vertices in each feature. The +default for fio-map is to wrap the result of every evaluated expression in +a GeoJSON feature; ``--raw`` disables this. The program ``jq`` provides a nice +way of summing the sequence of numbers. + +.. code-block:: console + + fio cat zip+https://github.com/Toblerity/Fiona/files/14749922/rmnp.zip \ + | fio map 'vertex_count g' --raw \ + | jq -s 'add' + 28915 + +Here's what the RMNP wilderness patrol zones features look like in QGIS. + +.. image:: img/zones.png + +Counting vertices after making a simplified buffer +++++++++++++++++++++++++++++++++++++++++++++++++++ + +One traditional way of simplifying an area of interest is to buffer and +simplify. There's no need to use ``jq`` here because ``fio reduce`` prints out +a sequence of exactly one feature. The effectiveness of this method depends +a bit on the nature of the data, especially the distance between vertices. The +total length of the perimeters of all zones is 889 kilometers. + +.. code-block:: console + + fio cat zip+https://github.com/Toblerity/Fiona/files/14749922/rmnp.zip \ + | fio map 'length g' --raw \ + | jq -s 'add' + 889332.0900809917 + +The mean distance between vertices on the edges of zones is 889332 / 28915, or +30.7 meters. You need to buffer and simplify by this value or more to get +a significant reduction in the number of vertices. Choosing 40 as a buffer +distance and simplification tolerance results in a shape with 469 vertices. +It's a suitable area of interest for applications that require this number to +be less than 500. + +.. code-block:: console + + fio cat zip+https://github.com/Toblerity/Fiona/files/14749922/rmnp.zip \ + | fio reduce 'unary_union c' \ + | fio map 'simplify (buffer g 40) 40' \ + | fio map 'vertex_count g' --raw + 469 + +.. image:: img/simplified-buffer.png + +Counting vertices after dissolving convex hulls of features ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Convex hulls are an easy means of simplification as there are no distance +parameters to tweak. The ``--dump-parts`` option of ``fio map`` turns the parts of +multi-part features into separate single-part features. This is one of the ways +in which fio-map can multiply its inputs, printing out more features than it +receives. + +.. code-block:: console + + fio cat zip+https://github.com/Toblerity/Fiona/files/14749922/rmnp.zip \ + | fio map 'convex_hull g' --dump-parts \ + | fio reduce 'unary_union c' \ + | fio map 'vertex_count g' --raw + 157 + +.. image:: img/convex.png + +Counting vertices after dissolving concave hulls of features +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Convex hulls simplify, but also dilate concave areas of interest. They fill the +"bays", so to speak, and this can be undesirable. Concave hulls do a better job +at preserving the concave nature of a shape and result in a smaller increase of +area. + +.. code-block:: console + + fio cat zip+https://github.com/Toblerity/Fiona/files/14749922/rmnp.zip \ + | fio map 'concave_hull g :ratio 0.4' --dump-parts \ + | fio reduce 'unary_union c' \ + | fio map 'vertex_count g' --raw + 301 + +.. image:: img/concave.png + diff --git a/docs/img/concave.png b/docs/img/concave.png new file mode 100644 index 000000000..e8abc90e0 Binary files /dev/null and b/docs/img/concave.png differ diff --git a/docs/img/convex.png b/docs/img/convex.png new file mode 100644 index 000000000..ed5072a5f Binary files /dev/null and b/docs/img/convex.png differ diff --git a/docs/img/simplified-buffer.png b/docs/img/simplified-buffer.png new file mode 100644 index 000000000..aa9787f30 Binary files /dev/null and b/docs/img/simplified-buffer.png differ diff --git a/docs/img/zones.png b/docs/img/zones.png new file mode 100644 index 000000000..5024f8707 Binary files /dev/null and b/docs/img/zones.png differ diff --git a/fiona/_vendor/snuggs.py b/fiona/_vendor/snuggs.py new file mode 100644 index 000000000..638f39796 --- /dev/null +++ b/fiona/_vendor/snuggs.py @@ -0,0 +1,298 @@ +"""Snuggs are s-expressions for Numpy.""" + +# This file is a modified version of snuggs 1.4.7. The numpy +# requirement has been removed and support for keyword arguments in +# expressions has been added. +# +# The original license follows. +# +# Copyright (c) 2014 Mapbox +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from collections import OrderedDict +import functools +import operator +import re +from typing import Mapping + +from pyparsing import ( # type: ignore + Keyword, + oneOf, + Literal, + QuotedString, + ParseException, + Forward, + Group, + OneOrMore, + ParseResults, + Regex, + ZeroOrMore, + alphanums, + pyparsing_common, + replace_with, +) + +__all__ = ["eval"] +__version__ = "1.4.7" + + +class Context(object): + def __init__(self): + self._data = OrderedDict() + + def add(self, name, val): + self._data[name] = val + + def get(self, name): + return self._data[name] + + def lookup(self, index, subindex=None): + s = list(self._data.values())[int(index) - 1] + if subindex: + return s[int(subindex) - 1] + else: + return s + + def clear(self): + self._data = OrderedDict() + + +_ctx = Context() + + +class ctx(object): + def __init__(self, kwd_dict=None, **kwds): + self.kwds = kwd_dict or kwds + + def __enter__(self): + _ctx.clear() + for k, v in self.kwds.items(): + _ctx.add(k, v) + return self + + def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): + self.kwds = None + _ctx.clear() + + +class ExpressionError(SyntaxError): + """A Snuggs-specific syntax error.""" + + filename = "" + lineno = 1 + + +op_map = { + "*": lambda *args: functools.reduce(lambda x, y: operator.mul(x, y), args), + "+": lambda *args: functools.reduce(lambda x, y: operator.add(x, y), args), + "/": lambda *args: functools.reduce(lambda x, y: operator.truediv(x, y), args), + "-": lambda *args: functools.reduce(lambda x, y: operator.sub(x, y), args), + "&": lambda *args: functools.reduce(lambda x, y: operator.and_(x, y), args), + "|": lambda *args: functools.reduce(lambda x, y: operator.or_(x, y), args), + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, + "truth": operator.truth, + "is": operator.is_, + "not": operator.not_, +} + + +def compose(f, g): + """Compose two functions. + + compose(f, g)(x) = f(g(x)). + + """ + return lambda x, *args, **kwds: f(g(x)) + + +func_map: Mapping = {} + +higher_func_map: Mapping = { + "compose": compose, + "map": map, + "partial": functools.partial, + "reduce": functools.reduce, + "attrgetter": operator.attrgetter, + "methodcaller": operator.methodcaller, + "itemgetter": operator.itemgetter, +} + +nil = Keyword("null").set_parse_action(replace_with(None)) +true = Keyword("true").set_parse_action(replace_with(True)) +false = Keyword("false").set_parse_action(replace_with(False)) + + +def resolve_var(source, loc, toks): + try: + return _ctx.get(toks[0]) + except KeyError: + err = ExpressionError("name '{}' is not defined".format(toks[0])) + err.text = source + err.offset = loc + 1 + raise err + + +var = pyparsing_common.identifier.set_parse_action(resolve_var) +string = QuotedString("'") | QuotedString('"') +lparen = Literal("(").suppress() +rparen = Literal(")").suppress() +op = oneOf(" ".join(op_map.keys())).set_parse_action( + lambda source, loc, toks: op_map[toks[0]] +) + + +def resolve_func(source, loc, toks): + try: + return func_map[toks[0]] + except (AttributeError, KeyError): + err = ExpressionError("'{}' is not a function or operator".format(toks[0])) + err.text = source + err.offset = loc + 1 + raise err + + +# The look behind assertion is to disambiguate between functions and +# variables. +func = Regex(r"(?<=\()[{}]+".format(alphanums + "_")).set_parse_action(resolve_func) + +higher_func = oneOf(" ".join(higher_func_map.keys())).set_parse_action( + lambda source, loc, toks: higher_func_map[toks[0]] +) + +func_expr = Forward() +higher_func_expr = Forward() +expr = higher_func_expr | func_expr + + +class KeywordArg: + def __init__(self, name): + self.name = name + + +kwarg = Regex(r":[{}]+".format(alphanums + "_")).set_parse_action( + lambda source, loc, toks: KeywordArg(toks[0][1:]) +) + +operand = ( + higher_func_expr + | func_expr + | true + | false + | nil + | var + | kwarg + | pyparsing_common.sci_real + | pyparsing_common.real + | pyparsing_common.signed_integer + | string +) + +func_expr << Group( + lparen + (higher_func_expr | op | func) + OneOrMore(operand) + rparen +) + +higher_func_expr << Group( + lparen + + higher_func + + (nil | higher_func_expr | op | func | OneOrMore(operand)) + + ZeroOrMore(operand) + + rparen +) + + +def processArg(arg): + if isinstance(arg, ParseResults): + return processList(arg) + else: + return arg + + +def processList(lst): + items = [processArg(x) for x in lst[1:]] + args = [] + kwds = {} + + # An iterator is used instead of implicit iteration to allow + # skipping ahead in the keyword argument case. + itemitr = iter(items) + + for item in itemitr: + if isinstance(item, KeywordArg): + # The next item after the keyword arg marker is its value. + # This advances the iterator in a way that is compatible + # with the for loop. + val = next(itemitr) + key = item.name + kwds[key] = val + else: + args.append(item) + + func = processArg(lst[0]) + + # list and tuple are two builtins that take a single argument, + # whereas args is a list. On a KeyError, the call is retried + # without arg unpacking. + try: + return func(*args, **kwds) + except TypeError: + return func(args, **kwds) + + +def handleLine(line): + try: + result = expr.parseString(line) + return processList(result[0]) + except ParseException as exc: + text = str(exc) + m = re.search(r"(Expected .+) \(at char (\d+)\), \(line:(\d+)", text) + msg = m.group(1) + if "map|partial" in msg: + msg = "expected a function or operator" + err = ExpressionError(msg) + err.text = line + err.offset = int(m.group(2)) + 1 + raise err + + +def eval(source, kwd_dict=None, **kwds): + """Evaluate a snuggs expression. + + Parameters + ---------- + source : str + Expression source. + kwd_dict : dict + A dict of items that form the evaluation context. Deprecated. + kwds : dict + A dict of items that form the valuation context. + + Returns + ------- + object + + """ + kwd_dict = kwd_dict or kwds + with ctx(kwd_dict): + return handleLine(source) diff --git a/fiona/errors.py b/fiona/errors.py index 66e8d86d6..8f857d0e1 100644 --- a/fiona/errors.py +++ b/fiona/errors.py @@ -89,3 +89,7 @@ class FionaDeprecationWarning(DeprecationWarning): class FeatureWarning(UserWarning): """A warning about serialization of a feature""" + + +class ReduceError(FionaError): + """"Raised when reduce operation fails.""" diff --git a/fiona/features.py b/fiona/features.py new file mode 100644 index 000000000..4afd1f3b7 --- /dev/null +++ b/fiona/features.py @@ -0,0 +1,316 @@ +"""Operations on GeoJSON feature and geometry objects.""" + +from collections import UserDict +from functools import wraps +import itertools +from typing import Generator, Iterable, Mapping, Union + +from fiona.transform import transform_geom # type: ignore +import shapely # type: ignore +import shapely.ops # type: ignore +from shapely.geometry import mapping, shape # type: ignore +from shapely.geometry.base import BaseGeometry, BaseMultipartGeometry # type: ignore + +from .errors import ReduceError +from ._vendor import snuggs + +# Patch snuggs's func_map, extending it with Python builtins, geometry +# methods and attributes, and functions exported in the shapely module +# (such as set_precision). + + +class FuncMapper(UserDict, Mapping): + """Resolves functions from names in pipeline expressions.""" + + def __getitem__(self, key): + """Get a function by its name.""" + if key in self.data: + return self.data[key] + elif key in __builtins__ and not key.startswith("__"): + return __builtins__[key] + elif key in dir(shapely): + return lambda g, *args, **kwargs: getattr(shapely, key)(g, *args, **kwargs) + elif key in dir(shapely.ops): + return lambda g, *args, **kwargs: getattr(shapely.ops, key)( + g, *args, **kwargs + ) + else: + return ( + lambda g, *args, **kwargs: getattr(g, key)(*args, **kwargs) + if callable(getattr(g, key)) + else getattr(g, key) + ) + + +def collect(geoms: Iterable) -> object: + """Turn a sequence of geometries into a single GeometryCollection. + + Parameters + ---------- + geoms : Iterable + A sequence of geometry objects. + + Returns + ------- + Geometry + + """ + return shapely.GeometryCollection(list(geoms)) + + +def dump(geom: Union[BaseGeometry, BaseMultipartGeometry]) -> Generator: + """Get the individual parts of a geometry object. + + If the given geometry object has a single part, e.g., is an + instance of LineString, Point, or Polygon, this function yields a + single result, the geometry itself. + + Parameters + ---------- + geom : a shapely geometry object. + + Yields + ------ + A shapely geometry object. + + """ + if hasattr(geom, "geoms"): + parts = geom.geoms + else: + parts = [geom] + for part in parts: + yield part + + +def identity(obj: object) -> object: + """Get back the given argument. + + To help in making expression lists, where the first item must be a + callable object. + + Parameters + ---------- + obj : objeect + + Returns + ------- + obj + + """ + return obj + + +def vertex_count(obj: object) -> int: + """Count the vertices of a GeoJSON-like geometry object. + + Parameters + ---------- + obj: object + A GeoJSON-like mapping or an object that provides + __geo_interface__. + + Returns + ------- + int + + """ + shp = shape(obj) + if hasattr(shp, "geoms"): + return sum(vertex_count(part) for part in shp.geoms) + elif hasattr(shp, "exterior"): + return vertex_count(shp.exterior) + sum( + vertex_count(ring) for ring in shp.interiors + ) + else: + return len(shp.coords) + + +def binary_projectable_property_wrapper(func): + """Project func's geometry args before computing a property. + + Parameters + ---------- + func : callable + Signature is func(geom1, geom2, *args, **kwargs) + + Returns + ------- + callable + Signature is func(geom1, geom2, projected=True, *args, **kwargs) + + """ + + @wraps(func) + def wrapper(geom1, geom2, *args, projected=True, **kwargs): + if projected: + geom1 = shape(transform_geom("OGC:CRS84", "EPSG:6933", mapping(geom1))) + geom2 = shape(transform_geom("OGC:CRS84", "EPSG:6933", mapping(geom2))) + + return func(geom1, geom2, *args, **kwargs) + + return wrapper + + +def unary_projectable_property_wrapper(func): + """Project func's geometry arg before computing a property. + + Parameters + ---------- + func : callable + Signature is func(geom1, *args, **kwargs) + + Returns + ------- + callable + Signature is func(geom1, projected=True, *args, **kwargs) + + """ + + @wraps(func) + def wrapper(geom, *args, projected=True, **kwargs): + if projected: + geom = shape(transform_geom("OGC:CRS84", "EPSG:6933", mapping(geom))) + + return func(geom, *args, **kwargs) + + return wrapper + + +def unary_projectable_constructive_wrapper(func): + """Project func's geometry arg before constructing a new geometry. + + Parameters + ---------- + func : callable + Signature is func(geom1, *args, **kwargs) + + Returns + ------- + callable + Signature is func(geom1, projected=True, *args, **kwargs) + + """ + + @wraps(func) + def wrapper(geom, *args, projected=True, **kwargs): + if projected: + geom = shape(transform_geom("OGC:CRS84", "EPSG:6933", mapping(geom))) + product = func(geom, *args, **kwargs) + return shape(transform_geom("EPSG:6933", "OGC:CRS84", mapping(product))) + else: + return func(geom, *args, **kwargs) + + return wrapper + + +area = unary_projectable_property_wrapper(shapely.area) +buffer = unary_projectable_constructive_wrapper(shapely.buffer) +distance = binary_projectable_property_wrapper(shapely.distance) +set_precision = unary_projectable_constructive_wrapper(shapely.set_precision) +simplify = unary_projectable_constructive_wrapper(shapely.simplify) +length = unary_projectable_property_wrapper(shapely.length) + +snuggs.func_map = FuncMapper( + area=area, + buffer=buffer, + collect=collect, + distance=distance, + dump=dump, + identity=identity, + length=length, + simplify=simplify, + set_precision=set_precision, + vertex_count=vertex_count, + **{ + k: getattr(itertools, k) + for k in dir(itertools) + if not k.startswith("_") and callable(getattr(itertools, k)) + }, +) + + +def map_feature( + expression: str, feature: Mapping, dump_parts: bool = False +) -> Generator: + """Map a pipeline expression to a feature. + + Yields one or more values. + + Parameters + ---------- + expression : str + A snuggs expression. The outermost parentheses are optional. + feature : dict + A Fiona feature object. + dump_parts : bool, optional (default: False) + If True, the parts of the feature's geometry are turned into + new features. + + Yields + ------ + object + + """ + if not (expression.startswith("(") and expression.endswith(")")): + expression = f"({expression})" + + try: + geom = shape(feature.get("geometry", None)) + if dump_parts and hasattr(geom, "geoms"): + parts = geom.geoms + else: + parts = [geom] + except (AttributeError, KeyError): + parts = [None] + + for part in parts: + result = snuggs.eval(expression, g=part, f=feature) + if isinstance(result, (str, float, int, Mapping)): + yield result + elif isinstance(result, (BaseGeometry, BaseMultipartGeometry)): + yield mapping(result) + else: + try: + for item in result: + if isinstance(item, (BaseGeometry, BaseMultipartGeometry)): + item = mapping(item) + yield item + except TypeError: + yield result + + +def reduce_features(expression: str, features: Iterable[Mapping]) -> Generator: + """Reduce a collection of features to a single value. + + The pipeline is a string that, when evaluated by snuggs, produces + a new value. The name of the input feature collection in the + context of the pipeline is "c". + + Parameters + ---------- + pipeline : str + Geometry operation pipeline such as "(unary_union c)". + features : iterable + A sequence of Fiona feature objects. + + Yields + ------ + object + + Raises + ------ + ReduceError + + """ + if not (expression.startswith("(") and expression.endswith(")")): + expression = f"({expression})" + + collection = (shape(feat["geometry"]) for feat in features) + result = snuggs.eval(expression, c=collection) + + if isinstance(result, (str, float, int, Mapping)): + yield result + elif isinstance(result, (BaseGeometry, BaseMultipartGeometry)): + yield mapping(result) + else: + raise ReduceError("Expression failed to reduce to a single value.") diff --git a/fiona/fio/features.py b/fiona/fio/features.py new file mode 100644 index 000000000..201faf827 --- /dev/null +++ b/fiona/fio/features.py @@ -0,0 +1,267 @@ +"""Fiona CLI commands.""" + +from collections import defaultdict +from copy import copy +import itertools +import json +import logging +import warnings + +import click +from cligj import use_rs_opt # type: ignore + +from fiona.features import map_feature, reduce_features +from fiona.fio import with_context_env +from fiona.fio.helpers import obj_gen, eval_feature_expression # type: ignore + +log = logging.getLogger(__name__) + + +@click.command( + "map", + short_help="Map a pipeline expression over GeoJSON features.", +) +@click.argument("pipeline") +@click.option( + "--raw", + "-r", + is_flag=True, + default=False, + help="Print raw result, do not wrap in a GeoJSON Feature.", +) +@click.option( + "--no-input", + "-n", + is_flag=True, + default=False, + help="Do not read input from stream.", +) +@click.option( + "--dump-parts", + is_flag=True, + default=False, + help="Dump parts of geometries to create new inputs before evaluating pipeline.", +) +@use_rs_opt +def map_cmd(pipeline, raw, no_input, dump_parts, use_rs): + """Map a pipeline expression over GeoJSON features. + + Given a sequence of GeoJSON features (RS-delimited or not) on stdin + this prints copies with geometries that are transformed using a + provided transformation pipeline. In "raw" output mode, this + command prints pipeline results without wrapping them in a feature + object. + + The pipeline is a string that, when evaluated by fio-map, produces + a new geometry object. The pipeline consists of expressions in the + form of parenthesized lists that may contain other expressions. + The first item in a list is the name of a function or method, or an + expression that evaluates to a function. The second item is the + function's first argument or the object to which the method is + bound. The remaining list items are the positional and keyword + arguments for the named function or method. The names of the input + feature and its geometry in the context of these expressions are + "f" and "g". + + For example, this pipeline expression + + '(simplify (buffer g 100.0) 5.0)' + + buffers input geometries and then simplifies them so that no + vertices are closer than 5 units. Keyword arguments for the shapely + methods are supported. A keyword argument is preceded by ':' and + followed immediately by its value. For example: + + '(simplify g 5.0 :preserve_topology true)' + + and + + '(buffer g 100.0 :resolution 8 :join_style 1)' + + Numerical and string arguments may be replaced by expressions. The + buffer distance could be a function of a geometry's area. + + '(buffer g (/ (area g) 100.0))' + + """ + if no_input: + features = [None] + else: + stdin = click.get_text_stream("stdin") + features = obj_gen(stdin) + + for feat in features: + for i, value in enumerate(map_feature(pipeline, feat, dump_parts=dump_parts)): + if use_rs: + click.echo("\x1e", nl=False) + if raw: + click.echo(json.dumps(value)) + else: + new_feat = copy(feat) + new_feat["id"] = f"{feat.get('id', '0')}:{i}" + new_feat["geometry"] = value + click.echo(json.dumps(new_feat)) + + +@click.command( + "filter", + short_help="Evaluate pipeline expressions to filter GeoJSON features.", +) +@click.argument("pipeline") +@use_rs_opt +@click.option( + "--snuggs-only", + "-s", + is_flag=True, + default=False, + help="Strictly require snuggs style expressions and skip check for type of expression.", +) +@click.pass_context +@with_context_env +def filter_cmd(ctx, pipeline, use_rs, snuggs_only): + """Evaluate pipeline expressions to filter GeoJSON features. + + The pipeline is a string that, when evaluated, gives a new value for + each input feature. If the value evaluates to True, the feature + passes through the filter. Otherwise the feature does not pass. + + The pipeline consists of expressions in the form of parenthesized + lists that may contain other expressions. The first item in a list + is the name of a function or method, or an expression that evaluates + to a function. The second item is the function's first argument or + the object to which the method is bound. The remaining list items + are the positional and keyword arguments for the named function or + method. The names of the input feature and its geometry in the + context of these expressions are "f" and "g". + + For example, this pipeline expression + + '(< (distance g (Point 4 43)) 1)' + + lets through all features that are less than one unit from the given + point and filters out all other features. + + *New in version 1.10*: these parenthesized list expressions. + + The older style Python expressions like + + 'f.properties.area > 1000.0' + + are deprecated and will not be supported in version 2.0. + + """ + stdin = click.get_text_stream("stdin") + features = obj_gen(stdin) + + if not snuggs_only: + try: + from pyparsing.exceptions import ParseException + from fiona._vendor.snuggs import ExpressionError, expr + + if not pipeline.startswith("("): + test_string = f"({pipeline})" + expr.parseString(test_string) + except ExpressionError: + # It's a snuggs expression. + log.info("Detected a snuggs expression.") + pass + except ParseException: + # It's likely an old-style Python expression. + log.info("Detected a legacy Python expression.") + warnings.warn( + "This style of filter expression is deprecated. " + "Version 2.0 will only support the new parenthesized list expressions.", + FutureWarning, + ) + for i, obj in enumerate(features): + feats = obj.get("features") or [obj] + for j, feat in enumerate(feats): + if not eval_feature_expression(feat, pipeline): + continue + if use_rs: + click.echo("\x1e", nl=False) + click.echo(json.dumps(feat)) + return + + for feat in features: + for value in map_feature(pipeline, feat): + if value: + if use_rs: + click.echo("\x1e", nl=False) + click.echo(json.dumps(feat)) + + +@click.command("reduce", short_help="Reduce a stream of GeoJSON features to one value.") +@click.argument("pipeline") +@click.option( + "--raw", + "-r", + is_flag=True, + default=False, + help="Print raw result, do not wrap in a GeoJSON Feature.", +) +@use_rs_opt +@click.option( + "--zip-properties", + is_flag=True, + default=False, + help="Zip the items of input feature properties together for output.", +) +def reduce_cmd(pipeline, raw, use_rs, zip_properties): + """Reduce a stream of GeoJSON features to one value. + + Given a sequence of GeoJSON features (RS-delimited or not) on stdin + this prints a single value using a provided transformation pipeline. + + The pipeline is a string that, when evaluated, produces + a new geometry object. The pipeline consists of expressions in the + form of parenthesized lists that may contain other expressions. + The first item in a list is the name of a function or method, or an + expression that evaluates to a function. The second item is the + function's first argument or the object to which the method is + bound. The remaining list items are the positional and keyword + arguments for the named function or method. The set of geometries + of the input features in the context of these expressions is named + "c". + + For example, the pipeline expression + + '(unary_union c)' + + dissolves the geometries of input features. + + To keep the properties of input features while reducing them to a + single feature, use the --zip-properties flag. The properties of the + input features will surface in the output feature as lists + containing the input values. + + """ + stdin = click.get_text_stream("stdin") + features = (feat for feat in obj_gen(stdin)) + + if zip_properties: + prop_features, geom_features = itertools.tee(features) + properties = defaultdict(list) + for feat in prop_features: + for key, val in feat["properties"].items(): + properties[key].append(val) + else: + geom_features = features + properties = {} + + for result in reduce_features(pipeline, geom_features): + if use_rs: + click.echo("\x1e", nl=False) + if raw: + click.echo(json.dumps(result)) + else: + click.echo( + json.dumps( + { + "type": "Feature", + "properties": properties, + "geometry": result, + "id": "0", + } + ) + ) diff --git a/fiona/fio/filter.py b/fiona/fio/filter.py deleted file mode 100644 index 5e7e71804..000000000 --- a/fiona/fio/filter.py +++ /dev/null @@ -1,54 +0,0 @@ -"""$ fio filter""" - -import json -import logging - -import click -from cligj import use_rs_opt - -from fiona.fio.helpers import obj_gen, eval_feature_expression -from fiona.fio import with_context_env - - -logger = logging.getLogger(__name__) - - -@click.command() -@click.argument('filter_expression') -@use_rs_opt -@click.pass_context -@with_context_env -def filter(ctx, filter_expression, use_rs): - """ - Filter GeoJSON features by python expression. - - Features are read from stdin. - - The expression is evaluated in a restricted namespace containing: - - sum, pow, min, max and the imported math module - - shape (optional, imported from shapely.geometry if available) - - bool, int, str, len, float type conversions - - f (the feature to be evaluated, - allows item access via javascript-style dot notation using munch) - - The expression will be evaluated for each feature and, if true, - the feature will be included in the output. For example: - - \b - $ fio cat data.shp \\ - | fio filter "f.properties.area > 1000.0" \\ - | fio collect > large_polygons.geojson - - """ - stdin = click.get_text_stream('stdin') - source = obj_gen(stdin) - - for i, obj in enumerate(source): - features = obj.get("features") or [obj] - for j, feat in enumerate(features): - if not eval_feature_expression(feat, filter_expression): - continue - - if use_rs: - click.echo("\x1e", nl=False) - click.echo(json.dumps(feat)) diff --git a/fiona/fio/main.py b/fiona/fio/main.py index 8f6a6c760..1190a5ccc 100644 --- a/fiona/fio/main.py +++ b/fiona/fio/main.py @@ -19,6 +19,28 @@ import fiona from fiona import __version__ as fio_version from fiona.session import AWSSession, DummySession +from fiona.fio.bounds import bounds +from fiona.fio.calc import calc +from fiona.fio.cat import cat +from fiona.fio.collect import collect +from fiona.fio.distrib import distrib +from fiona.fio.dump import dump +from fiona.fio.env import env +from fiona.fio.info import info +from fiona.fio.insp import insp +from fiona.fio.load import load +from fiona.fio.ls import ls +from fiona.fio.rm import rm + +# The "calc" extras require pyparsing and shapely. +try: + import pyparsing + import shapely + from fiona.fio.features import filter_cmd, map_cmd, reduce_cmd + + supports_calc = True +except ImportError: + supports_calc = False def configure_logging(verbosity): @@ -28,7 +50,6 @@ def configure_logging(verbosity): @with_plugins( itertools.chain( - entry_points(group="fiona.fio_commands"), entry_points(group="fiona.fio_plugins"), ) ) @@ -71,3 +92,22 @@ def main_group( else: session = DummySession() ctx.obj["env"] = fiona.Env(session=session, **envopts) + + +main_group.add_command(bounds) +main_group.add_command(calc) +main_group.add_command(cat) +main_group.add_command(collect) +main_group.add_command(distrib) +main_group.add_command(dump) +main_group.add_command(env) +main_group.add_command(info) +main_group.add_command(insp) +main_group.add_command(load) +main_group.add_command(ls) +main_group.add_command(rm) + +if supports_calc: + main_group.add_command(map_cmd) + main_group.add_command(filter_cmd) + main_group.add_command(reduce_cmd) diff --git a/pyproject.toml b/pyproject.toml index 1a176bee0..486036c65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ [project.optional-dependencies] all = ["fiona[calc,s3,test]"] -calc = ["shapely"] +calc = ["pyparsing", "shapely"] s3 = ["boto3>=1.3.1"] test = [ "fiona[s3]", @@ -51,21 +51,6 @@ test = [ [project.scripts] fio = "fiona.fio.main:main_group" -[project.entry-points."fiona.fio_commands"] -bounds = "fiona.fio.bounds:bounds" -calc = "fiona.fio.calc:calc" -cat = "fiona.fio.cat:cat" -collect = "fiona.fio.collect:collect" -distrib = "fiona.fio.distrib:distrib" -dump = "fiona.fio.dump:dump" -env = "fiona.fio.env:env" -filter = "fiona.fio.filter:filter" -info = "fiona.fio.info:info" -insp = "fiona.fio.insp:insp" -load = "fiona.fio.load:load" -ls = "fiona.fio.ls:ls" -rm = "fiona.fio.rm:rm" - [project.urls] Documentation = "https://fiona.readthedocs.io/" Repository = "https://github.com/Toblerity/Fiona" diff --git a/requirements-dev.txt b/requirements-dev.txt index 637a1cff9..01a158f00 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,9 +5,11 @@ boto3>=1.3.1 coverage~=6.5 cython>=3 fsspec +pyparsing pytest~=7.2 pytest-cov~=4.0 pytz==2022.6 requests setuptools +shapely wheel diff --git a/setup.cfg b/setup.cfg index f1c7afc83..a61c037d0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,17 +1,3 @@ [options.entry_points] console_scripts = fio = fiona.fio.main:main_group -fiona.fio_commands = - bounds = fiona.fio.bounds:bounds - calc = fiona.fio.calc:calc - cat = fiona.fio.cat:cat - collect = fiona.fio.collect:collect - distrib = fiona.fio.distrib:distrib - dump = fiona.fio.dump:dump - env = fiona.fio.env:env - filter = fiona.fio.filter:filter - info = fiona.fio.info:info - insp = fiona.fio.insp:insp - load = fiona.fio.load:load - ls = fiona.fio.ls:ls - rm = fiona.fio.rm:rm diff --git a/tests/data/rmnp.geojson b/tests/data/rmnp.geojson new file mode 100644 index 000000000..41a701670 --- /dev/null +++ b/tests/data/rmnp.geojson @@ -0,0 +1 @@ +{"features": [{"geometry": {"coordinates": [[[[-105.82993236599998, 40.23429787900005], [-105.82532010999995, 40.234090120000076], [-105.80130186799994, 40.238022355000055], [-105.79971981686558, 40.23834806093726], [-105.80615125399999, 40.248891729000036], [-105.80604781499994, 40.25259529400006], [-105.80323751899999, 40.256223652000074], [-105.84344629799995, 40.256590533000065], [-105.85209887999997, 40.27187296100004], [-105.85215931999994, 40.27205192000008], [-105.86034979199997, 40.30162991100008], [-105.86041877199995, 40.30189618600008], [-105.86043595199999, 40.302194443000076], [-105.86044234999997, 40.30242113800006], [-105.85872471399995, 40.305752203000054], [-105.85865781799998, 40.30587968700007], [-105.85858570499994, 40.30600321500003], [-105.85779012199998, 40.306659301000025], [-105.87150570599994, 40.31460928800004], [-105.87096989399998, 40.34002330200008], [-105.87089412899996, 40.34472017200005], [-105.86851234235445, 40.36866639070081], [-105.87465775499999, 40.36700144500003], [-105.89323599499994, 40.37071941500005], [-105.91207991899995, 40.37242368300008], [-105.91272218299997, 40.37249360700008], [-105.91287933999996, 40.37253276000007], [-105.91360267899995, 40.372845693000045], [-105.91372432699995, 40.37293256000004], [-105.90505138299994, 40.39897570900007], [-105.90346006599998, 40.431925275000026], [-105.90344864899998, 40.43216639100007], [-105.89927318399998, 40.465591937000056], [-105.89922084599999, 40.465786036000054], [-105.89880951399999, 40.466564007000045], [-105.89822195199997, 40.46764514500006], [-105.89732067099999, 40.46917015400004], [-105.89188595999997, 40.476450914000054], [-105.89177666799998, 40.47657788200007], [-105.89159512399999, 40.47670540800004], [-105.88663187599997, 40.47736520700005], [-105.85499564999998, 40.486197454000035], [-105.85496571099998, 40.48620149300007], [-105.85489999899994, 40.486210355000026], [-105.85474552299996, 40.48623118900008], [-105.83766572599995, 40.48284897600007], [-105.83753977299995, 40.48282070100004], [-105.82538607593936, 40.477886708345466], [-105.81588885599996, 40.484865049000064], [-105.80526615399998, 40.49096921100005], [-105.80478208599999, 40.49119330100007], [-105.79614713399997, 40.49419001000007], [-105.76340551899995, 40.51327554900007], [-105.75138246899996, 40.52241727900008], [-105.75068548999997, 40.52287225300006], [-105.75041200099997, 40.52302718000004], [-105.74898221500828, 40.52333502440806], [-105.75294155799997, 40.53132139600007], [-105.75297840399998, 40.531415749000075], [-105.75298352099998, 40.531510308000065], [-105.75304362899999, 40.53500084500007], [-105.75046013199994, 40.538454529000035], [-105.75026447499994, 40.53870009700006], [-105.75022003999999, 40.538730858000065], [-105.73601753799994, 40.54698421300003], [-105.73543265699999, 40.54729423800006], [-105.73490789599998, 40.54750027600005], [-105.73417386699998, 40.54760404000007], [-105.72224790899998, 40.54923323300005], [-105.70060787699998, 40.553433977000054], [-105.69951366899994, 40.553625289000024], [-105.69846461499998, 40.55379379500005], [-105.69740136799999, 40.55376868700006], [-105.69590444199997, 40.553529971000046], [-105.68403573899997, 40.55086216500007], [-105.68354186699997, 40.55067590200008], [-105.68264478099996, 40.55025328600004], [-105.68224110499995, 40.550043963000064], [-105.68188181999994, 40.54981710000004], [-105.68171810199999, 40.549713723000025], [-105.68137426799996, 40.54947251200008], [-105.67910352499996, 40.54780573200003], [-105.67111994099997, 40.539478649000046], [-105.67089639699998, 40.53922769900004], [-105.67076231199997, 40.53906631800004], [-105.66685877099997, 40.533507919000044], [-105.66666479199995, 40.53322075400007], [-105.66460226864893, 40.52988730500347], [-105.64525353599998, 40.532981953000046], [-105.62241943799995, 40.53981883100005], [-105.62153453099995, 40.540057825000076], [-105.62042534799997, 40.54030702600005], [-105.62003581299996, 40.54036317100008], [-105.61955640099995, 40.54039727300005], [-105.61904733599994, 40.54033694000003], [-105.60211979999997, 40.53829128900003], [-105.60162589399994, 40.538230800000065], [-105.60016054299996, 40.53781500300005], [-105.59972731399995, 40.537565004000044], [-105.59929443299995, 40.53722941700005], [-105.59910215099995, 40.537014759000044], [-105.59906149099999, 40.53696936700004], [-105.59893700599997, 40.53683037400003], [-105.57756808599999, 40.51851144700004], [-105.57734419699995, 40.518386441000075], [-105.57649357899999, 40.51787717900004], [-105.57619528499998, 40.51763542600003], [-105.57028856699998, 40.513556933000075], [-105.57010850299997, 40.51345394200007], [-105.56840699299994, 40.51241728800005], [-105.53543969099997, 40.51327744200006], [-105.53504912299996, 40.513517988000046], [-105.53257093699995, 40.51499789300004], [-105.53209022499999, 40.51527037500006], [-105.53189886999996, 40.515325492000045], [-105.53156575899999, 40.515421436000054], [-105.52984182799997, 40.515825747000065], [-105.51756163999994, 40.51918768100006], [-105.51739464799999, 40.51920669100008], [-105.51700292299995, 40.51227282200006], [-105.51681120999996, 40.508877280000036], [-105.51661963499998, 40.50548173400006], [-105.51642772099996, 40.50208528600007], [-105.51623569999998, 40.498688833000074], [-105.51616764099998, 40.49747991800007], [-105.51615748699999, 40.497299560000044], [-105.51614708899996, 40.497114870000075], [-105.51604451099996, 40.49529272700005], [-105.51585310799999, 40.49189670900006], [-105.51566160099998, 40.488499699000045], [-105.51546987699999, 40.485102598000026], [-105.51527841899997, 40.48170648400003], [-105.51508686399995, 40.47831036700006], [-105.51489496899995, 40.47491325900006], [-105.51470297699996, 40.471516149000024], [-105.51451112999996, 40.46811993700004], [-105.51273101209253, 40.43773005864719], [-105.51268141899999, 40.43771853100003], [-105.49359377399998, 40.433632946000046], [-105.49384450399998, 40.42265419300003], [-105.49445713499995, 40.40834343900008], [-105.49462260299998, 40.404728686000055], [-105.51175716599994, 40.392979152000066], [-105.52357935999999, 40.38692618600004], [-105.53802058899998, 40.38253243100007], [-105.55202337099996, 40.38337468200007], [-105.56086616230968, 40.38911621698159], [-105.55199686599997, 40.37976664800004], [-105.54432240599999, 40.36429782000005], [-105.54390615499995, 40.36299049200005], [-105.54997263999996, 40.36027357100005], [-105.57119972699996, 40.348654846000045], [-105.58199381599997, 40.32891757900006], [-105.58199446199995, 40.32889092400006], [-105.58205034399998, 40.326759225000046], [-105.57230443299994, 40.32527691300004], [-105.56764467899995, 40.32528722600006], [-105.56268212499998, 40.325360086000046], [-105.55805262399997, 40.325314278000064], [-105.55330571199994, 40.32532461400007], [-105.54849582199995, 40.325334963000046], [-105.54383945699999, 40.318265127000075], [-105.53509543499996, 40.31407892900006], [-105.53403439699997, 40.30697408900005], [-105.53362917399994, 40.29996820400004], [-105.53743801499996, 40.30014405900005], [-105.53829550999995, 40.300192554000034], [-105.53857702999994, 40.300208473000055], [-105.54211996799995, 40.30046056600003], [-105.55233481799996, 40.29336605000003], [-105.55659629599995, 40.284294583000076], [-105.55658333899999, 40.28274042900006], [-105.55638339299998, 40.27564530300003], [-105.54206666299996, 40.26143052400005], [-105.54175131899996, 40.24686076900008], [-105.54177471199995, 40.24486010000004], [-105.53265796099998, 40.225078703000065], [-105.53268332499994, 40.21936352200004], [-105.53370137907686, 40.21890372610939], [-105.53264866599994, 40.216782204000026], [-105.53282174599997, 40.21053215400008], [-105.54226766999994, 40.19058178800003], [-105.57130617599995, 40.165553240000065], [-105.57154539499999, 40.16537187000006], [-105.57194869999995, 40.16510860400007], [-105.57850588199994, 40.16121546900007], [-105.57889369499998, 40.16107838800008], [-105.57917676799997, 40.16100940200005], [-105.57944527699999, 40.16095400200004], [-105.59509419199998, 40.15830693100003], [-105.59555592999999, 40.15823698500003], [-105.59665870999999, 40.158082652000076], [-105.59761194299995, 40.15807323100006], [-105.60285317299997, 40.15809561300006], [-105.60294910099998, 40.15811294400004], [-105.60370143199998, 40.158248859000025], [-105.62170529799994, 40.16156817700005], [-105.65465245599995, 40.16767968000005], [-105.66688537484622, 40.17000001598165], [-105.66686555399997, 40.16991096000004], [-105.67000857399995, 40.16730004600004], [-105.67016297799995, 40.167195606000064], [-105.67141738899994, 40.16646097700004], [-105.67154528999998, 40.16646576100004], [-105.69696639199998, 40.162049558000035], [-105.69735396799996, 40.16194812100008], [-105.69783084399995, 40.161823620000064], [-105.69809911899995, 40.16179047000003], [-105.69856090999997, 40.161769666000055], [-105.69885836099996, 40.161790394000036], [-105.70324941299998, 40.16243050500003], [-105.70347260999995, 40.16247419700005], [-105.70375553299999, 40.16254455300003], [-105.70390424299995, 40.16258869600006], [-105.70415694699994, 40.16271779600004], [-105.70531632599995, 40.16340446600003], [-105.71618835799995, 40.16834696700005], [-105.71659006199997, 40.16850665100003], [-105.71685758599995, 40.16864464100007], [-105.71702112799994, 40.16880579800005], [-105.71722881499994, 40.169011728000044], [-105.71743665699995, 40.169267209000054], [-105.71771866899996, 40.16963034200006], [-105.74348628399997, 40.168688426000074], [-105.74383073499996, 40.16800149800008], [-105.74411460099998, 40.167648307000036], [-105.74439820099997, 40.16737169700008], [-105.74468148799997, 40.16723473400003], [-105.75252233799995, 40.16418840700004], [-105.75953475599994, 40.16484084700005], [-105.75999619699996, 40.16488737000003], [-105.76080028999996, 40.16499020200007], [-105.76197617699995, 40.16516265800004], [-105.76516258499998, 40.16561463800008], [-105.76550512899996, 40.16569345800008], [-105.77107118899994, 40.167417862000036], [-105.79663688699998, 40.17037553900008], [-105.79802255699997, 40.16995606900008], [-105.79875292299994, 40.16973931200005], [-105.79921441199997, 40.169691080000064], [-105.81725341699996, 40.16740740300003], [-105.81999581999997, 40.16951501300008], [-105.82021443499997, 40.17101724500003], [-105.82022265399996, 40.171120795000036], [-105.82022362999999, 40.17120187300003], [-105.82600834399994, 40.19147235300005], [-105.83044446399998, 40.19612547200006], [-105.83056168099995, 40.19622373400006], [-105.83415290399995, 40.19890521700006], [-105.83590610799996, 40.20007279200007], [-105.83622886799998, 40.20031371300007], [-105.83724425399998, 40.201819940000064], [-105.83997704999996, 40.20730591200004], [-105.84082691599997, 40.20925216400008], [-105.84235351099994, 40.214368886000045], [-105.84235770799995, 40.21444093000008], [-105.84207465799994, 40.21575384300007], [-105.83545929499996, 40.234072627000046], [-105.83526359799998, 40.23451919900003], [-105.82993236599998, 40.23429787900005]], [[-105.58371488559169, 40.40142222697275], [-105.57356491679676, 40.39736138986373], [-105.57949503399999, 40.40121175500008], [-105.58371488559169, 40.40142222697275]]], [[[-105.52387338099999, 40.29587849500007], [-105.51833431299997, 40.289190561000055], [-105.51805444999997, 40.289083703000074], [-105.51783916499994, 40.28900808800006], [-105.51771470599999, 40.28891414100008], [-105.51770271699996, 40.28890509200005], [-105.51753073399999, 40.28877072200004], [-105.51610842599996, 40.28685359100007], [-105.51610838199997, 40.28685353000003], [-105.51606038099999, 40.28678602700006], [-105.51593599699999, 40.286611109000035], [-105.51585722899995, 40.286453797000036], [-105.51584008899994, 40.28641720100006], [-105.51577090499995, 40.28626949100004], [-105.51573753999998, 40.28616337100004], [-105.51568463099994, 40.285995092000064], [-105.51342582599995, 40.27889229300007], [-105.51204804099996, 40.27581721400003], [-105.51178041799994, 40.27498503600003], [-105.51168516799999, 40.27464401300006], [-105.51191604799999, 40.274650560000055], [-105.51389726599996, 40.27470672100003], [-105.51864548699996, 40.27484292500003], [-105.52323037599996, 40.26789955000004], [-105.52782832999998, 40.26816456300003], [-105.52793138299995, 40.27156944300003], [-105.52804855399995, 40.27511655600006], [-105.52817409, 40.27859085100005], [-105.52823393999995, 40.28210154200008], [-105.52831281699997, 40.28556538600003], [-105.53304071299999, 40.289422278000075], [-105.53352711399998, 40.297435663000044], [-105.53362917399994, 40.29996820400004], [-105.52898800099996, 40.29965667500005], [-105.52419156499997, 40.29937584700008], [-105.52417450299998, 40.298356116000036], [-105.52387338099999, 40.29587849500007]]]], "type": "MultiPolygon"}, "id": "0:0", "properties": {}, "type": "Feature"}], "type": "FeatureCollection"} diff --git a/tests/data/trio.geojson b/tests/data/trio.geojson new file mode 100644 index 000000000..9e7b71b28 --- /dev/null +++ b/tests/data/trio.geojson @@ -0,0 +1,76 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "name": "Le château d'eau" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 3.869011402130127, + 43.611401128587104 + ] + } + }, + { + "type": "Feature", + "properties": { + "aqueduct": "yes" + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 3.8645052909851074, + 43.61172738574996 + ], + [ + 3.868989944458008, + 43.61140889663537 + ] + ] + } + }, + { + "type": "Feature", + "properties": {"name": "promenade du Peyrou", "architect": "Giral"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 3.8684856891632085, + 43.61205364114294 + ], + [ + 3.8683247566223145, + 43.6108340583545 + ], + [ + 3.8685393333435054, + 43.610748608951816 + ], + [ + 3.871554136276245, + 43.610577709782206 + ], + [ + 3.871725797653198, + 43.61063208684338 + ], + [ + 3.8719189167022705, + 43.61183613774427 + ], + [ + 3.8684856891632085, + 43.61205364114294 + ] + ] + ] + } + } + ] +} diff --git a/tests/data/trio.seq b/tests/data/trio.seq new file mode 100644 index 000000000..a24422b1b --- /dev/null +++ b/tests/data/trio.seq @@ -0,0 +1,3 @@ +{"geometry": {"coordinates": [3.869011402130127, 43.611401128587104], "type": "Point"}, "id": "0", "properties": {"name": "Le ch\u00e2teau d'eau"}, "type": "Feature"} +{"geometry": {"coordinates": [[3.8645052909851074, 43.61172738574996], [3.868989944458008, 43.61140889663537]], "type": "LineString"}, "id": "1", "properties": {"aqueduct": "yes"}, "type": "Feature"} +{"geometry": {"coordinates": [[[3.8684856891632085, 43.61205364114294], [3.8683247566223145, 43.6108340583545], [3.8685393333435054, 43.610748608951816], [3.871554136276245, 43.610577709782206], [3.871725797653198, 43.61063208684338], [3.8719189167022705, 43.61183613774427], [3.8684856891632085, 43.61205364114294]]], "type": "Polygon"}, "id": "2", "properties": {"architect": "Giral", "name": "promenade du Peyrou"}, "type": "Feature"} diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 000000000..47255e9ac --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,330 @@ +# Python module tests + +import json + +import pytest # type: ignore +import shapely # type: ignore +from shapely.geometry import LineString, MultiPoint, Point, mapping, shape # type: ignore + +from fiona.errors import ReduceError +from fiona.features import ( # type: ignore + map_feature, + reduce_features, + vertex_count, + area, + buffer, + collect, + distance, + dump, + identity, + length, + unary_projectable_property_wrapper, + unary_projectable_constructive_wrapper, + binary_projectable_property_wrapper, +) + + +def test_modulate_simple(): + """Set a feature's geometry.""" + # map_feature() is a generator. list() materializes the values. + feat = list(map_feature("Point 0 0", {"type": "Feature"})) + assert len(feat) == 1 + + feat = feat[0] + assert "Point" == feat["type"] + assert (0.0, 0.0) == feat["coordinates"] + + +def test_modulate_complex(): + """Exercise a fairly complicated pipeline.""" + bufkwd = "resolution" if shapely.__version__.startswith("1") else "quad_segs" + + with open("tests/data/trio.geojson") as src: + collection = json.loads(src.read()) + + feat = collection["features"][0] + results = list( + map_feature( + f"simplify (buffer g (* 0.1 2) :projected false :{bufkwd} (- 4 3)) 0.001 :projected false :preserve_topology false", + feat, + ) + ) + assert 1 == len(results) + + geom = results[0] + assert geom["type"] == "Polygon" + assert len(geom["coordinates"][0]) == 5 + + +@pytest.mark.parametrize( + "obj, count", + [ + (Point(0, 0), 1), + (MultiPoint([(0, 0), (1, 1)]), 2), + (Point(0, 0).buffer(10.0).difference(Point(0, 0).buffer(1.0)), 130), + ], +) +def test_vertex_count(obj, count): + """Check vertex counting correctness.""" + assert count == vertex_count(obj) + + +@pytest.mark.parametrize( + "obj, count", + [ + (Point(0, 0), 1), + (MultiPoint([(0, 0), (1, 1)]), 2), + (Point(0, 0).buffer(10.0).difference(Point(0, 0).buffer(1.0)), 130), + ], +) +def test_calculate_vertex_count(obj, count): + """Confirm vertex counting is in func_map.""" + feat = {"type": "Feature", "properties": {}, "geometry": mapping(obj)} + assert count == list(map_feature("vertex_count g", feat))[0] + + +def test_calculate_builtin(): + """Confirm builtin function evaluation.""" + assert 42 == list(map_feature("int '42'", None))[0] + + +def test_calculate_feature_attr(): + """Confirm feature attr evaluation.""" + assert "LOLWUT" == list(map_feature("upper f", "lolwut"))[0] + + +def test_calculate_point(): + """Confirm feature attr evaluation.""" + result = list(map_feature("Point 0 0", None))[0] + assert "Point" == result["type"] + + +def test_calculate_points(): + """Confirm feature attr evaluation.""" + result = list(map_feature("list (Point 0 0) (buffer (Point 1 1) 1)", None)) + assert 2 == len(result) + assert "Point" == result[0]["type"] + assert "Polygon" == result[1]["type"] + + +def test_reduce_len(): + """Reduce can count the number of input features.""" + with open("tests/data/trio.seq") as seq: + data = [json.loads(line) for line in seq.readlines()] + + # reduce() is a generator. list() materializes the values. + assert 3 == list(reduce_features("len c", data))[0] + + +def test_reduce_union(): + """Reduce yields one feature by default.""" + with open("tests/data/trio.seq") as seq: + data = [json.loads(line) for line in seq.readlines()] + + # reduce() is a generator. list() materializes the values. + result = list(reduce_features("unary_union c", data)) + assert len(result) == 1 + + val = result[0] + assert "GeometryCollection" == val["type"] + assert 2 == len(val["geometries"]) + + +def test_reduce_union_area(): + """Reduce can yield total area using raw output.""" + with open("tests/data/trio.seq") as seq: + data = [json.loads(line) for line in seq.readlines()] + + # reduce() is a generator. + result = list(reduce_features("area (unary_union c)", data)) + assert len(result) == 1 + + val = result[0] + assert isinstance(val, float) + assert 3e4 < val < 4e4 + + +def test_reduce_union_geom_type(): + """Reduce and print geom_type using raw output.""" + with open("tests/data/trio.seq") as seq: + data = [json.loads(line) for line in seq.readlines()] + + # reduce() is a generator. + result = list(reduce_features("geom_type (unary_union c)", data)) + assert len(result) == 1 + assert "GeometryCollection" == result[0] + + +def test_reduce_error(): + """Raise ReduceError when expression doesn't reduce.""" + with open("tests/data/trio.seq") as seq: + data = [json.loads(line) for line in seq.readlines()] + + with pytest.raises(ReduceError): + list(reduce_features("(identity c)", data)) + + +@pytest.mark.parametrize( + "obj, count", + [ + (MultiPoint([(0, 0), (1, 1)]), 2), + ], +) +def test_dump_eval(obj, count): + feature = {"type": "Feature", "properties": {}, "geometry": mapping(obj)} + result = map_feature("identity g", feature, dump_parts=True) + assert len(list(result)) == count + + +def test_collect(): + """Collect two points.""" + geom = collect((Point(0, 0), Point(1, 1))) + assert geom.geom_type == "GeometryCollection" + + +def test_dump(): + """Dump a point.""" + geoms = list(dump(Point(0, 0))) + assert len(geoms) == 1 + assert geoms[0].geom_type == "Point" + + +def test_dump_multi(): + """Dump two points.""" + geoms = list(dump(MultiPoint([(0, 0), (1, 1)]))) + assert len(geoms) == 2 + assert all(g.geom_type == "Point" for g in geoms) + + +def test_identity(): + """Check identity.""" + geom = Point(1.1, 2.2) + assert geom == identity(geom) + + +def test_area(): + """Check projected area of RMNP against QGIS.""" + with open("tests/data/rmnp.geojson", "rb") as f: + collection = json.load(f) + + geom = shape(collection["features"][0]["geometry"]) + + # QGIS uses a geodesic area computation and WGS84 ellipsoid. + qgis_ellipsoidal_area = 1117.433937055 # kilometer squared + + # We expect no more than a 0.0001 km^2 difference. That's .00001%. + assert round(qgis_ellipsoidal_area, 4) == round(area(geom) / 1e6, 4) + + +@pytest.mark.parametrize( + ["kwargs", "exp_distance"], + [({}, 9648.6280), ({"projected": True}, 9648.6280), ({"projected": False}, 0.1)], +) +def test_distance(kwargs, exp_distance): + """Distance measured properly.""" + assert round(exp_distance, 4) == round( + distance(Point(0, 0), Point(0.1, 0), **kwargs), 4 + ) + + +@pytest.mark.parametrize( + ["kwargs", "distance", "exp_area"], + [ + ({}, 1.0e4, 312e6), + ({"projected": True}, 10000.0, 312e6), + ({"projected": False}, 0.1, 0.0312), + ], +) +def test_buffer(kwargs, distance, exp_area): + """Check area of a point buffered by 10km using 8 quadrant segments, should be ~312 km2.""" + # float(f"{x:.3g}") is used to round x to 3 significant figures. + assert exp_area == float( + f"{area(buffer(Point(0, 0), distance, **kwargs), **kwargs):.3g}" + ) + + +@pytest.mark.parametrize( + ["kwargs", "exp_length"], + [({}, 9648.6280), ({"projected": True}, 9648.6280), ({"projected": False}, 0.1)], +) +def test_length(kwargs, exp_length): + """Length measured properly.""" + assert round(exp_length, 4) == round( + length(LineString([(0, 0), (0.1, 0)]), **kwargs), 4 + ) + + +@pytest.mark.parametrize( + ["in_xy", "exp_xy", "kwargs"], + [ + ((0.1, 0.0), (9648.628, 0.0), {}), + ((0.1, 0.0), (9648.628, 0.0), {"projected": True}), + ((0.1, 0.0), (0.1, 0.0), {"projected": False}), + ], +) +def test_unary_property_wrapper(in_xy, exp_xy, kwargs): + """Correctly wraps a function like shapely.area.""" + + def func(geom, *args, **kwargs): + """Echoes its input.""" + return geom, args, kwargs + + wrapper = unary_projectable_property_wrapper(func) + assert wrapper.__doc__ == "Echoes its input." + assert wrapper.__name__ == "func" + g, *rest = wrapper(Point(*in_xy), "hello", this=True, **kwargs) + assert rest == [("hello",), {"this": True}] + assert round(g.x, 4) == round(exp_xy[0], 4) + assert round(g.y, 4) == round(exp_xy[1], 4) + + +@pytest.mark.parametrize( + ["in_xy", "exp_xy", "kwargs"], + [ + ((0.1, 0.0), (9648.628, 0.0), {}), + ((0.1, 0.0), (9648.628, 0.0), {"projected": True}), + ((0.1, 0.0), (0.1, 0.0), {"projected": False}), + ], +) +def test_unary_projectable_constructive_wrapper(in_xy, exp_xy, kwargs): + """Correctly wraps a function like shapely.buffer.""" + + def func(geom, required, this=False): + """Echoes its input geom.""" + assert round(geom.x, 4) == round(exp_xy[0], 4) + assert round(geom.y, 4) == round(exp_xy[1], 4) + assert this is True + return geom + + wrapper = unary_projectable_constructive_wrapper(func) + assert wrapper.__doc__ == "Echoes its input geom." + assert wrapper.__name__ == "func" + g = wrapper(Point(*in_xy), "hello", this=True, **kwargs) + assert round(g.x, 4) == round(in_xy[0], 4) + assert round(g.y, 4) == round(in_xy[1], 4) + + +@pytest.mark.parametrize( + ["in_xy", "exp_xy", "kwargs"], + [ + ((0.1, 0.0), (9648.628, 0.0), {}), + ((0.1, 0.0), (9648.628, 0.0), {"projected": True}), + ((0.1, 0.0), (0.1, 0.0), {"projected": False}), + ], +) +def test_binary_projectable_property_wrapper(in_xy, exp_xy, kwargs): + """Correctly wraps a function like shapely.distance.""" + + def func(geom1, geom2, *args, **kwargs): + """Echoes its inputs.""" + return geom1, geom2, args, kwargs + + wrapper = binary_projectable_property_wrapper(func) + assert wrapper.__doc__ == "Echoes its inputs." + assert wrapper.__name__ == "func" + g1, g2, *rest = wrapper(Point(*in_xy), Point(*in_xy), "hello", this=True, **kwargs) + assert rest == [("hello",), {"this": True}] + assert round(g1.x, 4) == round(exp_xy[0], 4) + assert round(g1.y, 4) == round(exp_xy[1], 4) + assert round(g2.x, 4) == round(exp_xy[0], 4) + assert round(g2.y, 4) == round(exp_xy[1], 4) diff --git a/tests/test_fio_features.py b/tests/test_fio_features.py new file mode 100644 index 000000000..d66318cc6 --- /dev/null +++ b/tests/test_fio_features.py @@ -0,0 +1,112 @@ +# CLI tests + +from click.testing import CliRunner + +from fiona.fio.main import main_group # type: ignore +import pytest # type: ignore + + +def test_map_count(): + """fio-map prints correct number of results.""" + with open("tests/data/trio.seq") as seq: + data = seq.read() + + runner = CliRunner() + result = runner.invoke( + main_group, + ["map", "centroid (buffer g 1.0)"], + input=data, + ) + + assert result.exit_code == 0 + assert result.output.count('"type": "Point"') == 3 + + +@pytest.mark.parametrize("raw_opt", ["--raw", "-r"]) +def test_reduce_area(raw_opt): + """Reduce features to their (raw) area.""" + with open("tests/data/trio.seq") as seq: + data = seq.read() + + runner = CliRunner() + result = runner.invoke( + main_group, + ["reduce", raw_opt, "area (unary_union c) :projected false"], + input=data, + ) + assert result.exit_code == 0 + assert 0 < float(result.output) < 1e-5 + + +def test_reduce_union(): + """Reduce features to one single feature.""" + with open("tests/data/trio.seq") as seq: + data = seq.read() + + # Define our reduce command using a mkdocs snippet. + arg = """ + --8<-- [start:reduce] + unary_union c + --8<-- [end:reduce] + """.splitlines()[ + 2 + ].strip() + + runner = CliRunner() + result = runner.invoke(main_group, ["reduce", arg], input=data) + assert result.exit_code == 0 + assert result.output.count('"type": "Polygon"') == 1 + assert result.output.count('"type": "LineString"') == 1 + assert result.output.count('"type": "GeometryCollection"') == 1 + + +def test_reduce_union_zip_properties(): + """Reduce features to one single feature, zipping properties.""" + with open("tests/data/trio.seq") as seq: + data = seq.read() + + runner = CliRunner() + result = runner.invoke( + main_group, ["reduce", "--zip-properties", "unary_union c"], input=data + ) + assert result.exit_code == 0 + assert result.output.count('"type": "Polygon"') == 1 + assert result.output.count('"type": "LineString"') == 1 + assert result.output.count('"type": "GeometryCollection"') == 1 + assert ( + """"name": ["Le ch\\u00e2teau d\'eau", "promenade du Peyrou"]""" + in result.output + ) + + +def test_filter(): + """Filter features by distance.""" + with open("tests/data/trio.seq") as seq: + data = seq.read() + + # Define our reduce command using a mkdocs snippet. + arg = """ + --8<-- [start:filter] + < (distance g (Point 4 43)) 62.5E3 + --8<-- [end:filter] + """.splitlines()[ + 2 + ].strip() + + runner = CliRunner() + result = runner.invoke( + main_group, + ["filter", arg], + input=data, + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output.count('"type": "Polygon"') == 1 + + +@pytest.mark.parametrize("opts", [["--no-input", "--raw"], ["-rn"]]) +def test_map_no_input(opts): + runner = CliRunner() + result = runner.invoke(main_group, ["map"] + opts + ["(Point 4 43)"]) + assert result.exit_code == 0 + assert result.output.count('"type": "Point"') == 1 diff --git a/tests/test_fio_filter.py b/tests/test_fio_filter.py index 4b47c0e84..5714ec5e5 100644 --- a/tests/test_fio_filter.py +++ b/tests/test_fio_filter.py @@ -1,28 +1,33 @@ -"""Tests for `$ fio filter`.""" +"""Tests for the legacy fio-filter.""" + +import pytest from fiona.fio.main import main_group def test_fail(runner): - result = runner.invoke(main_group, ['filter', - "f.properties.test > 5" - ], "{'type': 'no_properties'}") - assert result.exit_code == 1 + with pytest.warns(FutureWarning): + result = runner.invoke(main_group, ['filter', + "f.properties.test > 5" + ], "{'type': 'no_properties'}") + assert result.exit_code == 1 def test_seq(feature_seq, runner): - - result = runner.invoke(main_group, ['filter', - "f.properties.AREA > 0.01"], feature_seq, catch_exceptions=False) - assert result.exit_code == 0 - assert result.output.count('Feature') == 2 - - result = runner.invoke(main_group, ['filter', - "f.properties.AREA > 0.015"], feature_seq) - assert result.exit_code == 0 - assert result.output.count('Feature') == 1 - - result = runner.invoke(main_group, ['filter', - "f.properties.AREA > 0.02"], feature_seq) - assert result.exit_code == 0 - assert result.output.count('Feature') == 0 + with pytest.warns(FutureWarning): + result = runner.invoke(main_group, ['filter', + "f.properties.AREA > 0.01"], feature_seq, catch_exceptions=False) + assert result.exit_code == 0 + assert result.output.count('Feature') == 2 + + with pytest.warns(FutureWarning): + result = runner.invoke(main_group, ['filter', + "f.properties.AREA > 0.015"], feature_seq) + assert result.exit_code == 0 + assert result.output.count('Feature') == 1 + + with pytest.warns(FutureWarning): + result = runner.invoke(main_group, ['filter', + "f.properties.AREA > 0.02"], feature_seq) + assert result.exit_code == 0 + assert result.output.count('Feature') == 0 diff --git a/tests/test_snuggs.py b/tests/test_snuggs.py new file mode 100644 index 000000000..082b7e765 --- /dev/null +++ b/tests/test_snuggs.py @@ -0,0 +1,25 @@ +# Python module tests + +"""Tests of the snuggs module.""" + +import pytest # type: ignore + +from fiona._vendor import snuggs + + +@pytest.mark.parametrize("arg", ["''", "null", "false", 0]) +def test_truth_false(arg): + """Expression is not true.""" + assert not snuggs.eval(f"(truth {arg})") + + +@pytest.mark.parametrize("arg", ["'hi'", "true", 1]) +def test_truth(arg): + """Expression is true.""" + assert snuggs.eval(f"(truth {arg})") + + +@pytest.mark.parametrize("arg", ["''", "null", "false", 0]) +def test_not(arg): + """Expression is true.""" + assert snuggs.eval(f"(not {arg})")