From 9c16722d041124eab8342274b648c1bfd4849870 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sat, 4 Jan 2020 16:10:00 +0100 Subject: [PATCH] New flex backend This introduces a new "flex" backend which allows much more flexibility in choosing the database format and the transformation from OSM data to the database format. The user defines all this in a Lua script. --- README.md | 6 + docs/flex.md | 282 ++++++ docs/osm2pgsql.1 | 3 +- flex-config/README.md | 48 + flex-config/compatible.lua | 577 +++++++++++ flex-config/data-types.lua | 120 +++ flex-config/default-config.lua | 266 +++++ flex-config/geometries.lua | 178 ++++ flex-config/places.lua | 52 + flex-config/route-relations.lua | 95 ++ flex-config/simple.lua | 154 +++ flex-config/unitable.lua | 71 ++ src/CMakeLists.txt | 14 +- src/db-copy.cpp | 44 + src/db-copy.hpp | 42 + src/expire-tiles.cpp | 19 + src/expire-tiles.hpp | 2 + src/flex-table-column.cpp | 160 +++ src/flex-table-column.hpp | 118 +++ src/flex-table.cpp | 330 ++++++ src/flex-table.hpp | 222 ++++ src/geom-transform.cpp | 166 +++ src/geom-transform.hpp | 107 ++ src/init.lua | 94 ++ src/lua-init.cpp.in | 6 + src/lua-init.hpp | 6 + src/lua-utils.cpp | 116 +++ src/lua-utils.hpp | 49 + src/osmdata.cpp | 4 + src/output-flex.cpp | 1344 +++++++++++++++++++++++++ src/output-flex.hpp | 169 ++++ src/output.cpp | 19 +- src/output.hpp | 2 + src/version.cpp.in | 4 + src/version.hpp | 1 + tests/CMakeLists.txt | 8 + tests/common-options.hpp | 14 + tests/data/test_output_flex.lua | 139 +++ tests/data/test_output_flex_attr.lua | 20 + tests/data/test_output_flex_extra.lua | 86 ++ tests/data/test_output_flex_uni.lua | 115 +++ tests/test-output-flex-area.cpp | 59 ++ tests/test-output-flex-attr.cpp | 82 ++ tests/test-output-flex-extra.cpp | 194 ++++ tests/test-output-flex-tablespace.cpp | 36 + tests/test-output-flex-uni.cpp | 248 +++++ tests/test-output-flex-update.cpp | 224 +++++ tests/test-output-flex-validgeom.cpp | 25 + tests/test-output-flex.cpp | 132 +++ 49 files changed, 6268 insertions(+), 4 deletions(-) create mode 100644 docs/flex.md create mode 100644 flex-config/README.md create mode 100644 flex-config/compatible.lua create mode 100644 flex-config/data-types.lua create mode 100644 flex-config/default-config.lua create mode 100644 flex-config/geometries.lua create mode 100644 flex-config/places.lua create mode 100644 flex-config/route-relations.lua create mode 100644 flex-config/simple.lua create mode 100644 flex-config/unitable.lua create mode 100644 src/flex-table-column.cpp create mode 100644 src/flex-table-column.hpp create mode 100644 src/flex-table.cpp create mode 100644 src/flex-table.hpp create mode 100644 src/geom-transform.cpp create mode 100644 src/geom-transform.hpp create mode 100644 src/init.lua create mode 100644 src/lua-init.cpp.in create mode 100644 src/lua-init.hpp create mode 100644 src/lua-utils.cpp create mode 100644 src/lua-utils.hpp create mode 100644 src/output-flex.cpp create mode 100644 src/output-flex.hpp create mode 100644 tests/data/test_output_flex.lua create mode 100644 tests/data/test_output_flex_attr.lua create mode 100644 tests/data/test_output_flex_extra.lua create mode 100644 tests/data/test_output_flex_uni.lua create mode 100644 tests/test-output-flex-area.cpp create mode 100644 tests/test-output-flex-attr.cpp create mode 100644 tests/test-output-flex-extra.cpp create mode 100644 tests/test-output-flex-tablespace.cpp create mode 100644 tests/test-output-flex-uni.cpp create mode 100644 tests/test-output-flex-update.cpp create mode 100644 tests/test-output-flex-validgeom.cpp create mode 100644 tests/test-output-flex.cpp diff --git a/README.md b/README.md index 3191d4442..c1fafc2e2 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,12 @@ null backend for testing. For flexibility a new [multi](docs/multi.md) backend is also available which allows the configuration of custom PostgreSQL tables instead of those provided in the pgsql backend. +Also available is the new [flex](docs/flex.md) backend. It is much more +flexible than the other backends. IT IS CURRENTLY EXPERIMENTAL AND SUBJECT +TO CHANGE. The flex backend is only available if you have compiled osm2pgsql +with Lua support. More details at +https://github.com/openstreetmap/osm2pgsql/issues/1036 . + ## LuaJIT support ## To speed up Lua tag transformations, [LuaJIT](https://luajit.org/) can be optionally diff --git a/docs/flex.md b/docs/flex.md new file mode 100644 index 000000000..3db8f9170 --- /dev/null +++ b/docs/flex.md @@ -0,0 +1,282 @@ + +# The Flex Backend + +**The Flex Backend is experimental. Everything in here is subject to change.** + +The "Flex" backend, as the name suggests, allows for a more flexible +configuration that tells osm2pgsql what OSM data to store in your database and +exactly where and how. It is configured through a Lua file which + +* defines the structure of the output tables and +* defines functions to map the OSM data to the database data format + +See also the example config files in the `flex-config` directory which contain +lots of comments to get you started. + +## The Lua config file + +All configuration is done through the `osm2pgsql` object in Lua. It has the +following fields: + +* `osm2pgsql.version`: The version of osm2pgsql as a string. +* `osm2pgsql.srid`: The SRID set on the command line (with `-l|--latlong`, + `-m|--merc`, or `-E|--proj`). +* `osm2pgsql.mode`: Either `"create"` or `"append"` depending on the command + line options (`--create` or `-a|--append`). +* `osm2pgsql.stage`: Either `1` or `2` (1st/2nd stage processing of the data). + See below. + +The following functions are defined: + +* `osm2pgsql.define_node_table(name, columns)`: Define a node table. +* `osm2pgsql.define_way_table(name, columns)`: Define a way table. +* `osm2pgsql.define_relation_table(name, columns)`: Define a relation table. +* `osm2pgsql.define_area_table(name, columns)`: Define an area table. +* `osm2pgsql.define_table()`: Define a table. This is the more flexible + function behind all the other `define_*_table()` functions. It gives you + more control than the more convenient other functions. +* `osm2pgsql.mark_way(id)`: Mark the OSM way with the specified id. This way + will be processed (again) in stage 2. +* `osm2pgsql.mark_relation(id)`: Mark the OSM relation with the specified id. + This relation will be processed (again) in stage 2. + +You are expected to define one or more of the following functions: + +* `osm2pgsql.process_node()`: Called for each node. +* `osm2pgsql.process_way()`: Called for each way. +* `osm2pgsql.process_relation()`: Called for each relation. + +### Defining a table + +You have to define one or more tables where your data should end up. This +is done with the `osm2pgsql.define_table()` function or one of the slightly +more convenient functions `osm2pgsql.define_(node|way|relation|area)_table()`. + +Each table is either a *node table*, *way table*, *relation table*, or *area +table*. This means that the data for that table comes primarily from a node, +way, relation, or area, respectively. Osm2pgsql makes sure that the OSM object +id will be stored in the table so that later updates to those OSM objects (or +deletions) will be properly reflected in the tables. Area tables are special, +they can contain data derived from ways or from relations. Way ids will be +stored as is, relation ids will be stored as negative numbers. + +With the `osm2pgsql.define_table()` function you can also define tables that +* don't have any ids, but those tables will never be updated by osm2pgsql +* take *any OSM object*, in this case the type of object is stored in an + additional column. +* are in a specific PostgresSQL tablespace (set `data_tablespace`) or that + get their indexes created in a specific tablespace (set `index_tablespace`). + +If you are using the `osm2pgsql.define_(node|way|relation|area)_table()` +convenience functions, osm2pgsql will automatically create an id column named +`(node|way|relation|area)_id`, respectively. If you want more control over +the id column(s), use the `osm2pgsql.define_table()` function. + +Most tables will have a geometry column. (Currently only zero or one geometry +columns are supported.) The types of the geometry column possible depend on +the type of the input data. For node tables you are pretty much restricted +to point geometries, but there is a variety of options for relation tables +for instance. + +The supported geometry types are: +* `point`: Point geometry, usually created from nodes. +* `linestring`: Linestring geometry, usually created from ways. +* `polygon`: Polygon geometry for area tables, created from ways or relations. +* `multipoint`: Currently not used. +* `multilinestring`: Created from (possibly split up) ways or relations. +* `multipolygon`: For area tables, created from ways or relations. +* `geometry`: Any kind of geometry. Also used for area tables that should hold + both polygon and multipolygon geometries. + +A column of type `area` will be filled automatically with the area of the +geometry. This will only work for (multi)polygons. + +In addition to id and geometry columns, each table can have any number of +"normal" columns using any type supported by PostgreSQL. Some types are +specially recognized by osm2pgsql: + +* `text`: A text string. +* `boolean`: Interprets string values `"true"`, `"yes"` as `true` and all + others as `false`. Boolean and integer values will also work in the usual + way. +* `int2`, `smallint`: 16bit signed integer. Values too large to fit will be + truncated in some unspecified way. +* `int4`, `int`, `integer`: 32bit signed integer. Values too large to fit will + be truncated in some unspecified way. +* `int8`, `bigint`: 64bit signed integer. Values too large to fit will be + truncated in some unspecified way. +* `real`: A real number. +* `hstore`: Automatically filled from a Lua table with only strings as keys + and values. +* `direction`: Interprets values `"true"`, `"yes"`, and `"1"` as 1, `"-1"` as + `-1`, and everything else as `0`. Useful for `oneway` tags etc. + +Instead of the above types you can use any SQL type you want. If you do that +you have to supply the PostgreSQL string representation for that type when +adding data to such columns (or Lua nil to set the column to `NULL`). + +### Processing callbacks + +You are expected to define one or more of the following functions: + +* `osm2pgsql.process_node(object)`: Called for each node. +* `osm2pgsql.process_way(object)`: Called for each way. +* `osm2pgsql.process_relation(object)`: Called for each relation. + +They all have a single argument of type table (here called `object`) and no +return value. If you are not interested in all object types, you do not have +to supply all the functions. + +These functions are called for each new or modified OSM object in the input +file. No function is called for deleted objects, osm2pgsql will automatically +delete all data in your database tables that derived from deleted objects. +Modifications are handled as deletions followed by creation of a "new" object, +for which the functions are called. + +The parameter table (`object`) has the following fields: + +* `id`: The id of the node, way, or relation. +* `tags`: A table with all the tags of the object. +* `version`, `timestamp`, `changeset`, `uid`, and `user`: Attributes of the + OSM object. These are only available if the `-x|--extra-attributes` option + is used and the OSM input file actually contains those fields. The + `timestamp` contains the time in seconds since the epoch (midnight + 1970-01-01). +* `grab_tag(KEY)`: Return the tag value of the specified key and remove the + tag from the list of tags. (Example: `local name = object:grab_tag('name')`) + This is often used when you want to store some tags in special columns and + the rest of the tags in an hstore column. +* `get_bbox()`: Get the bounding box of the current node or way. (It doesn't + work for relations currently.) + +Ways have the following additional fields: +* `is_closed`: A boolean telling you whether the way geometry is closed, ie + the first and last node are the same. +* `nodes`: An array with the way node ids. + +Relations have the following additional field: +* `members`: An array with member tables. Each member table has the fields + `type` (values `n`, `w`, or `r`), `ref` (member id) and `role`. + +You can do anything in those processing functions to decide what to do with +this data. If you are not interested in that OSM object, simply return from the +function. If you want to add the OSM object to some table call the `add_row()` +function on that table: + +``` +-- definition of the table: +table_pois = osm2pgsql.define_node_table('pois', { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'point' }, +}) +... +function osm2pgsql.process_node(object) +... + table_pois:add_row({ + tags = object.tags, + name = object.tags.name, + geom = { create = 'point' } + }) +... +end +``` + +The `add_row()` function takes a single table parameter, that describes what to +fill into all the database columns. Any column not mentioned will be set to +`NULL`. + +The geometry column in somewhat special. You have to define a *geometry +transformation* that will be used to transform the OSM object data into +a geometry that fits into the geometry column. See the next section for +details. + +Note that you can't set the object id, this will be handled for you behind the +scenes. + +## Geometry transformations + +Currently these geometry transformations are supported: + +* `{ create = 'point'}`. Only valid for nodes, create a 'point' geometry. +* `{ create = 'line'}`. For ways or relations. Create a 'linestring' or + 'multilinestring' geometry. +* `{ create = 'area'}` For ways or relations. Create a 'polygon' or + 'multipolygon' geometry. + +Some of these transformations can have parameters: + +* The `line` transformation has an optional parameter `split_at`. If this + is set to anything other than 0, linestrings longer than this value will + be split up into parts no longer than this value. +* The `area` transformation has an optional parameter `multi`. If this is + set to `false` (the default), a multipolygon geometry will be split up into + several polygons. If this is set to `true`, the multipolygon geometry is + kept as one. It depends on this parameter whether you need a polygon + or multipolygon geometry column. + +If no geometry transformation is set, osm2pgsql will, in some cases, assume +a default transformation. These are the defaults: + +* For node tables, a `point` column gets the node location. +* For way tables, a `linestring` column gets the complete way geometry, a + `polygon` column gets the way geometry as area (if the way is closed and + the area is valid). + +## Stages + +Osm2pgsql processes the data in up to two stages. You can mark ways or +relations in stage 1 for processing in stage 2 by calling +`osm2pgsql.mark_way(id)` or `osm2pgsql.mark_relation(id)`, respectively. If you +don't mark any objects, nothing will be done in stage 2. + +You can look at `osm2pgsql.stage` to see in which stage you are. + +In stage 1 you can only look at each OSM object on its own. You can see +its id and tags (and possibly timestamp, changeset, user, etc.), but you don't +know how this OSM objects relates to other OSM objects (for instance whether a +way you are looking at is a member in a relation). If this is enough to decide +in which database table(s) and with what data an OSM object should end up in, +then you can process the OSM object in stage 1. If, on the other hand, you +need some extra information, you have to defer processing to the second stage. + +You want to do all the processing you can in stage 1, because it is faster +and there is less memory overhead. For most use cases, stage 1 is enough. If +it is not, use stage 1 to store information about OSM objects you will need +in stage 2 in some global variable. In stage 2 you can read this information +again and use it to decide where and how to store the data in the database. + +## Command line options + +Use the command line option `-O flex` or `--output=flex` to enable the flex +backend and the `-S|--style` option to set the Lua config file. + +The following command line options have a somewhat different meaning when +using the flex backend: + +* `-p|--prefix`: The table names you are setting in your Lua config files + will *not* get this prefix. You can easily add the prefix in the Lua config + yourself. +* `-S|--style`: Use this to specify the Lua config file. Without it, osm2pgsql + will not work, because it will try to read the default style file. +* `-G|--multi-geometry` is not used. Instead, set the type of the geometry + column to the type you want, ie `polygon` vs. `multipolygon`. + +The following command line options are ignored by `osm2pgsl` when using the +flex backend, because they don't make sense in that context: + +* `-k|--hstore` +* `-j|--hstore-all` +* `-z|--hstore-column` +* `--hstore-match-only` +* `--hstore-add-index` +* `-K|--keep-coastlines` (Coastline tags are not handled specially in the + flex backend.) +* `--tag-transform-script` (Set the Lua config file with the `-S|--style` + option.) +* `-G|--multi-geometry` (Use the `multi` option on the geometry transformation + instead.) +* The command line options to set the tablespace are ignored by the flex + backend, instead use the `data_tablespace` or `index_tablespace` options + when defining your table. + diff --git a/docs/osm2pgsql.1 b/docs/osm2pgsql.1 index c699bcc54..771ee9e5b 100644 --- a/docs/osm2pgsql.1 +++ b/docs/osm2pgsql.1 @@ -156,7 +156,8 @@ Specifies the output back\-end or database schema to use. Currently osm2pgsql supports \fBpgsql\fR, \fBgazetteer\fR and \fBnull\fR. \fBpgsql\fR is the default output back\-end / schema and is optimized for rendering with Mapnik. \fBgazetteer\fR is a db schema optimized for geocoding and is used by Nominatim. -The \fBmulti\fR backend allows more customization of tables. +The \fBmulti\fR backend allows more customization of tables. The experimental +\fBflex\fR backend allows more flexible configuration. \fBnull\fR does not write any output and is only useful for testing or with \-\-slim for creating slim tables. .TP diff --git a/flex-config/README.md b/flex-config/README.md new file mode 100644 index 000000000..58e0561a2 --- /dev/null +++ b/flex-config/README.md @@ -0,0 +1,48 @@ + +# Flex Backend Configuration + +**The Flex Backend is experimental. Everything in here is subject to change.** + +See the [Flex Backend Documentation](docs/flex.md) for all the details. + +## Example config files + +This directory contains example config files for the flex backend. All config +files are documented extensively with inline comments. + +If you are learning about the flex backend, read the config files in the +following order (from easiest to understand to the more complex ones): + +1. [simple.lua](simple.lua) -- Introduction to config file format +2. [geometries.lua](geometries.lua) -- Geometry column options +3. [data-types.lua](data-types.lua) -- Data types and how to handle them + +After that you can dive into more advanced topics: + +* [route-relations.lua](route-relations.lua) -- Use multi-stage processing + to bring tags from relations to member ways +* [unitable.lua](unitable.lua) -- Put all OSM data into a single table +* [places.lua](places.lua) -- Creating JSON/JSONB columns + +The "default" configuration is a full-featured but simple configuration that +is a good starting point for your own real-world configuration: + +* [default-config.lua](default-config.lua) + +The following config file tries to be more or less compatible with the old +osm2pgsql C transforms: + +* [compatible.lua](compatible.lua) + +## Dependencies + +Some of the example files use the `inspect` Lua library to show debugging +output. It is not needed for the actual functionality of the examples, so if +you don't have the library, you can remove all uses of `inspect` and the +scripts should still work. + +The library is available from [the +source](https://github.com/kikito/inspect.lua) or using +[LuaRocks](https://luarocks.org/modules/kikito/inspect). Debian/Ubuntu users +can install the `lua-inspect` package. + diff --git a/flex-config/compatible.lua b/flex-config/compatible.lua new file mode 100644 index 000000000..077209442 --- /dev/null +++ b/flex-config/compatible.lua @@ -0,0 +1,577 @@ + +-- This configuration for the flex backend tries to be compatible with the +-- original pgsql backend. + +-- Objects with any of the following keys will be treated as polygon +local polygon_keys = { + 'aeroway', + 'amenity', + 'building', + 'harbour', + 'historic', + 'landuse', + 'leisure', + 'man_made', + 'military', + 'natural', + 'office', + 'place', + 'power', + 'public_transport', + 'shop', + 'sport', + 'tourism', + 'water', + 'waterway', + 'wetland' +} + +-- Objects without any of the following keys will be deleted +local generic_keys = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'area', + 'barrier', + 'bicycle', + 'boundary', + 'brand', + 'bridge', + 'building', + 'capital', + 'construction', + 'covered', + 'culvert', + 'cutting', + 'denomination', + 'disused', + 'ele', + 'embarkment', + 'foot', + 'generation:source', + 'harbour', + 'highway', + 'historic', + 'hours', + 'intermittent', + 'junction', + 'landuse', + 'layer', + 'leisure', + 'lock', + 'man_made', + 'military', + 'motor_car', + 'name', + 'natural', + 'office', + 'oneway', + 'operator', + 'place', + 'population', + 'power', + 'power_source', + 'public_transport', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'sport', + 'surface', + 'toll', + 'tourism', + 'tower:type', + 'tracktype', + 'tunnel', + 'type', + 'water', + 'waterway', + 'wetland', + 'width', + 'wood' +} + +-- The following keys will be deleted +local delete_tags = { + 'FIXME', + 'note', + 'source', + 'way', + 'way_area', + 'z_order', +} + +-- Array used to specify z_order per key/value combination. +-- Each element has the form {key, value, z_order, is_road}. +-- If is_road=1, the object will be added to planet_osm_roads. +local zordering_tags = {{ 'railway', nil, 5, 1}, { 'boundary', 'administrative', 0, 1}, + { 'bridge', 'yes', 10, 0 }, { 'bridge', 'true', 10, 0 }, { 'bridge', 1, 10, 0 }, + { 'tunnel', 'yes', -10, 0}, { 'tunnel', 'true', -10, 0}, { 'tunnel', 1, -10, 0}, + { 'highway', 'minor', 3, 0}, { 'highway', 'road', 3, 0 }, { 'highway', 'unclassified', 3, 0 }, + { 'highway', 'residential', 3, 0 }, { 'highway', 'tertiary_link', 4, 0}, { 'highway', 'tertiary', 4, 0}, + { 'highway', 'secondary_link', 6, 1}, { 'highway', 'secondary', 6, 1}, + { 'highway', 'primary_link', 7, 1}, { 'highway', 'primary', 7, 1}, + { 'highway', 'trunk_link', 8, 1}, { 'highway', 'trunk', 8, 1}, + { 'highway', 'motorway_link', 9, 1}, { 'highway', 'motorway', 9, 1}, +} + +local tables = {} + +tables.point = osm2pgsql.define_table{ + name = 'planet_osm_point', + ids = { type = 'node', id_column = 'osm_id' }, + columns = { + { column = 'access', type = 'text' }, + { column = 'addr:housename', type = 'text' }, + { column = 'addr:housenumber', type = 'text' }, + { column = 'addr:interpolation', type = 'text' }, + { column = 'admin_level', type = 'text' }, + { column = 'aerialway', type = 'text' }, + { column = 'aeroway', type = 'text' }, + { column = 'amenity', type = 'text' }, + { column = 'area', type = 'text' }, + { column = 'barrier', type = 'text' }, + { column = 'bicycle', type = 'text' }, + { column = 'brand', type = 'text' }, + { column = 'bridge', type = 'text' }, + { column = 'boundary', type = 'text' }, + { column = 'building', type = 'text' }, + { column = 'capital', type = 'text' }, + { column = 'construction', type = 'text' }, + { column = 'covered', type = 'text' }, + { column = 'culvert', type = 'text' }, + { column = 'cutting', type = 'text' }, + { column = 'denomination', type = 'text' }, + { column = 'disused', type = 'text' }, + { column = 'ele', type = 'text' }, + { column = 'embankment', type = 'text' }, + { column = 'foot', type = 'text' }, + { column = 'generator:source', type = 'text' }, + { column = 'harbour', type = 'text' }, + { column = 'highway', type = 'text' }, + { column = 'historic', type = 'text' }, + { column = 'horse', type = 'text' }, + { column = 'intermittent', type = 'text' }, + { column = 'junction', type = 'text' }, + { column = 'landuse', type = 'text' }, + { column = 'layer', type = 'text' }, + { column = 'leisure', type = 'text' }, + { column = 'lock', type = 'text' }, + { column = 'man_made', type = 'text' }, + { column = 'military', type = 'text' }, + { column = 'motorcar', type = 'text' }, + { column = 'name', type = 'text' }, + { column = 'natural', type = 'text' }, + { column = 'office', type = 'text' }, + { column = 'oneway', type = 'text' }, + { column = 'operator', type = 'text' }, + { column = 'place', type = 'text' }, + { column = 'population', type = 'text' }, + { column = 'power', type = 'text' }, + { column = 'power_source', type = 'text' }, + { column = 'public_transport', type = 'text' }, + { column = 'railway', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'religion', type = 'text' }, + { column = 'route', type = 'text' }, + { column = 'service', type = 'text' }, + { column = 'shop', type = 'text' }, + { column = 'sport', type = 'text' }, + { column = 'surface', type = 'text' }, + { column = 'toll', type = 'text' }, + { column = 'tourism', type = 'text' }, + { column = 'tower:type', type = 'text' }, + { column = 'tunnel', type = 'text' }, + { column = 'water', type = 'text' }, + { column = 'waterway', type = 'text' }, + { column = 'wetland', type = 'text' }, + { column = 'width', type = 'text' }, + { column = 'wood', type = 'text' }, + { column = 'z_order', type = 'int' }, + { column = 'way', type = 'point' }, + } +} + +tables.line = osm2pgsql.define_table{ + name = 'planet_osm_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = { + { column = 'access', type = 'text' }, + { column = 'addr:housename', type = 'text' }, + { column = 'addr:housenumber', type = 'text' }, + { column = 'addr:interpolation', type = 'text' }, + { column = 'admin_level', type = 'text' }, + { column = 'aerialway', type = 'text' }, + { column = 'aeroway', type = 'text' }, + { column = 'amenity', type = 'text' }, + { column = 'area', type = 'text' }, + { column = 'barrier', type = 'text' }, + { column = 'bicycle', type = 'text' }, + { column = 'brand', type = 'text' }, + { column = 'bridge', type = 'text' }, + { column = 'boundary', type = 'text' }, + { column = 'building', type = 'text' }, + { column = 'construction', type = 'text' }, + { column = 'covered', type = 'text' }, + { column = 'culvert', type = 'text' }, + { column = 'cutting', type = 'text' }, + { column = 'denomination', type = 'text' }, + { column = 'disused', type = 'text' }, + { column = 'embankment', type = 'text' }, + { column = 'foot', type = 'text' }, + { column = 'generator:source', type = 'text' }, + { column = 'harbour', type = 'text' }, + { column = 'highway', type = 'text' }, + { column = 'historic', type = 'text' }, + { column = 'horse', type = 'text' }, + { column = 'intermittent', type = 'text' }, + { column = 'junction', type = 'text' }, + { column = 'landuse', type = 'text' }, + { column = 'layer', type = 'text' }, + { column = 'leisure', type = 'text' }, + { column = 'lock', type = 'text' }, + { column = 'man_made', type = 'text' }, + { column = 'military', type = 'text' }, + { column = 'motorcar', type = 'text' }, + { column = 'name', type = 'text' }, + { column = 'natural', type = 'text' }, + { column = 'office', type = 'text' }, + { column = 'oneway', type = 'text' }, + { column = 'operator', type = 'text' }, + { column = 'place', type = 'text' }, + { column = 'population', type = 'text' }, + { column = 'power', type = 'text' }, + { column = 'power_source', type = 'text' }, + { column = 'public_transport', type = 'text' }, + { column = 'railway', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'religion', type = 'text' }, + { column = 'route', type = 'text' }, + { column = 'service', type = 'text' }, + { column = 'shop', type = 'text' }, + { column = 'sport', type = 'text' }, + { column = 'surface', type = 'text' }, + { column = 'toll', type = 'text' }, + { column = 'tourism', type = 'text' }, + { column = 'tower:type', type = 'text' }, + { column = 'tracktype', type = 'text' }, + { column = 'tunnel', type = 'text' }, + { column = 'water', type = 'text' }, + { column = 'waterway', type = 'text' }, + { column = 'wetland', type = 'text' }, + { column = 'width', type = 'text' }, + { column = 'wood', type = 'text' }, + { column = 'z_order', type = 'int' }, + { column = 'way_area', type = 'area' }, + { column = 'way', type = 'linestring' }, + } +} + +tables.polygon = osm2pgsql.define_table{ + name = 'planet_osm_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = { + { column = 'access', type = 'text' }, + { column = 'addr:housename', type = 'text' }, + { column = 'addr:housenumber', type = 'text' }, + { column = 'addr:interpolation', type = 'text' }, + { column = 'admin_level', type = 'text' }, + { column = 'aerialway', type = 'text' }, + { column = 'aeroway', type = 'text' }, + { column = 'amenity', type = 'text' }, + { column = 'area', type = 'text' }, + { column = 'barrier', type = 'text' }, + { column = 'bicycle', type = 'text' }, + { column = 'brand', type = 'text' }, + { column = 'bridge', type = 'text' }, + { column = 'boundary', type = 'text' }, + { column = 'building', type = 'text' }, + { column = 'construction', type = 'text' }, + { column = 'covered', type = 'text' }, + { column = 'culvert', type = 'text' }, + { column = 'cutting', type = 'text' }, + { column = 'denomination', type = 'text' }, + { column = 'disused', type = 'text' }, + { column = 'embankment', type = 'text' }, + { column = 'foot', type = 'text' }, + { column = 'generator:source', type = 'text' }, + { column = 'harbour', type = 'text' }, + { column = 'highway', type = 'text' }, + { column = 'historic', type = 'text' }, + { column = 'horse', type = 'text' }, + { column = 'intermittent', type = 'text' }, + { column = 'junction', type = 'text' }, + { column = 'landuse', type = 'text' }, + { column = 'layer', type = 'text' }, + { column = 'leisure', type = 'text' }, + { column = 'lock', type = 'text' }, + { column = 'man_made', type = 'text' }, + { column = 'military', type = 'text' }, + { column = 'motorcar', type = 'text' }, + { column = 'name', type = 'text' }, + { column = 'natural', type = 'text' }, + { column = 'office', type = 'text' }, + { column = 'oneway', type = 'text' }, + { column = 'operator', type = 'text' }, + { column = 'place', type = 'text' }, + { column = 'population', type = 'text' }, + { column = 'power', type = 'text' }, + { column = 'power_source', type = 'text' }, + { column = 'public_transport', type = 'text' }, + { column = 'railway', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'religion', type = 'text' }, + { column = 'route', type = 'text' }, + { column = 'service', type = 'text' }, + { column = 'shop', type = 'text' }, + { column = 'sport', type = 'text' }, + { column = 'surface', type = 'text' }, + { column = 'toll', type = 'text' }, + { column = 'tourism', type = 'text' }, + { column = 'tower:type', type = 'text' }, + { column = 'tracktype', type = 'text' }, + { column = 'tunnel', type = 'text' }, + { column = 'water', type = 'text' }, + { column = 'waterway', type = 'text' }, + { column = 'wetland', type = 'text' }, + { column = 'width', type = 'text' }, + { column = 'wood', type = 'text' }, + { column = 'z_order', type = 'int' }, + { column = 'way_area', type = 'area' }, + { column = 'way', type = 'geometry' }, + } +} + +tables.roads = osm2pgsql.define_table{ + name = 'planet_osm_roads', + ids = { type = 'way', id_column = 'osm_id' }, + columns = { + { column = 'access', type = 'text' }, + { column = 'addr:housename', type = 'text' }, + { column = 'addr:housenumber', type = 'text' }, + { column = 'addr:interpolation', type = 'text' }, + { column = 'admin_level', type = 'text' }, + { column = 'aerialway', type = 'text' }, + { column = 'aeroway', type = 'text' }, + { column = 'amenity', type = 'text' }, + { column = 'area', type = 'text' }, + { column = 'barrier', type = 'text' }, + { column = 'bicycle', type = 'text' }, + { column = 'brand', type = 'text' }, + { column = 'bridge', type = 'text' }, + { column = 'boundary', type = 'text' }, + { column = 'building', type = 'text' }, + { column = 'construction', type = 'text' }, + { column = 'covered', type = 'text' }, + { column = 'culvert', type = 'text' }, + { column = 'cutting', type = 'text' }, + { column = 'denomination', type = 'text' }, + { column = 'disused', type = 'text' }, + { column = 'embankment', type = 'text' }, + { column = 'foot', type = 'text' }, + { column = 'generator:source', type = 'text' }, + { column = 'harbour', type = 'text' }, + { column = 'highway', type = 'text' }, + { column = 'historic', type = 'text' }, + { column = 'horse', type = 'text' }, + { column = 'intermittent', type = 'text' }, + { column = 'junction', type = 'text' }, + { column = 'landuse', type = 'text' }, + { column = 'layer', type = 'text' }, + { column = 'leisure', type = 'text' }, + { column = 'lock', type = 'text' }, + { column = 'man_made', type = 'text' }, + { column = 'military', type = 'text' }, + { column = 'motorcar', type = 'text' }, + { column = 'name', type = 'text' }, + { column = 'natural', type = 'text' }, + { column = 'office', type = 'text' }, + { column = 'oneway', type = 'text' }, + { column = 'operator', type = 'text' }, + { column = 'place', type = 'text' }, + { column = 'population', type = 'text' }, + { column = 'power', type = 'text' }, + { column = 'power_source', type = 'text' }, + { column = 'public_transport', type = 'text' }, + { column = 'railway', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'religion', type = 'text' }, + { column = 'route', type = 'text' }, + { column = 'service', type = 'text' }, + { column = 'shop', type = 'text' }, + { column = 'sport', type = 'text' }, + { column = 'surface', type = 'text' }, + { column = 'toll', type = 'text' }, + { column = 'tourism', type = 'text' }, + { column = 'tower:type', type = 'text' }, + { column = 'tracktype', type = 'text' }, + { column = 'tunnel', type = 'text' }, + { column = 'water', type = 'text' }, + { column = 'waterway', type = 'text' }, + { column = 'wetland', type = 'text' }, + { column = 'width', type = 'text' }, + { column = 'wood', type = 'text' }, + { column = 'z_order', type = 'int' }, + { column = 'way_area', type = 'area' }, + { column = 'way', type = 'linestring' }, + } +} + +function get_z_order(keyvalues) + -- The default z_order is 0 + local z_order = 0 + local roads = false + + -- Add the value of the layer key times 10 to z_order + if (keyvalues.layer ~= nil and tonumber(keyvalues.layer)) then + z_order = 10*keyvalues.layer + end + + -- Increase or decrease z_order based on the specific key/value combination as specified in zordering_tags + for i,k in ipairs(zordering_tags) do + -- If the value in zordering_tags is specified, match key and value. Otherwise, match key only. + if ((k[2] and keyvalues[k[1]] == k[2]) or (k[2] == nil and keyvalues[k[1]] ~= nil)) then + -- If the fourth component of the element of zordering_tags is 1, add the object to planet_osm_roads + if (k[4] == 1) then + roads = true + end + z_order = z_order + k[3] + end + end + + return z_order, roads +end + +-- Helper function to check whether a table is empty +function is_empty(some_table) + return next(some_table) == nil +end + +function has_generic_tag(tags) + for k, v in pairs(tags) do + for j, k2 in ipairs(generic_keys) do + if k == k2 then + return true + end + end + end + return false +end + +function osm2pgsql.process_node(object) + if is_empty(object.tags) then + return + end + + for i,k in ipairs(delete_tags) do + object.tags[k] = nil + end + + if not has_generic_tag(object.tags) then + return + end + + tables.point:add_row(object.tags) +end + +-- Treat objects with a key in polygon_keys as polygon +function is_polygon(tags) + for i,k in ipairs(polygon_keys) do + if tags[k] then + return true + end + end + return false +end + +function osm2pgsql.process_way(object) + if is_empty(object.tags) then + return + end + + for i,k in ipairs(delete_tags) do + object.tags[k] = nil + end + + if not has_generic_tag(object.tags) then + return + end + + local polygon = is_polygon(object.tags) + -- Treat objects tagged as area=yes, area=1, or area=true as polygon, + -- and treat objects tagged as area=no, area=0, or area=false not as polygon + local area_tag = object.tags.area + if area_tag == 'yes' or area_tag == '1' or area_tag == 'true' then + polygon = true + elseif area_tag == 'no' or area_tag == '0' or area_tag == 'false' then + polygon = false + end + + local z_order, roads = get_z_order(object.tags) + object.tags.z_order = z_order + + if polygon then + object.tags.way = { create = 'area' } + tables.polygon:add_row(object.tags) + else + object.tags.way = { create = 'line', split_at = 100000 } + tables.line:add_row(object.tags) + end + + if roads then + object.tags.way = { create = 'line', split_at = 100000 } + tables.roads:add_row(object.tags) + end +end + +function osm2pgsql.process_relation(object) + if is_empty(object.tags) then + return + end + + local type = object.tags.type + if (type ~= 'route') and (type ~= 'multipolygon') and (type ~= 'boundary') then + return + end + + for i,k in ipairs(delete_tags) do + object.tags[k] = nil + end + + if not has_generic_tag(object.tags) then + return + end + + local z_order, roads = get_z_order(object.tags) + object.tags.z_order = z_order + + local linestring = false + if type == 'boundary' then + linestring = true + elseif type == 'multipolygon' then + if object.tags.boundary then + linestring = true + else + object.tags.way = { create = 'area' } + tables.polygon:add_row(object.tags) + end + end + + if linestring then + object.tags.way = { create = 'line', split_at = 100000 } + tables.line:add_row(object.tags) + end + + if roads then + object.tags.way = { create = 'line', split_at = 100000 } + tables.roads:add_row(object.tags) + end +end + diff --git a/flex-config/data-types.lua b/flex-config/data-types.lua new file mode 100644 index 000000000..2765290b1 --- /dev/null +++ b/flex-config/data-types.lua @@ -0,0 +1,120 @@ + +-- This is a very simple Lua config for the Flex Backend not intended for +-- real-world use. Look at and understand "simple.lua" first, before looking +-- at this file. This file demonstrates some column data type options. + +local highways = osm2pgsql.define_way_table('highways', { + { column = 'name', type = 'text' }, + { column = 'type', type = 'text' }, + + -- type "direction" is special, see below + { column = 'oneway', type = 'direction' }, + { column = 'maxspeed', type = 'int' }, + + -- type "bool" is special, see below + { column = 'lit', type = 'bool' }, + { column = 'tags', type = 'hstore' }, + + -- an PostgreSQL array type, not specially handled by osm2pgsql, see below + { column = 'nodes', type = 'int8[]' }, + { column = 'geom', type = 'linestring' }, +}) + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +local highway_types = { + 'motorway', + 'motorway_link', + 'trunk', + 'trunk_link', + 'primary', + 'primary_link', + 'secondary', + 'secondary_link', + 'tertiary', + 'tertiary_link', + 'unclassified', + 'residential', + 'track', + 'service', +} + +-- Prepare table "types" for quick checking of highway types +local types = {} +for _, k in ipairs(highway_types) do + types[k] = 1 +end + +-- Parse a maxspeed value like "30" or "55 mph" and return a number in km/h +function parse_speed(input) + if not input then + return nil + end + + local maxspeed = tonumber(input) + + -- If maxspeed is just a number, it is in km/h, so just return it + if maxspeed then + return maxspeed + end + + -- If there is an 'mph' at the end, convert to km/h and return + if input:sub(-3) == 'mph' then + local num = tonumber(input:sub(1, -4)) + if num then + return math.floor(num * 1.60934) + end + end + + return nil +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + -- Get the type of "highway" and remove it from the tags + local type = object:grab_tag('highway') + + -- We are only interested in highways of the given types + if not types[type] then + return + end + + -- We want to put the name in its own column + local name = object:grab_tag('name') + + highways:add_row({ + name = name, + type = type, + + -- The 'maxspeed' column gets the maxspeed in km/h + maxspeed = parse_speed(object.tags.maxspeed), + + -- The 'oneway' column has the special type "direction", which will + -- store "yes", "true" and "1" as 1, "-1" as -1, and everything else + -- as 0. + oneway = object.tags.oneway or 0, + + -- The 'lit' column has the special type "bool", which will store + -- "yes" and "true" as true and everything else as false value. + lit = object.tags.lit, + + -- The way node ids are put into a format that PostgreSQL understands + -- for a column of type "int8[]". + nodes = '{' .. table.concat(object.nodes, ',') .. '}', + + tags = object.tags + }) +end + diff --git a/flex-config/default-config.lua b/flex-config/default-config.lua new file mode 100644 index 000000000..4aceb6b5d --- /dev/null +++ b/flex-config/default-config.lua @@ -0,0 +1,266 @@ + +-- This is a default configuration that is a good starting point for +-- real-world projects. + +local tables = {} + +tables.points = osm2pgsql.define_node_table('points', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point' }, +}) + +tables.lines = osm2pgsql.define_way_table('lines', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'linestring' }, +}) + +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'geometry' }, + { column = 'area', type = 'area' }, +}) + +tables.routes = osm2pgsql.define_relation_table('routes', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'multilinestring' }, +}) + +tables.boundaries = osm2pgsql.define_relation_table('boundaries', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'multilinestring' }, +}) + +-- These tag keys are generally regarded as useless for most rendering. Most +-- of them are from imports or intended as internal information for mappers. +-- +-- If a key ends in '*' it will match all keys with the specified prefix. +-- +-- If you want some of these keys, perhaps for a debugging layer, just +-- delete the corresponding lines. +local delete_keys = { + 'attribution', + 'comment', + 'created_by', + 'fixme', + 'note', + 'note:*', + 'odbl', + 'odbl:note', + 'source', + 'source:*', + 'source_ref', + + -- Lots of import tags + -- Corine (CLC) (Europe) + 'CLC:*', + + -- Geobase (CA) + 'geobase:*', + -- CanVec (CA) + 'canvec:*', + + -- osak (DK) + 'osak:*', + -- kms (DK) + 'kms:*', + + -- ngbe (ES) + -- See also note:es and source:file above + 'ngbe:*', + + -- Friuli Venezia Giulia (IT) + 'it:fvg:*', + + -- KSJ2 (JA) + -- See also note:ja and source_ref above + 'KSJ2:*', + -- Yahoo/ALPS (JA) + 'yh:*', + + -- LINZ (NZ) + 'LINZ2OSM:*', + 'linz2osm:*', + 'LINZ:*', + + -- WroclawGIS (PL) + 'WroclawGIS:*', + -- Naptan (UK) + 'naptan:*', + + -- TIGER (US) + 'tiger:*', + -- GNIS (US) + 'gnis:*', + -- National Hydrography Dataset (US) + 'NHD:*', + 'nhd:*', + -- mvdgis (Montevideo, UY) + 'mvdgis:*', + + -- EUROSHA (Various countries) + 'project:eurosha_2012', + + -- UrbIS (Brussels, BE) + 'ref:UrbIS', + + -- NHN (CA) + 'accuracy:meters', + 'sub_sea:type', + 'waterway:type', + -- StatsCan (CA) + 'statscan:rbuid', + + -- RUIAN (CZ) + 'ref:ruian:addr', + 'ref:ruian', + 'building:ruian:type', + -- DIBAVOD (CZ) + 'dibavod:id', + -- UIR-ADR (CZ) + 'uir_adr:ADRESA_KOD', + + -- GST (DK) + 'gst:feat_id', + + -- Maa-amet (EE) + 'maaamet:ETAK', + -- FANTOIR (FR) + 'ref:FR:FANTOIR', + + -- 3dshapes (NL) + '3dshapes:ggmodelk', + -- AND (NL) + 'AND_nosr_r', + + -- OPPDATERIN (NO) + 'OPPDATERIN', + -- Various imports (PL) + 'addr:city:simc', + 'addr:street:sym_ul', + 'building:usage:pl', + 'building:use:pl', + -- TERYT (PL) + 'teryt:simc', + + -- RABA (SK) + 'raba:id', + -- DCGIS (Washington DC, US) + 'dcgis:gis_id', + -- Building Identification Number (New York, US) + 'nycdoitt:bin', + -- Chicago Building Inport (US) + 'chicago:building_id', + -- Louisville, Kentucky/Building Outlines Import (US) + 'lojic:bgnum', + -- MassGIS (Massachusetts, US) + 'massgis:way_id', + + -- misc + 'import', + 'import_uuid', + 'OBJTYPE', + 'SK53_bulk:load' +} + +-- The osm2pgsql.make_clean_tags_func() function takes the list of keys +-- and key prefixes defined above and returns a function that can be used +-- to clean those tags out of a Lua table. The clean_tags function will +-- return true if it removed all tags from the table. +local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys) + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] + or tags['building:part'] +end + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + tables.points:add_row({ + tags = object.tags + }) +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + if object.is_closed and has_area_tags(object.tags) then + tables.polygons:add_row({ + tags = object.tags, + geom = { create = 'area' } + }) + else + tables.lines:add_row({ + tags = object.tags + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + local type = object.tags.type + + if type == 'route' then + tables.routes:add_row({ + tags = object.tags + }) + return + end + + if type == 'boundary' or (type == 'multipolygon' and object.tags.boundary) then + tables.boundaries:add_row({ + tags = object.tags, + geom = { create = 'line' } + }) + return + end + + if object.tags.type == 'multipolygon' then + tables.polygons:add_row({ + tags = object.tags, + geom = { create = 'area' } + }) + end +end + diff --git a/flex-config/geometries.lua b/flex-config/geometries.lua new file mode 100644 index 000000000..802bc6035 --- /dev/null +++ b/flex-config/geometries.lua @@ -0,0 +1,178 @@ + +-- This is a very simple Lua config for the Flex Backend not intended for +-- real-world use. Look at and understand "simple.lua" first, before looking +-- at this file. This file will show some options around geometry processing. +-- After you have understood this file, go on to "data-types.lua". + +-- The projection configured on the command line is available in Lua: +print('osm2pgsql srid: ' .. osm2pgsql.srid) + +-- You can use it, for instance, to figure out the maximum lengths for line +-- geometries, see below for how this is used. +if osm2pgsql.srid == 3857 then + max_length = 100000 +else + max_length = 1 +end + +local tables = {} + +tables.pois = osm2pgsql.define_node_table('pois', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point' }, +}) + +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'linestring' }, +}) + +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'geometry' }, + -- The 'area' type is used to store the calculated area of a polygon + -- feature. This can be used in style sheets to only render larger polygons + -- in small zoom levels. + { column = 'area', type = 'area' }, +}) + +tables.boundaries = osm2pgsql.define_relation_table('boundaries', { + { column = 'type', type = 'text' }, + { column = 'tags', type = 'hstore' }, + -- Boundaries will be stitched together from relation members into long + -- linestrings. This is a multilinestring column because sometimes the + -- boundaries are not contiguous. + { column = 'geom', type = 'multilinestring' }, +}) + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] +end + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + -- The 'geom' column is not mentioned here. So the default geometry + -- transformation for a column of type 'point' will be used and the + -- node location will be written as Point geometry into the database. + tables.pois:add_row({ + tags = object.tags + }) +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + -- A closed way that also has the right tags for an area is a polygon. + if object.is_closed and has_area_tags(object.tags) then + tables.polygons:add_row({ + tags = object.tags, + -- The 'geom' column of the 'polygons' table is of type 'geometry'. + -- There are several ways a way geometry could be converted to + -- a geometry so you have to specify the geometry transformation. + -- In this case we want to convert the way data to an area. + geom = { create = 'area' } + }) + else + -- The 'geom' column of the 'ways' table is of type 'linestring'. + -- Osm2pgsql knows how to create a linestring from a way, but + -- if you want to specify extra parameters to this conversion, + -- you have to do this explicitly. In this case we want to split + -- long linestrings. + -- + -- Set "split_at" to the maximum length the pieces should have. This + -- length is in map units, so it depends on the projection used. + -- "Traditional" osm2pgsql sets this to 1 for 4326 geometries and + -- 100000 for 3857 (web mercator) geometries. The default is 0.0, which + -- means no splitting. + -- + -- Note that if a way is split this will automatically create + -- multiple rows that are identical except for the geometry. + tables.ways:add_row({ + tags = object.tags, + geom = { create = 'line', split_at = max_length } + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + local type = object:grab_tag('type') + + -- Store boundary relations as multilinestrings + if type == 'boundary' then + tables.boundaries:add_row({ + type = object:grab_tag('boundary'), + tags = object.tags, + -- For relations there is no clear definition what their geometry + -- is, so you have to declare the geometry transformation + -- explicitly. + geom = { create = 'line' } + }) + return + end + + -- Store multipolygon relations as polygons + if type == 'multipolygon' then + tables.polygons:add_row({ + tags = object.tags, + -- For relations there is no clear definition what their geometry + -- is, so you have to declare the geometry transformation + -- explicitly. + geom = { create = 'area' } + }) + end +end + diff --git a/flex-config/places.lua b/flex-config/places.lua new file mode 100644 index 000000000..61fb1383b --- /dev/null +++ b/flex-config/places.lua @@ -0,0 +1,52 @@ + +-- This example shows how you can use JSON and JSONB columns in your database. + +-- You need the 'dkjson' JSON library for Lua. Install 'dkjson' with LuaRocks +-- or install from http://dkolf.de/src/dkjson-lua.fsl/home . Debian/Ubuntu +-- users can install the 'lua-dkjson' package. + +-- Use JSON encoder +local json = require('dkjson') + +local places = osm2pgsql.define_node_table('places', { + -- The jsonb column is handled by osm2pgsql just like a normal text + -- column. It is the job of the Lua script to put correct json there. + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point' }, +}) + +local function starts_with(str, start) + return str:sub(1, #start) == start +end + +function osm2pgsql.process_node(object) + if not object.tags.place then + return + end + + -- Put all name:* tags in their own substructure + local names = {} + local has_names = false + for k, v in pairs(object.tags) do + if k == 'name' then + names[''] = v + object.tags.name = nil + has_names = true + elseif starts_with(k, "name:") then + -- extract language + local lang = k:sub(6, -1) + names[lang] = v + object.tags[k] = nil + has_names = true + end + end + + if has_names then + object.tags.names = names + end + + places:add_row({ + tags = json.encode(object.tags) + }) +end + diff --git a/flex-config/route-relations.lua b/flex-config/route-relations.lua new file mode 100644 index 000000000..cfd251c5d --- /dev/null +++ b/flex-config/route-relations.lua @@ -0,0 +1,95 @@ + +-- This file shows how to use multi-stage processing to bring tags from +-- relations into ways. + +-- This will only import ways tagged as highway. The 'rel_refs' text column +-- will contain a comma-separated list of all ref tags found in parent +-- relations with type=route and route=road. The 'rel_ids' column will be +-- an integer array containing the relation ids. These could be used, for +-- instance, to look up other relation tags from the 'routes' table. + +local tables = {} + +tables.highways = osm2pgsql.define_way_table('highways', { + { column = 'tags', type = 'hstore' }, + { column = 'rel_refs', type = 'text' }, -- for the refs from the relations + { column = 'rel_ids', type = 'int8[]' }, -- array with integers (for relation IDs) + { column = 'geom', type = 'linestring' }, +}) + +-- Tables don't have to have a geometry column +tables.routes = osm2pgsql.define_relation_table('routes', { + { column = 'tags', type = 'hstore' }, +}) + +-- This will be used to store lists of relation ids queryable by way id +by_way_id = {} + +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +function osm2pgsql.process_way(object) + -- We are only interested in highways + if not object.tags.highway then + return + end + + -- In stage 1: Mark all remaining ways so we will see them again in stage 2 + if osm2pgsql.stage == 1 then + osm2pgsql.mark_way(object.id) + return + end + + -- We are now in stage 2 + + clean_tags(object.tags) + + -- Data we will store in the "highways" table always has the way tags + local row = { + tags = object.tags + } + + -- If there is any data from relations, add it in + local d = by_way_id[object.id] + if d then + table.sort(d.refs) + table.sort(d.ids) + row.rel_refs = table.concat(d.refs, ',') + row.rel_ids = '{' .. table.concat(d.ids, ',') .. '}' + end + + tables.highways:add_row(row) +end + +function osm2pgsql.process_relation(object) + -- Only interested in relations with type=route, route=road and a ref + if object.tags.type == 'route' and object.tags.route == 'road' and object.tags.ref then + tables.routes:add_row({ + tags = object.tags, + geom = { create = 'line' } + }) + + -- Go through all the members and store relation ids and refs so it + -- can be found by the way id. + for _, member in ipairs(object.members) do + if member.type == 'w' then + if not by_way_id[member.ref] then + by_way_id[member.ref] = { + ids = {}, + refs = {} + } + end + local d = by_way_id[member.ref] + table.insert(d.ids, object.id) + table.insert(d.refs, object.tags.ref) + end + end + end +end + diff --git a/flex-config/simple.lua b/flex-config/simple.lua new file mode 100644 index 000000000..d561f3e31 --- /dev/null +++ b/flex-config/simple.lua @@ -0,0 +1,154 @@ + +-- This is a very simple Lua config for the Flex Backend not intended for +-- real-world use. Use it do understand the basic principles of the +-- configuration. After reading and understanding this, have a look at +-- "geometries.lua". + +-- For debugging +inspect = require('inspect') + +-- The global variable "osm2pgsql" is used to talk to the main osm2pgsql code. +-- You can, for instance, get the version of osm2pgsql: +print('osm2pgsql version: ' .. osm2pgsql.version) + +-- A place to store the SQL tables we will define shortly. +local tables = {} + +-- Create a new table called "pois" with the given columns. When running in +-- "create" mode, this will do the `CREATE TABLE`, when running in "append" +-- mode, this will only declare the table for use. +-- +-- This is a "node table", it can only contain data derived from nodes and will +-- contain a "node_id" column (SQL type INT8) as first column. When running in +-- "append" mode, osm2pgsql will automatically update this table using the node +-- ids. +tables.pois = osm2pgsql.define_node_table('pois', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point' }, -- will be something like `GEOMETRY(Point, 4326)` in SQL +}) + +-- A special table for restaurants to demonstrate that we can have any tables +-- with any columns we want. +tables.restaurants = osm2pgsql.define_node_table('restaurants', { + { column = 'name', type = 'text' }, + { column = 'cuisine', type = 'text' }, + { column = 'geom', type = 'point' }, +}) + +-- This is a "way table", it can only contain data derived from ways and will +-- contain a "way_id" column. When running in "append" mode, osm2pgsql will +-- automatically update this table using the way ids. +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'linestring' }, +}) + +-- This is an "area table", it can contain data derived from ways or relations +-- and will contain an "area_id" column. Way ids will be stored "as is" in the +-- "area_id" column, for relations the negative id will be stored. When +-- running in "append" mode, osm2pgsql will automatically update this table +-- using the way/relation ids. +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'hstore' }, + -- The type of the `geom` column is `geometry`, because we need to store + -- polygons AND multipolygons + { column = 'geom', type = 'geometry' }, +}) + +-- Debug output: Show definition of tables +for name, table in pairs(tables) do + print("\ntable '" .. name .. "':") + print(" name='" .. table:name() .. "'") + print(" columns=" .. inspect(table:columns())) +end + +-- Helper function to check whether a table is empty +function is_empty(some_table) + return next(some_table) == nil +end + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Called for every node in the input. The `object` argument contains all the +-- attributes of the node like `id`, `version`, etc. as well as all tags as a +-- Lua table (`object.tags`). +function osm2pgsql.process_node(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + if object.tags.amenity == 'restaurant' then + -- Add a row to the SQL table. The keys in the parameter table + -- correspond to the table columns, if one is missing the column will + -- be NULL. Id and geometry columns will be filled automatically. + tables.restaurants:add_row({ + name = object.tags.name, + cuisine = object.tags.cuisine + }) + else + tables.pois:add_row({ + -- We know `tags` is of type `hstore` so this will do the + -- right thing. + tags = object.tags + }) + end +end + +-- Called for every way in the input. The `object` argument contains the same +-- information as with nodes and additionally a boolean `is_closed` flag and +-- the list of node IDs referenced by the way (`object.nodes`). +function osm2pgsql.process_way(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + -- Very simple check to decide whether a way is a polygon or not, in a + -- real stylesheet we'd have to also look at the tags... + if object.is_closed then + tables.polygons:add_row({ + tags = object.tags, + geom = { create = 'area' } + }) + else + tables.ways:add_row({ + tags = object.tags + }) + end +end + +-- Called for every relation in the input. The `object` argument contains the +-- same information as with nodes and additionally an array of members +-- (`object.members`). +function osm2pgsql.process_relation(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + -- Store multipolygons and boundaries as polygons + if object.tags.type == 'multipolygon' or + object.tags.type == 'boundary' then + tables.polygons:add_row({ + tags = object.tags, + geom = { create = 'area' } + }) + end +end + diff --git a/flex-config/unitable.lua b/flex-config/unitable.lua new file mode 100644 index 000000000..20832c8cd --- /dev/null +++ b/flex-config/unitable.lua @@ -0,0 +1,71 @@ + +-- Put all OSM data into a single table + +inspect = require('inspect') + +-- We define a single table that can take any OSM object and any geometry. +-- XXX expire will currently not work on these tables. +local dtable = osm2pgsql.define_table{ + name = "data", + -- This will generate a column "osm_id INT8" for the id, and a column + -- "osm_type CHAR(1)" for the type of object: N(ode), W(way), R(relation) + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'attrs', type = 'hstore' }, + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'geometry' }, + } +} + +print("columns=" .. inspect(dtable:columns())) + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +function process(object, geometry_type) + if clean_tags(object.tags) then + return + end + dtable:add_row({ + attrs = { + version = object.version, + timestamp = object.timestamp, + }, + tags = object.tags, + geom = { create = geometry_type } + }) +end + +function osm2pgsql.process_node(object) + process(object, 'point') +end + +function osm2pgsql.process_way(object) + process(object, 'line') +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then + dtable:add_row({ + attrs = { + version = object.version, + timestamp = object.timestamp, + }, + tags = object.tags, + geom = { create = 'area' } + }) + end +end + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 71f3c9131..67ce8bb15 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(osm2pgsql_lib_SOURCES expire-tiles.cpp gazetteer-style.cpp geometry-processor.cpp + geom-transform.cpp id-tracker.cpp middle-pgsql.cpp middle-ram.cpp @@ -34,6 +35,7 @@ set(osm2pgsql_lib_SOURCES expire-tiles.hpp gazetteer-style.hpp geometry-processor.hpp + geom-transform.hpp id-tracker.hpp middle-pgsql.hpp middle-ram.hpp @@ -66,7 +68,17 @@ set(osm2pgsql_lib_SOURCES ) if (LUA_FOUND OR LUAJIT_FOUND) - list(APPEND osm2pgsql_lib_SOURCES tagtransform-lua.cpp) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/init.lua") + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/init.lua" LUA_INIT_CODE) + configure_file(lua-init.cpp.in lua-init.cpp @ONLY) + list(APPEND osm2pgsql_lib_SOURCES + flex-table.cpp + flex-table-column.cpp + lua-utils.cpp + output-flex.cpp + tagtransform-lua.cpp + ${CMAKE_CURRENT_BINARY_DIR}/lua-init.cpp) endif() list(APPEND osm2pgsql_lib_SOURCES ${PROJECT_BINARY_DIR}/src/version.cpp) diff --git a/src/db-copy.cpp b/src/db-copy.cpp index 5b02e6757..771308df7 100644 --- a/src/db-copy.cpp +++ b/src/db-copy.cpp @@ -24,6 +24,50 @@ void db_deleter_by_id_t::delete_rows(std::string const &table, conn->exec(fmt::to_string(sql)); } +void db_deleter_by_type_and_id_t::delete_rows(std::string const &table, + std::string const &column, + pg_conn_t *conn) +{ + assert(!m_deletables.empty()); + + fmt::memory_buffer sql; + // Need a VALUES line for each deletable: type (3 bytes), id (15 bytes), + // braces etc. (4 bytes). And additional space for the remainder of the + // SQL command. + sql.reserve(m_deletables.size() * 22 + 200); + + if (m_has_type) { + fmt::format_to(sql, "DELETE FROM {} p USING (VALUES ", table); + + for (auto const &item : m_deletables) { + fmt::format_to(sql, FMT_STRING("('{}',{}),"), item.osm_type, + item.osm_id); + } + + sql.resize(sql.size() - 1); + + auto const pos = column.find(','); + assert(pos != std::string::npos); + std::string type = column.substr(0, pos); + + fmt::format_to(sql, + ") AS t (osm_type, osm_id) WHERE" + " p.{} = t.osm_type AND p.{} = t.osm_id", + type, column.c_str() + pos + 1); + + conn->exec(fmt::to_string(sql)); + } else { + fmt::format_to(sql, FMT_STRING("DELETE FROM {} WHERE {} IN ("), table, + column); + + for (auto const &item : m_deletables) { + format_to(sql, FMT_STRING("{},"), item.osm_id); + } + sql[sql.size() - 1] = ')'; + conn->exec(fmt::to_string(sql)); + } +} + db_copy_thread_t::db_copy_thread_t(std::string const &conninfo) { // conninfo is captured by copy here, because we don't know wether the diff --git a/src/db-copy.hpp b/src/db-copy.hpp index 863cc05bf..51b071b40 100644 --- a/src/db-copy.hpp +++ b/src/db-copy.hpp @@ -67,6 +67,48 @@ class db_deleter_by_id_t std::vector m_deletables; }; +/** + * Deleter which removes objects by (optional) type and id from the database. + */ +class db_deleter_by_type_and_id_t +{ + enum + { + // There is a trade-off here between sending as few DELETE SQL as + // possible and keeping the size of the deletable vector managable. + Max_entries = 1000000 + }; + + struct item_t + { + osmid_t osm_id; + char osm_type; + + item_t(char t, osmid_t i) : osm_id(i), osm_type(t) {} + }; + +public: + bool has_data() const noexcept { return !m_deletables.empty(); } + + void add(char type, osmid_t osm_id) + { + m_deletables.emplace_back(type, osm_id); + if (type != 'X') { + m_has_type = true; + } + } + + void delete_rows(std::string const &table, std::string const &column, + pg_conn_t *conn); + + bool is_full() const noexcept { return m_deletables.size() > Max_entries; } + +private: + /// Vector with object to delete before copying + std::vector m_deletables; + bool m_has_type = false; +}; + /** * A command for the copy thread to execute. */ diff --git a/src/expire-tiles.cpp b/src/expire-tiles.cpp index 1bc2e20dd..e28f613f1 100644 --- a/src/expire-tiles.cpp +++ b/src/expire-tiles.cpp @@ -419,6 +419,25 @@ int expire_tiles::from_db(table_t *table, osmid_t osm_id) return wkbs.get_count(); } +int expire_tiles::from_result(pg_result_t const &result, osmid_t osm_id) +{ + //bail if we dont care about expiry + if (maxzoom == 0) { + return -1; + } + + //dirty the stuff + auto const num_tuples = result.num_tuples(); + for (int i = 0; i < num_tuples; ++i) { + char const *const wkb = result.get_value(i, 0); + auto const binwkb = ewkb::parser_t::wkb_from_hex(wkb); + from_wkb(binwkb.c_str(), osm_id); + } + + //return how many rows were affected + return num_tuples; +} + void expire_tiles::merge_and_destroy(expire_tiles &other) { if (map_width != other.map_width) { diff --git a/src/expire-tiles.hpp b/src/expire-tiles.hpp index 626257ad6..d167ff9a6 100644 --- a/src/expire-tiles.hpp +++ b/src/expire-tiles.hpp @@ -5,6 +5,7 @@ #include #include "osmtypes.hpp" +#include "pgsql.hpp" class reprojection; class table_t; @@ -55,6 +56,7 @@ struct expire_tiles double max_lat); void from_wkb(const char *wkb, osmid_t osm_id); int from_db(table_t *table, osmid_t osm_id); + int from_result(pg_result_t const &result, osmid_t osm_id); /** * Write the list of expired tiles to a file. diff --git a/src/flex-table-column.cpp b/src/flex-table-column.cpp new file mode 100644 index 000000000..f0449d689 --- /dev/null +++ b/src/flex-table-column.cpp @@ -0,0 +1,160 @@ +#include "flex-table-column.hpp" + +#include +#include +#include +#include +#include +#include + +struct column_type_lookup +{ + char const *name; + table_column_type type; +}; + +static column_type_lookup const column_types[] = { + {"sql", table_column_type::sql}, + {"text", table_column_type::text}, + {"boolean", table_column_type::boolean}, + {"bool", table_column_type::boolean}, + {"int2", table_column_type::int2}, + {"smallint", table_column_type::int2}, + {"int4", table_column_type::int4}, + {"int", table_column_type::int4}, + {"integer", table_column_type::int4}, + {"int8", table_column_type::int8}, + {"bigint", table_column_type::int8}, + {"real", table_column_type::real}, + {"hstore", table_column_type::hstore}, + {"direction", table_column_type::direction}, + {"geometry", table_column_type::geometry}, + {"point", table_column_type::point}, + {"linestring", table_column_type::linestring}, + {"polygon", table_column_type::polygon}, + {"multipoint", table_column_type::multipoint}, + {"multilinestring", table_column_type::multilinestring}, + {"multipolygon", table_column_type::multipolygon}, + {"area", table_column_type::area}, + {"id_type", table_column_type::id_type}, + {"id_num", table_column_type::id_num}}; + +static table_column_type +get_column_type_from_string(std::string const &type) noexcept +{ + auto const it = + std::find_if(std::begin(column_types), std::end(column_types), + [&type](column_type_lookup name_type) { + return type == name_type.name; + }); + + if (it != std::end(column_types)) { + return it->type; + } + + // If we don't recognize the column type, we just assume its a valid SQL + // type and use it "as is". + return table_column_type::sql; +} + +static std::string lowercase(std::string const &str) +{ + std::string result; + + for (char c : str) { + result += + static_cast(std::tolower(static_cast(c))); + } + + return result; +} + +flex_table_column_t::flex_table_column_t(std::string name, + std::string const &type) +: m_name(std::move(name)), m_type_name(lowercase(type)), + m_type(get_column_type_from_string(m_type_name)) +{} + +std::string flex_table_column_t::sql_type_name(int srid) const +{ + switch (m_type) { + case table_column_type::sql: + return m_type_name; + break; + case table_column_type::text: + return "TEXT"; + break; + case table_column_type::boolean: + return "BOOLEAN"; + break; + case table_column_type::int2: + return "INT2"; + break; + case table_column_type::int4: + return "INT4"; + break; + case table_column_type::int8: + return "INT8"; + break; + case table_column_type::real: + return "REAL"; + break; + case table_column_type::hstore: + return "HSTORE"; + break; + case table_column_type::direction: + return "INT2"; + break; + case table_column_type::geometry: + return "GEOMETRY(GEOMETRY, {})"_format(srid); + break; + case table_column_type::point: + return "GEOMETRY(POINT, {})"_format(srid); + break; + case table_column_type::linestring: + return "GEOMETRY(LINESTRING, {})"_format(srid); + break; + case table_column_type::polygon: + return "GEOMETRY(POLYGON, {})"_format(srid); + break; + case table_column_type::multipoint: + return "GEOMETRY(MULTIPOINT, {})"_format(srid); + break; + case table_column_type::multilinestring: + return "GEOMETRY(MULTILINESTRING, {})"_format(srid); + break; + case table_column_type::multipolygon: + return "GEOMETRY(MULTIPOLYGON, {})"_format(srid); + break; + case table_column_type::area: + return "REAL"; + break; + case table_column_type::id_type: + return "CHAR(1)"; + break; + case table_column_type::id_num: + return "INT8"; + break; + } + throw std::runtime_error{"Unknown column type"}; +} + +std::string flex_table_column_t::sql_modifiers() const +{ + std::string modifiers; + + if ((m_flags & table_column_flags::not_null) != 0U) { + modifiers += "NOT NULL "; + } + + if (!modifiers.empty()) { + modifiers.resize(modifiers.size() - 1); + } + + return modifiers; +} + +std::string flex_table_column_t::sql_create(int srid) const +{ + return "\"{}\" {} {},"_format(m_name, sql_type_name(srid), sql_modifiers()); +} diff --git a/src/flex-table-column.hpp b/src/flex-table-column.hpp new file mode 100644 index 000000000..459a3edc7 --- /dev/null +++ b/src/flex-table-column.hpp @@ -0,0 +1,118 @@ +#ifndef OSM2PGSQL_FLEX_TABLE_COLUMN_HPP +#define OSM2PGSQL_FLEX_TABLE_COLUMN_HPP + +#include "format.hpp" + +#include +#include + +enum class table_column_type : uint8_t +{ + sql, + + text, + + boolean, + + int2, + int4, + int8, + + real, + + hstore, + + direction, + + geometry, + point, + linestring, + polygon, + multipoint, + multilinestring, + multipolygon, + + area, + + id_type, + id_num +}; + +enum table_column_flags : uint8_t +{ + none = 0, + not_null = 1 +}; + +/** + * A column in a flex_table_t. + */ +class flex_table_column_t +{ +public: + flex_table_column_t(std::string name, std::string const &type); + + std::string const &name() const noexcept { return m_name; } + + table_column_type type() const noexcept { return m_type; } + + table_column_flags flags() const noexcept { return m_flags; } + + bool is_point_column() const noexcept + { + return (m_type == table_column_type::point) || + (m_type == table_column_type::multipoint); + } + + bool is_linestring_column() const noexcept + { + return (m_type == table_column_type::linestring) || + (m_type == table_column_type::multilinestring); + } + + bool is_polygon_column() const noexcept + { + return (m_type == table_column_type::geometry) || + (m_type == table_column_type::polygon) || + (m_type == table_column_type::multipolygon); + } + + bool is_geometry_column() const noexcept + { + return (m_type >= table_column_type::geometry) && + (m_type <= table_column_type::multipolygon); + } + + std::string const &type_name() const noexcept { return m_type_name; } + + void set_not_null_constraint() noexcept + { + m_flags = static_cast(m_flags | + table_column_flags::not_null); + } + + std::string sql_type_name(int srid) const; + std::string sql_modifiers() const; + std::string sql_create(int srid) const; + +private: + /// The name of the database table column. + std::string m_name; + + /** + * The type name of the database table column. Either a name we recognize + * or just an SQL snippet. + */ + std::string m_type_name; + + /** + * The type of database column. Use table_column_type::sql as fallback + * in which case m_type_name is the SQL type used in the database. + */ + table_column_type m_type; + + /// Flags like NOT NULL. + table_column_flags m_flags = table_column_flags::none; +}; + +#endif // OSM2PGSQL_FLEX_TABLE_COLUMN_HPP diff --git a/src/flex-table.cpp b/src/flex-table.cpp new file mode 100644 index 000000000..4147b7509 --- /dev/null +++ b/src/flex-table.cpp @@ -0,0 +1,330 @@ +#include "flex-table.hpp" +#include "format.hpp" + +#include +#include +#include + +char const *type_to_char(osmium::item_type type) noexcept +{ + switch (type) { + case osmium::item_type::node: + return "N"; + case osmium::item_type::way: + return "W"; + case osmium::item_type::relation: + return "R"; + default: + break; + } + return "X"; +} + +static std::string tablespace_clause(const std::string &tablespace) +{ + if (tablespace.empty()) { + return ""; + } + + return " TABLESPACE \"{}\" "_format(tablespace); +} + +bool flex_table_t::has_multicolumn_id_index() const noexcept +{ + return m_columns[0].type() == table_column_type::id_type; +} + +std::string flex_table_t::id_column_names() const +{ + std::string name; + + if (!has_id_column()) { + return name; + } + + name = m_columns[0].name(); + if (has_multicolumn_id_index()) { + name += ','; + name += m_columns[1].name(); + } + + return name; +} + +std::string flex_table_t::full_name() const +{ + return "\"" + schema() + "\".\"" + name() + "\""; +} + +std::string flex_table_t::full_tmp_name() const +{ + return "\"" + schema() + "\".\"" + name() + "_tmp\""; +} + +flex_table_column_t &flex_table_t::add_column(std::string const &name, + std::string const &type) +{ + // id_type (optional) and id_num must always be the first columns + assert(type != "id_type" || m_columns.empty()); + assert(type != "id_num" || m_columns.empty() || + (m_columns.size() == 1 && + m_columns[0].type() == table_column_type::id_type)); + + m_columns.emplace_back(name, type); + auto &column = m_columns.back(); + + if (column.is_geometry_column()) { + m_geom_column = m_columns.size() - 1; + column.set_not_null_constraint(); + } + + return column; +} + +std::string flex_table_t::build_sql_prepare_get_wkb() const +{ + if (has_geom_column()) { + if (has_multicolumn_id_index()) { + return "PREPARE get_wkb (TEXT, BIGINT) AS" + " SELECT \"{}\" FROM {} WHERE \"{}\" = '$1' AND \"{}\" = $2"_format( + geom_column().name(), full_name(), m_columns[0].name(), + m_columns[1].name()); + } + return "PREPARE get_wkb (BIGINT) AS" + " SELECT \"{}\" FROM {} WHERE \"{}\" = $1"_format( + geom_column().name(), full_name(), id_column_names()); + } + return "PREPARE get_wkb (BIGINT) AS SELECT ''"; +} + +std::string flex_table_t::build_sql_create_table(bool final_table) const +{ + assert(!m_columns.empty()); + + std::string sql = "CREATE {} TABLE IF NOT EXISTS {} ("_format( + final_table ? "" : "UNLOGGED", + final_table ? full_tmp_name() : full_name()); + + for (auto const &column : m_columns) { + sql += column.sql_create(m_srid); + } + + assert(sql.back() == ','); + sql.back() = ')'; + + if (!final_table) { + sql += " WITH ( autovacuum_enabled = FALSE )"; + } + + sql += tablespace_clause(m_data_tablespace); + + return sql; +} + +std::string flex_table_t::build_sql_column_list() const +{ + assert(!m_columns.empty()); + + std::string result; + for (auto const &column : m_columns) { + result += '"'; + result += column.name(); + result += '"'; + result += ','; + } + result.resize(result.size() - 1); + + return result; +} + +void flex_table_t::connect(std::string const &conninfo) +{ + m_db_connection.reset(new pg_conn_t{conninfo}); + m_db_connection->exec("SET synchronous_commit TO off"); +} + +void flex_table_t::start(std::string const &conninfo) +{ + if (m_db_connection) { + throw std::runtime_error(name() + " cannot start, its already started"); + } + + connect(conninfo); + + m_db_connection->exec("SET client_min_messages = WARNING"); + + if (!m_append) { + m_db_connection->exec( + "DROP TABLE IF EXISTS {} CASCADE"_format(full_name())); + } + + // These _tmp tables can be left behind if we run out of disk space. + m_db_connection->exec("DROP TABLE IF EXISTS {}"_format(full_tmp_name())); + m_db_connection->exec("RESET client_min_messages"); + + if (!m_append) { + if (schema() != "public") { + m_db_connection->exec( + "CREATE SCHEMA IF NOT EXISTS \"{}\""_format(schema())); + } + m_db_connection->exec(build_sql_create_table(false)); + } else { + //check the columns against those in the existing table + auto const res = m_db_connection->query( + PGRES_TUPLES_OK, "SELECT * FROM {} LIMIT 0"_format(full_name())); + + for (auto const &column : m_columns) { + if (res.get_column_number(column.name()) < 0) { + fmt::print(stderr, "Adding new column '{}' to '{}'\n", + column.name(), name()); + m_db_connection->exec( + "ALTER TABLE {} ADD COLUMN \"{}\" {}"_format( + full_name(), column.name(), + column.sql_type_name(m_srid))); + } + // Note: we do not verify the type or delete unused columns + } + + //TODO: change the type of the geometry column if needed - this can only change to a more permissive type + } + + prepare(); +} + +void flex_table_t::stop(bool updateable) +{ + m_copy_mgr.sync(); + + if (m_append) { + teardown(); + return; + } + + std::time_t const start = std::time(nullptr); + + if (has_geom_column()) { + fmt::print(stderr, "Clustering table '{}' by geometry...\n", name()); + + // Notices about invalid geometries are expected and can be ignored + // because they say nothing about the validity of the geometry in OSM. + m_db_connection->exec("SET client_min_messages = WARNING"); + + m_db_connection->exec(build_sql_create_table(true)); + + std::string sql = "INSERT INTO {} SELECT * FROM {}"_format( + full_tmp_name(), full_name()); + + if (m_srid != 4326) { + // libosmium assures validity of geometries in 4326. + // Transformation to another projection could make the geometry + // invalid. Therefore add a filter to drop those. + sql += " WHERE ST_IsValid(\"{}\")"_format(geom_column().name()); + } + + auto const res = m_db_connection->query( + PGRES_TUPLES_OK, + "SELECT regexp_split_to_table(postgis_lib_version(), '\\.')"); + auto const postgis_major = std::stoi(res.get_value_as_string(0, 0)); + auto const postgis_minor = std::stoi(res.get_value_as_string(1, 0)); + + sql += " ORDER BY "; + if (postgis_major == 2 && postgis_minor < 4) { + fmt::print(stderr, "Using GeoHash for clustering\n"); + if (m_srid == 4326) { + sql += "ST_GeoHash({},10)"_format(geom_column().name()); + } else { + sql += + "ST_GeoHash(ST_Transform(ST_Envelope({}),4326),10)"_format( + geom_column().name()); + } + sql += " COLLATE \"C\""; + } else { + fmt::print(stderr, "Using native order for clustering\n"); + // Since Postgis 2.4 the order function for geometries gives + // useful results. + sql += geom_column().name(); + } + + m_db_connection->exec(sql); + + m_db_connection->exec("DROP TABLE {}"_format(full_name())); + m_db_connection->exec( + "ALTER TABLE {} RENAME TO \"{}\""_format(full_tmp_name(), name())); + + fmt::print(stderr, "Creating geometry index on table '{}'...\n", + name()); + + // Use fillfactor 100 for un-updateable imports + m_db_connection->exec( + "CREATE INDEX ON {} USING GIST (\"{}\") {} {}"_format( + full_name(), geom_column().name(), + (updateable ? "" : "WITH (FILLFACTOR=100)"), + tablespace_clause(m_index_tablespace))); + } + + if (updateable && has_id_column()) { + fmt::print(stderr, "Creating id index on table '{}'...\n", name()); + create_id_index(); + + if (m_srid != 4326 && has_geom_column()) { + m_db_connection->exec( + "CREATE OR REPLACE FUNCTION {}_osm2pgsql_valid()\n" + "RETURNS TRIGGER AS $$\n" + "BEGIN\n" + " IF ST_IsValid(NEW.{}) THEN \n" + " RETURN NEW;\n" + " END IF;\n" + " RETURN NULL;\n" + "END;" + "$$ LANGUAGE plpgsql;"_format(name(), geom_column().name())); + + m_db_connection->exec( + "CREATE TRIGGER \"{0}_osm2pgsql_valid\"" + " BEFORE INSERT OR UPDATE" + " ON {1}" + " FOR EACH ROW EXECUTE PROCEDURE" + " {0}_osm2pgsql_valid();"_format(name(), full_name())); + } + } + + fmt::print(stderr, "Analyzing table '{}'...\n", name()); + m_db_connection->exec("ANALYZE \"{}\""_format(name())); + + std::time_t const end = std::time(nullptr); + fmt::print(stderr, "All postprocessing on table '{}' done in {}s.\n", + name(), end - start); + + teardown(); +} + +void flex_table_t::create_id_index() +{ + m_db_connection->exec("CREATE INDEX ON {} USING BTREE ({}) {}"_format( + full_name(), id_column_names(), tablespace_clause(m_index_tablespace))); +} + +void flex_table_t::delete_rows_with(osmium::item_type type, osmid_t id) +{ + m_copy_mgr.new_line(m_target); + + // If the table id type is some specific type, we don't care about the + // type of the individual object, because they all will be the same. + if (m_id_type != osmium::item_type::undefined) { + type = osmium::item_type::undefined; + } + m_copy_mgr.delete_object(type_to_char(type)[0], id); +} + +pg_result_t flex_table_t::get_geom_by_id(osmium::item_type type, + osmid_t id) const +{ + assert(has_geom_column()); + assert(m_db_connection); + std::string const id_str = fmt::to_string(id); + if (has_multicolumn_id_index()) { + char const *param_values[] = {type_to_char(type), id_str.c_str()}; + return m_db_connection->exec_prepared("get_wkb", 2, param_values); + } + char const *param_values[] = {id_str.c_str()}; + return m_db_connection->exec_prepared("get_wkb", 1, param_values); +} diff --git a/src/flex-table.hpp b/src/flex-table.hpp new file mode 100644 index 000000000..b8eee170b --- /dev/null +++ b/src/flex-table.hpp @@ -0,0 +1,222 @@ +#ifndef OSM2PGSQL_FLEX_TABLE_HPP +#define OSM2PGSQL_FLEX_TABLE_HPP + +#include "db-copy-mgr.hpp" +#include "flex-table-column.hpp" +#include "pgsql.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +/** + * An output table (in the SQL sense) for the flex backend. + */ +class flex_table_t +{ +public: + flex_table_t(std::string const &name, int srid, + std::shared_ptr const ©_thread, + bool append) + : m_name(name), m_srid(srid), m_copy_mgr(copy_thread), m_append(append) + {} + + std::string const &name() const noexcept { return m_name; } + + std::string const &schema() const noexcept { return m_schema; } + + std::string const &data_tablespace() const noexcept + { + return m_data_tablespace; + } + + std::string const &index_tablespace() const noexcept + { + return m_index_tablespace; + } + + void set_schema(std::string const &schema) noexcept { m_schema = schema; } + + void set_data_tablespace(std::string const &tablespace) noexcept + { + m_data_tablespace = tablespace; + } + + void set_index_tablespace(std::string const &tablespace) noexcept + { + m_index_tablespace = tablespace; + } + + osmium::item_type id_type() const noexcept { return m_id_type; } + + void set_id_type(osmium::item_type type) noexcept { m_id_type = type; } + + bool has_id_column() const noexcept + { + if (m_columns.empty()) { + return false; + } + return (m_columns[0].type() == table_column_type::id_type) || + (m_columns[0].type() == table_column_type::id_num); + } + + std::size_t num_columns() const noexcept { return m_columns.size(); } + + std::vector::const_iterator begin() const noexcept + { + return m_columns.begin(); + } + + std::vector::const_iterator end() const noexcept + { + return m_columns.end(); + } + + bool has_geom_column() const noexcept + { + return m_geom_column != std::numeric_limits::max(); + } + + // XXX should we allow several geometry columns? + flex_table_column_t const &geom_column() const noexcept + { + assert(has_geom_column()); + return m_columns[m_geom_column]; + } + + int srid() const noexcept { return m_srid; } + + std::string build_sql_prepare_get_wkb() const; + + std::string build_sql_create_table(bool final_table) const; + + std::string build_sql_column_list() const; + + /// Does this table take objects of the specified type? + bool matches_type(osmium::item_type type) const noexcept + { + if (m_id_type == osmium::item_type::undefined) { + return true; + } + if (type == m_id_type) { + return true; + } + return m_id_type == osmium::item_type::area && + type != osmium::item_type::node; + } + + /// Map way/node/relation ID to id value used in database table column + osmid_t map_id(osmium::item_type type, osmid_t id) const noexcept + { + if (m_id_type == osmium::item_type::area && + type == osmium::item_type::relation) { + return -id; + } + return id; + } + + flex_table_column_t &add_column(std::string const &name, + std::string const &type); + + void init() + { + auto const columns = build_sql_column_list(); + auto const id_columns = id_column_names(); + m_target = std::make_shared( + name().c_str(), id_columns.c_str(), columns.c_str()); + } + + void connect(std::string const &conninfo); + + void commit() { m_copy_mgr.sync(); } + + void new_line() { m_copy_mgr.new_line(m_target); } + + void teardown() { m_db_connection.reset(); } + + void prepare() + { + assert(m_db_connection); + if (has_id_column()) { + m_db_connection->exec(build_sql_prepare_get_wkb()); + } + } + + void start(std::string const &conninfo); + + void stop(bool updateable); + + void create_id_index(); + + void delete_rows_with(osmium::item_type type, osmid_t id); + + pg_result_t get_geom_by_id(osmium::item_type type, osmid_t id) const; + + db_copy_mgr_t *copy_mgr() noexcept + { + return &m_copy_mgr; + } + +private: + bool has_multicolumn_id_index() const noexcept; + std::string id_column_names() const; + std::string full_name() const; + std::string full_tmp_name() const; + + /// The name of the table + std::string m_name; + + /// The schema this table is in + std::string m_schema{"public"}; + + /// The table space used for this table (empty for default tablespace) + std::string m_data_tablespace; + + /** + * The table space used for indexes on this table (empty for default + * tablespace) + */ + std::string m_index_tablespace; + + /** + * The columns in this table (The first zero, one or two columns are always + * the id columns). + */ + std::vector m_columns; + + /// Index of the geometry column in m_columns. Default means no geometry. + std::size_t m_geom_column = std::numeric_limits::max(); + + /** + * Type of Id stored in this table (node, way, relation, area, or + * undefined for any type). + */ + osmium::item_type m_id_type = osmium::item_type::undefined; + + /// The SRID all geometries in this table use. + int m_srid; + + /** + * The copy manager responsible for sending data through the COPY mechanism + * to the database server. + */ + db_copy_mgr_t m_copy_mgr; + + /// The connection to the database server. + std::unique_ptr m_db_connection; + + std::shared_ptr m_target; + + /// Are we in append mode? + bool m_append; + +}; // class flex_table_t + +char const *type_to_char(osmium::item_type type) noexcept; + +#endif // OSM2PGSQL_FLEX_TABLE_HPP diff --git a/src/geom-transform.cpp b/src/geom-transform.cpp new file mode 100644 index 000000000..56b88b081 --- /dev/null +++ b/src/geom-transform.cpp @@ -0,0 +1,166 @@ + +#include "geom-transform.hpp" + +#include + +#include +#include + +bool geom_transform_point_t::is_compatible_with( + table_column_type geom_type) const noexcept +{ + return geom_type == table_column_type::point || + geom_type == table_column_type::geometry; +} + +geom::osmium_builder_t::wkbs_t +geom_transform_point_t::run(geom::osmium_builder_t *builder, + osmium::Node const &node) const +{ + assert(builder); + + return {builder->get_wkb_node(node.location())}; +} + +bool geom_transform_line_t::set_param(char const *name, lua_State *lua_state) +{ + if (std::strcmp(name, "split_at") != 0) { + return false; + } + + if (lua_type(lua_state, -1) != LUA_TNUMBER) { + throw std::runtime_error{ + "The 'split_at' field in a geometry transformation " + "description must be a number"}; + } + m_split_at = lua_tonumber(lua_state, -1); + + return true; +} + +bool geom_transform_line_t::is_compatible_with( + table_column_type geom_type) const noexcept +{ + return geom_type == table_column_type::linestring || + geom_type == table_column_type::multilinestring || + geom_type == table_column_type::geometry; +} + +geom::osmium_builder_t::wkbs_t +geom_transform_line_t::run(geom::osmium_builder_t *builder, + osmium::Way *way) const +{ + assert(builder); + assert(way); + + return builder->get_wkb_line(way->nodes(), m_split_at); +} + +geom::osmium_builder_t::wkbs_t +geom_transform_line_t::run(geom::osmium_builder_t *builder, + osmium::Relation const & /*relation*/, + osmium::memory::Buffer const &buffer) const +{ + assert(builder); + + return builder->get_wkb_multiline(buffer, m_split_at); +} + +bool geom_transform_area_t::set_param(char const *name, lua_State *lua_state) +{ + if (std::strcmp(name, "multi") != 0) { + return false; + } + + if (lua_type(lua_state, -1) != LUA_TBOOLEAN) { + throw std::runtime_error{ + "The 'multi' field in a geometry transformation " + "description must be a boolean"}; + } + m_multi = lua_toboolean(lua_state, -1); + + return true; +} + +bool geom_transform_area_t::is_compatible_with( + table_column_type geom_type) const noexcept +{ + if (m_multi) { + return geom_type == table_column_type::multipolygon || + geom_type == table_column_type::geometry; + } + + return geom_type == table_column_type::polygon || + geom_type == table_column_type::geometry; +} + +geom::osmium_builder_t::wkbs_t +geom_transform_area_t::run(geom::osmium_builder_t *builder, + osmium::Way *way) const +{ + assert(builder); + assert(way); + + if (!way->is_closed()) { + return {}; + } + + return {builder->get_wkb_polygon(*way)}; +} + +geom::osmium_builder_t::wkbs_t +geom_transform_area_t::run(geom::osmium_builder_t *builder, + osmium::Relation const &relation, + osmium::memory::Buffer const &buffer) const +{ + assert(builder); + + return builder->get_wkb_multipolygon(relation, buffer, m_multi); +} + +std::unique_ptr create_geom_transform(char const *type) +{ + if (std::strcmp(type, "point") == 0) { + return std::unique_ptr{new geom_transform_point_t{}}; + } + + if (std::strcmp(type, "line") == 0) { + return std::unique_ptr{new geom_transform_line_t{}}; + } + + if (std::strcmp(type, "area") == 0) { + return std::unique_ptr{new geom_transform_area_t{}}; + } + + throw std::runtime_error{ + "Unknown geometry transformation '{}'"_format(type)}; +} + +void init_geom_transform(geom_transform_t *transform, lua_State *lua_state) +{ + static bool show_warning = true; + + assert(transform); + assert(lua_state); + + lua_pushnil(lua_state); + while (lua_next(lua_state, -2) != 0) { + char const *const field = lua_tostring(lua_state, -2); + if (field == nullptr) { + throw std::runtime_error{"All fields in geometry transformation " + "description must have string keys"}; + } + + if (std::strcmp(field, "create") != 0) { + if (!transform->set_param(field, lua_state) && show_warning) { + fmt::print(stderr, + "\nWarning! Ignoring unknown field '{}' in geometry " + "transformation description.\n", + field); + show_warning = false; + } + } + + lua_pop(lua_state, 1); + } +} diff --git a/src/geom-transform.hpp b/src/geom-transform.hpp new file mode 100644 index 000000000..3445798b7 --- /dev/null +++ b/src/geom-transform.hpp @@ -0,0 +1,107 @@ +#ifndef OSM2PGSQL_GEOM_TRANSFORM_HPP +#define OSM2PGSQL_GEOM_TRANSFORM_HPP + +#include "flex-table-column.hpp" +#include "lua.hpp" +#include "osmium-builder.hpp" + +#include + +#include + +/** + * Abstract base class for geometry transformations from nodes, ways, or + * relations to simple feature type geometries. + */ +class geom_transform_t +{ +public: + virtual ~geom_transform_t() = default; + + virtual bool set_param(char const * /*name*/, lua_State * /*lua_state*/) + { + return false; + } + + virtual bool is_compatible_with(table_column_type geom_type) const + noexcept = 0; + + virtual geom::osmium_builder_t::wkbs_t + run(geom::osmium_builder_t * /*builder*/, + osmium::Node const & /*node*/) const + { + return {}; + } + + virtual geom::osmium_builder_t::wkbs_t + run(geom::osmium_builder_t * /*builder*/, osmium::Way * /*way*/) const + { + return {}; + } + + virtual geom::osmium_builder_t::wkbs_t + run(geom::osmium_builder_t * /*builder*/, + osmium::Relation const & /*relation*/, + osmium::memory::Buffer const & /*buffer*/) const + { + return {}; + } + +}; // class geom_transform_t + +class geom_transform_point_t : public geom_transform_t +{ +public: + bool is_compatible_with(table_column_type geom_type) const + noexcept override; + + geom::osmium_builder_t::wkbs_t run(geom::osmium_builder_t *builder, + osmium::Node const &node) const override; + +}; // class geom_transform_point_t + +class geom_transform_line_t : public geom_transform_t +{ +public: + bool set_param(char const *name, lua_State *lua_state) override; + + bool is_compatible_with(table_column_type geom_type) const + noexcept override; + + geom::osmium_builder_t::wkbs_t run(geom::osmium_builder_t *builder, + osmium::Way *way) const override; + + geom::osmium_builder_t::wkbs_t + run(geom::osmium_builder_t *builder, osmium::Relation const &relation, + osmium::memory::Buffer const &buffer) const override; + +private: + double m_split_at = 0.0; + +}; // class geom_transform_line_t + +class geom_transform_area_t : public geom_transform_t +{ +public: + bool set_param(char const *name, lua_State *lua_state) override; + + bool is_compatible_with(table_column_type geom_type) const + noexcept override; + + geom::osmium_builder_t::wkbs_t run(geom::osmium_builder_t *builder, + osmium::Way *way) const override; + + geom::osmium_builder_t::wkbs_t + run(geom::osmium_builder_t *builder, osmium::Relation const &relation, + osmium::memory::Buffer const &buffer) const override; + +private: + bool m_multi = false; + +}; // class geom_transform_area_t + +std::unique_ptr create_geom_transform(char const *type); + +void init_geom_transform(geom_transform_t *transform, lua_State *lua_state); + +#endif // OSM2PGSQL_GEOM_TRANSFORM_HPP diff --git a/src/init.lua b/src/init.lua new file mode 100644 index 000000000..33969453a --- /dev/null +++ b/src/init.lua @@ -0,0 +1,94 @@ +-- +-- This will be compiled into osm2pgsql and run as initialization code in +-- the flex backend. +-- + +local _define_table_impl = function (_type, _name, _columns) + return osm2pgsql.define_table{ + name = _name, + ids = { type = _type, id_column = _type .. '_id' }, + columns = _columns, + } +end + +function osm2pgsql.define_node_table(_name, _columns) + return _define_table_impl('node', _name, _columns) +end + +function osm2pgsql.define_way_table(_name, _columns) + return _define_table_impl('way', _name, _columns) +end + +function osm2pgsql.define_relation_table(_name, _columns) + return _define_table_impl('relation', _name, _columns) +end + +function osm2pgsql.define_area_table(_name, _columns) + return _define_table_impl('area', _name, _columns) +end + +function osm2pgsql.mark_way(id) + return osm2pgsql.mark('w', id) +end + +function osm2pgsql.mark_relation(id) + return osm2pgsql.mark('r', id) +end + +function osm2pgsql.make_clean_tags_func(keys) + local keys_to_delete = {} + local prefixes_to_delete = {} + + for _, k in ipairs(keys) do + if k:sub(-1) == '*' then + prefixes_to_delete[#prefixes_to_delete + 1] = k:sub(1, -2) + else + keys_to_delete[#keys_to_delete + 1] = k + end + end + + return function(tags) + for _, k in ipairs(keys_to_delete) do + tags[k] = nil + end + + for tag, _ in pairs(tags) do + for _, k in ipairs(prefixes_to_delete) do + if tag:sub(1, k:len()) == k then + tags[tag] = nil + break + end + end + end + + return next(tags) == nil + end +end + +-- This will be the metatable for the OSM objects given to the process callback +-- functions. +local inner_metatable = { + __index = function(table, key) + if key == 'version' or key == 'timestamp' or + key == 'changeset' or key == 'uid' or key == 'user' then + return nil + end + error("unknown field '" .. key .. "'", 2) + end +} + +object_metatable = { + __index = { + grab_tag = function(data, tag) + if not tag then + error("Missing tag key", 2) + end + local v = data.tags[tag] + data.tags[tag] = nil + return v + end + } +} + +setmetatable(object_metatable.__index, inner_metatable) + diff --git a/src/lua-init.cpp.in b/src/lua-init.cpp.in new file mode 100644 index 000000000..be4503603 --- /dev/null +++ b/src/lua-init.cpp.in @@ -0,0 +1,6 @@ +char const *lua_init() noexcept +{ + return R"MaGiC( +@LUA_INIT_CODE@ +)MaGiC"; +} diff --git a/src/lua-init.hpp b/src/lua-init.hpp new file mode 100644 index 000000000..ea7edb64d --- /dev/null +++ b/src/lua-init.hpp @@ -0,0 +1,6 @@ +#ifndef OSM2PGSQL_LUA_INIT_HPP +#define OSM2PGSQL_LUA_INIT_HPP + +char const *lua_init() noexcept; + +#endif // OSM2PGSQL_LUA_INIT_HPP diff --git a/src/lua-utils.cpp b/src/lua-utils.cpp new file mode 100644 index 000000000..f70df05fb --- /dev/null +++ b/src/lua-utils.cpp @@ -0,0 +1,116 @@ +#include "lua-utils.hpp" +#include "format.hpp" + +extern "C" +{ +#include +} + +#include + +// The lua_getextraspace() function is only available from Lua 5.3. For +// earlier versions we fall back to storing the context pointer in the +// Lua registry which is somewhat more effort so will be slower. +#if LUA_VERSION_NUM >= 503 + +void luaX_set_context(lua_State *lua_state, void *ptr) noexcept +{ + assert(lua_state); + assert(ptr); + *static_cast(lua_getextraspace(lua_state)) = ptr; +} + +void *luaX_get_context(lua_State *lua_state) noexcept +{ + assert(lua_state); + return *static_cast(lua_getextraspace(lua_state)); +} + +#else + +// Unique key for lua registry +static char const *osm2pgsql_output_flex = "osm2pgsql_output_flex"; + +void luaX_set_context(lua_State *lua_state, void *ptr) noexcept +{ + assert(lua_state); + assert(ptr); + lua_pushlightuserdata(lua_state, (void *)osm2pgsql_output_flex); + lua_pushlightuserdata(lua_state, ptr); + lua_settable(lua_state, LUA_REGISTRYINDEX); +} + +void *luaX_get_context(lua_State *lua_state) noexcept +{ + assert(lua_state); + lua_pushlightuserdata(lua_state, (void *)osm2pgsql_output_flex); + lua_gettable(lua_state, LUA_REGISTRYINDEX); + auto *const ptr = lua_touserdata(lua_state, -1); + assert(ptr); + lua_pop(lua_state, 1); + return ptr; +} + +#endif + +void luaX_add_table_str(lua_State *lua_state, char const *key, + char const *value) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushstring(lua_state, value); + lua_rawset(lua_state, -3); +} + +void luaX_add_table_str(lua_State *lua_state, char const *key, + char const *value, std::size_t size) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushlstring(lua_state, value, size); + lua_rawset(lua_state, -3); +} + +void luaX_add_table_int(lua_State *lua_state, char const *key, + int64_t value) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushinteger(lua_state, value); + lua_rawset(lua_state, -3); +} + +void luaX_add_table_num(lua_State *lua_state, char const *key, + double value) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushnumber(lua_state, value); + lua_rawset(lua_state, -3); +} + +void luaX_add_table_bool(lua_State *lua_state, char const *key, + bool value) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushboolean(lua_state, value); + lua_rawset(lua_state, -3); +} + +void luaX_add_table_func(lua_State *lua_state, char const *key, + lua_CFunction func) noexcept +{ + lua_pushstring(lua_state, key); + lua_pushcfunction(lua_state, func); + lua_rawset(lua_state, -3); +} + +char const *luaX_get_table_string(lua_State *lua_state, char const *key, + int table_index, char const *error_msg) +{ + assert(lua_state); + assert(key); + assert(error_msg); + lua_getfield(lua_state, table_index, key); + if (!lua_isstring(lua_state, -1)) { + throw std::runtime_error{ + "{} must contain a '{}' string field"_format(error_msg, key)}; + } + return lua_tostring(lua_state, -1); +} diff --git a/src/lua-utils.hpp b/src/lua-utils.hpp new file mode 100644 index 000000000..2b5bcea71 --- /dev/null +++ b/src/lua-utils.hpp @@ -0,0 +1,49 @@ +#ifndef OSM2PGSQL_FLEX_LUA_HPP +#define OSM2PGSQL_FLEX_LUA_HPP + +// This file contains helper functions for talking to Lua. It is used from +// the flex output backend. All functions start with "luaX_". + +extern "C" +{ +#include +} + +#include +#include + +void luaX_set_context(lua_State *lua_state, void *ptr) noexcept; +void *luaX_get_context(lua_State *lua_state) noexcept; + +void luaX_add_table_str(lua_State *lua_state, char const *key, + char const *value) noexcept; +void luaX_add_table_str(lua_State *lua_state, char const *key, + char const *value, std::size_t size) noexcept; +void luaX_add_table_int(lua_State *lua_state, char const *key, + int64_t value) noexcept; +void luaX_add_table_num(lua_State *lua_state, char const *key, + double value) noexcept; +void luaX_add_table_bool(lua_State *lua_state, char const *key, + bool value) noexcept; +void luaX_add_table_func(lua_State *lua_state, char const *key, + lua_CFunction func) noexcept; + +template +void luaX_add_table_array(lua_State *lua_state, char const *key, + COLLECTION const &collection, FUNC &&func) +{ + lua_pushstring(lua_state, key); + lua_createtable(lua_state, (int)collection.size(), 0); + int n = 0; + for (auto const &member : collection) { + lua_pushinteger(lua_state, ++n); + std::forward(func)(member); + lua_rawset(lua_state, -3); + } + lua_rawset(lua_state, -3); +} + +char const *luaX_get_table_string(lua_State *lua_state, char const *key, + int table_index, char const *error_msg); + +#endif // OSM2PGSQL_FLEX_LUA_HPP diff --git a/src/osmdata.cpp b/src/osmdata.cpp index 364cf6c6b..77bec1f33 100644 --- a/src/osmdata.cpp +++ b/src/osmdata.cpp @@ -442,6 +442,10 @@ void osmdata_t::stop() const } } + for (auto &out : outs) { + out->stage2_proc(); + } + // Clustering, index creation, and cleanup. // All the intensive parts of this are long-running PostgreSQL commands { diff --git a/src/output-flex.cpp b/src/output-flex.cpp new file mode 100644 index 000000000..154475dcd --- /dev/null +++ b/src/output-flex.cpp @@ -0,0 +1,1344 @@ + +#include "config.h" + +#include "expire-tiles.hpp" +#include "geom-transform.hpp" +#include "lua-init.hpp" +#include "lua-utils.hpp" +#include "middle.hpp" +#include "options.hpp" +#include "osmtypes.hpp" +#include "output-flex.hpp" +#include "pgsql.hpp" +#include "reprojection.hpp" +#include "taginfo-impl.hpp" +#include "version.hpp" +#include "wkb.hpp" + +#include + +extern "C" +{ +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Mutex used to coordinate access to Lua code +static std::mutex lua_mutex; + +// Lua can't call functions on C++ objects directly. This macro defines simple +// C "trampoline" functions which are called from Lua which get the current +// context (the output_flex_t object) and call the respective function on the +// context object. +#define TRAMPOLINE(func_name, lua_name) \ + static int lua_trampoline_##func_name(lua_State *lua_state) noexcept \ + { \ + try { \ + return static_cast(luaX_get_context(lua_state)) \ + ->func_name(); \ + } catch (std::exception const &e) { \ + return luaL_error(lua_state, "Error in '" #lua_name "': %s\n", \ + e.what()); \ + } catch (...) { \ + return luaL_error(lua_state, \ + "Unknown error in '" #lua_name "'.\n"); \ + } \ + } + +TRAMPOLINE(app_define_table, define_table) +TRAMPOLINE(app_mark, mark) +TRAMPOLINE(app_get_bbox, get_bbox) +TRAMPOLINE(table_name, name) +TRAMPOLINE(table_schema, schema) +TRAMPOLINE(table_add_row, add_row) +TRAMPOLINE(table_columns, columns) +TRAMPOLINE(table_tostring, __tostring) + +static char const osm2pgsql_table_name[] = "osm2pgsql.table"; +static char const osm2pgsql_object_metatable[] = "osm2pgsql.object_metatable"; + +static void push_osm_object_to_lua_stack(lua_State *lua_state, + osmium::OSMObject const &object, + bool with_attributes) +{ + assert(lua_state); + + /** + * Table will have 7 fields (id, version, timestamp, changeset, uid, user, + * tags) for all object types plus 2 (is_closed, nodes) for ways or 1 + * (members) for relations. + */ + constexpr int const max_table_size = 9; + + lua_createtable(lua_state, 0, max_table_size); + + luaX_add_table_int(lua_state, "id", object.id()); + + if (with_attributes) { + if (object.version() != 0U) { + luaX_add_table_int(lua_state, "version", object.version()); + } else { + // This is a workaround, because the middle will give us the + // attributes as pseudo-tags. + char const *const val = object.tags()["osm_version"]; + if (val) { + luaX_add_table_int(lua_state, "version", + osmium::string_to_object_version(val)); + } + } + + if (object.timestamp().valid()) { + luaX_add_table_int(lua_state, "timestamp", + object.timestamp().seconds_since_epoch()); + } else { + // This is a workaround, because the middle will give us the + // attributes as pseudo-tags. + char const *const val = object.tags()["osm_timestamp"]; + if (val) { + auto const timestamp = osmium::Timestamp{val}; + luaX_add_table_int(lua_state, "timestamp", + timestamp.seconds_since_epoch()); + } + } + + if (object.changeset() != 0U) { + luaX_add_table_int(lua_state, "changeset", object.changeset()); + } else { + char const *const val = object.tags()["osm_changeset"]; + // This is a workaround, because the middle will give us the + // attributes as pseudo-tags. + if (val) { + luaX_add_table_int(lua_state, "changeset", + osmium::string_to_changeset_id(val)); + } + } + + if (object.uid() != 0U) { + luaX_add_table_int(lua_state, "uid", object.uid()); + } else { + // This is a workaround, because the middle will give us the + // attributes as pseudo-tags. + char const *const val = object.tags()["osm_uid"]; + if (val) { + luaX_add_table_int(lua_state, "uid", + osmium::string_to_uid(val)); + } + } + + if (object.user()[0] != '\0') { + luaX_add_table_str(lua_state, "user", object.user()); + } else { + // This is a workaround, because the middle will give us the + // attributes as pseudo-tags. + char const *const val = object.tags()["osm_user"]; + if (val) { + luaX_add_table_str(lua_state, "user", val); + } + } + } + + if (object.type() == osmium::item_type::way) { + auto const &way = static_cast(object); + luaX_add_table_bool(lua_state, "is_closed", way.is_closed()); + luaX_add_table_array(lua_state, "nodes", way.nodes(), + [&](osmium::NodeRef const &wn) { + lua_pushinteger(lua_state, wn.ref()); + }); + } else if (object.type() == osmium::item_type::relation) { + auto const &relation = static_cast(object); + luaX_add_table_array( + lua_state, "members", relation.members(), + [&](osmium::RelationMember const &member) { + lua_createtable(lua_state, 0, 3); + std::array tmp{"x"}; + tmp[0] = osmium::item_type_to_char(member.type()); + luaX_add_table_str(lua_state, "type", &tmp[0]); + luaX_add_table_int(lua_state, "ref", member.ref()); + luaX_add_table_str(lua_state, "role", member.role()); + }); + } + + lua_pushliteral(lua_state, "tags"); + lua_createtable(lua_state, 0, (int)object.tags().size()); + for (auto const &tag : object.tags()) { + luaX_add_table_str(lua_state, tag.key(), tag.value()); + } + lua_rawset(lua_state, -3); + + // Set the metatable of this object + lua_pushlightuserdata(lua_state, (void *)osm2pgsql_object_metatable); + lua_gettable(lua_state, LUA_REGISTRYINDEX); + lua_setmetatable(lua_state, -2); +} + +static bool str2bool(char const *str) noexcept +{ + return (std::strcmp(str, "yes") == 0) || (std::strcmp(str, "true") == 0); +} + +static int str2direction(char const *str) noexcept +{ + if ((std::strcmp(str, "yes") == 0) || (std::strcmp(str, "true") == 0) || + (std::strcmp(str, "1") == 0)) { + return 1; + } + + if (std::strcmp(str, "-1") == 0) { + return -1; + } + + return 0; +} + +static int sgn(int val) noexcept +{ + if (val > 0) { + return 1; + } + if (val < 0) { + return -1; + } + return 0; +} + +void output_flex_t::write_column( + db_copy_mgr_t *copy_mgr, + flex_table_column_t const &column) +{ + lua_getfield(lua_state(), -1, column.name().c_str()); + int const ltype = lua_type(lua_state(), -1); + + // A Lua nil value is always translated to a database NULL + if (ltype == LUA_TNIL) { + copy_mgr->add_null_column(); + lua_pop(lua_state(), 1); + return; + } + + if ((column.type() == table_column_type::sql) || + (column.type() == table_column_type::text)) { + auto const *const str = lua_tolstring(lua_state(), -1, nullptr); + if (!str) { + throw std::runtime_error{"Invalid type '{}' for text column"_format( + lua_typename(lua_state(), ltype))}; + } + copy_mgr->add_column(str); + } else if (column.type() == table_column_type::boolean) { + switch (ltype) { + case LUA_TBOOLEAN: + copy_mgr->add_column(lua_toboolean(lua_state(), -1) != 0); + break; + case LUA_TNUMBER: + copy_mgr->add_column(lua_tointeger(lua_state(), -1) != 0); + break; + case LUA_TSTRING: + copy_mgr->add_column( + str2bool(lua_tolstring(lua_state(), -1, nullptr))); + break; + default: + throw std::runtime_error{ + "Invalid type '{}' for boolean column"_format( + lua_typename(lua_state(), ltype))}; + } + } else if (column.type() == table_column_type::int2) { + // cast here is on okay, because the database column is only 16bit + copy_mgr->add_column((int16_t)lua_tointeger(lua_state(), -1)); + } else if (column.type() == table_column_type::int4) { + // cast here is on okay, because the database column is only 32bit + copy_mgr->add_column((int32_t)lua_tointeger(lua_state(), -1)); + } else if (column.type() == table_column_type::int8) { + copy_mgr->add_column(lua_tointeger(lua_state(), -1)); + } else if (column.type() == table_column_type::real) { + copy_mgr->add_column(lua_tonumber(lua_state(), -1)); + } else if (column.type() == table_column_type::hstore) { + if (ltype == LUA_TTABLE) { + copy_mgr->new_hash(); + + lua_pushnil(lua_state()); + while (lua_next(lua_state(), -2) != 0) { + char const *const key = lua_tostring(lua_state(), -2); + char const *const val = lua_tostring(lua_state(), -1); + if (key == nullptr) { + int const ltype_key = lua_type(lua_state(), -2); + throw std::runtime_error{ + "NULL key for hstore. Possibly this is due to" + "an incorrect data type '{}' as key."_format( + lua_typename(lua_state(), ltype_key))}; + } + if (val == nullptr) { + int const ltype_value = lua_type(lua_state(), -1); + throw std::runtime_error{ + "NULL value for hstore. Possibly this is due to" + "an incorrect data type '{}' for key '{}'."_format( + lua_typename(lua_state(), ltype_value), key)}; + } + copy_mgr->add_hash_elem(key, val); + lua_pop(lua_state(), 1); + } + + copy_mgr->finish_hash(); + } else { + throw std::runtime_error{ + "Invalid type '{}' for hstore column"_format( + lua_typename(lua_state(), ltype))}; + } + } else if (column.type() == table_column_type::direction) { + switch (ltype) { + case LUA_TBOOLEAN: + copy_mgr->add_column(lua_toboolean(lua_state(), -1)); + break; + case LUA_TNUMBER: + copy_mgr->add_column(sgn(lua_tointeger(lua_state(), -1))); + break; + case LUA_TSTRING: + copy_mgr->add_column( + str2direction(lua_tolstring(lua_state(), -1, nullptr))); + break; + default: + throw std::runtime_error{ + "Invalid type '{}' for direction column"_format( + lua_typename(lua_state(), ltype))}; + } + } else { + throw std::runtime_error{ + "Column type {} not implemented"_format(column.type())}; + } + + lua_pop(lua_state(), 1); +} + +void output_flex_t::write_row(flex_table_t *table, osmium::item_type id_type, + osmid_t id, std::string const &geom) +{ + assert(table); + table->new_line(); + auto *copy_mgr = table->copy_mgr(); + + for (auto const &column : *table) { + if (column.type() == table_column_type::id_type) { + copy_mgr->add_column(type_to_char(id_type)); + } else if (column.type() == table_column_type::id_num) { + copy_mgr->add_column(id); + } else if (column.is_geometry_column()) { + copy_mgr->add_hex_geom(geom); + } else if (column.type() == table_column_type::area) { + if (geom.empty()) { + copy_mgr->add_null_column(); + } else { + double const area = + get_options()->reproject_area + ? ewkb::parser_t(geom).get_area( + get_options()->projection.get()) + : ewkb::parser_t(geom) + .get_area(); + copy_mgr->add_column(area); + } + } else { + write_column(copy_mgr, column); + } + } + + copy_mgr->finish_line(); +} + +int output_flex_t::app_mark() +{ + char const *type_name = luaL_checkstring(lua_state(), 1); + if (!type_name) { + return 0; + } + + osmium::object_id_type const id = luaL_checkinteger(lua_state(), 2); + + if (type_name[0] == 'w') { + m_stage2_ways_tracker->mark(id); + } else if (type_name[0] == 'r') { + m_stage2_rels_tracker->mark(id); + } + + return 0; +} + +// Gets all way nodes from the middle the first time this is called. +std::size_t output_flex_t::get_way_nodes() +{ + assert(m_context_way); + if (m_num_way_nodes == std::numeric_limits::max()) { + m_num_way_nodes = m_mid->nodes_get_list(&m_context_way->nodes()); + } + + return m_num_way_nodes; +} + +int output_flex_t::app_get_bbox() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for get_box()"}; + } + + if (m_context_node) { + lua_pushnumber(lua_state(), m_context_node->location().lon()); + lua_pushnumber(lua_state(), m_context_node->location().lat()); + lua_pushnumber(lua_state(), m_context_node->location().lon()); + lua_pushnumber(lua_state(), m_context_node->location().lat()); + return 4; + } + + if (m_context_way) { + get_way_nodes(); + auto const bbox = m_context_way->envelope(); + if (bbox.valid()) { + lua_pushnumber(lua_state(), bbox.bottom_left().lon()); + lua_pushnumber(lua_state(), bbox.bottom_left().lat()); + lua_pushnumber(lua_state(), bbox.top_right().lon()); + lua_pushnumber(lua_state(), bbox.top_right().lat()); + return 4; + } + } + + return 0; +} + +static void check_name(std::string const &name, char const *in) +{ + auto const pos = name.find_first_of("\"',.;$%&/()<>{}=?^*#"); + + if (pos == std::string::npos) { + return; + } + + throw std::runtime_error{ + "Special characters are not allowed in {} names: '{}'"_format(in, + name)}; +} + +flex_table_t &output_flex_t::create_flex_table() +{ + std::string const table_name = + luaX_get_table_string(lua_state(), "name", -1, "The table"); + + check_name(table_name, "table"); + + auto const it = std::find_if(m_tables->cbegin(), m_tables->cend(), + [&table_name](flex_table_t const &table) { + return table.name() == table_name; + }); + if (it != m_tables->cend()) { + throw std::runtime_error{ + "Table with that name already exists: '{}'"_format(table_name)}; + } + + m_tables->emplace_back(table_name, get_options()->projection->target_srs(), + m_copy_thread, get_options()->append); + auto &new_table = m_tables->back(); + + lua_pop(lua_state(), 1); + + // optional "schema" field + lua_getfield(lua_state(), -1, "schema"); + if (lua_isstring(lua_state(), -1)) { + std::string const schema = lua_tostring(lua_state(), -1); + check_name(schema, "schame"); + new_table.set_schema(schema); + } + lua_pop(lua_state(), 1); + + // optional "data_tablespace" field + lua_getfield(lua_state(), -1, "data_tablespace"); + if (lua_isstring(lua_state(), -1)) { + std::string const schema = lua_tostring(lua_state(), -1); + check_name(schema, "data_tablespace"); + new_table.set_data_tablespace(schema); + } + lua_pop(lua_state(), 1); + + // optional "index_tablespace" field + lua_getfield(lua_state(), -1, "index_tablespace"); + if (lua_isstring(lua_state(), -1)) { + std::string const schema = lua_tostring(lua_state(), -1); + check_name(schema, "index_tablespace"); + new_table.set_index_tablespace(schema); + } + lua_pop(lua_state(), 1); + + return new_table; +} + +void output_flex_t::setup_id_columns(flex_table_t *table) +{ + assert(table); + lua_getfield(lua_state(), -1, "ids"); + if (lua_type(lua_state(), -1) != LUA_TTABLE) { + fmt::print(stderr, + "WARNING! Table '{}' doesn't have an 'ids' column. Updates" + " and expire will not work!\n", + table->name()); + lua_pop(lua_state(), 1); // ids + return; + } + + std::string const type{ + luaX_get_table_string(lua_state(), "type", -1, "The ids field")}; + + if (type == "node") { + table->set_id_type(osmium::item_type::node); + } else if (type == "way") { + table->set_id_type(osmium::item_type::way); + } else if (type == "relation") { + table->set_id_type(osmium::item_type::relation); + } else if (type == "area") { + table->set_id_type(osmium::item_type::area); + } else if (type == "any") { + std::string type_column_name{"osm_type"}; + lua_getfield(lua_state(), -1, "type_column"); + if (lua_isstring(lua_state(), -1)) { + type_column_name = lua_tolstring(lua_state(), -1, nullptr); + check_name(type_column_name, "column"); + } + lua_pop(lua_state(), 1); // type_column + auto &column = table->add_column(type_column_name, "id_type"); + column.set_not_null_constraint(); + table->set_id_type(osmium::item_type::undefined); + } else { + throw std::runtime_error{"Unknown ids type: " + type}; + } + + std::string const name = + luaX_get_table_string(lua_state(), "id_column", -2, "The ids field"); + check_name(name, "column"); + + auto &column = table->add_column(name, "id_num"); + column.set_not_null_constraint(); + lua_pop(lua_state(), 3); // id_column, type, ids +} + +void output_flex_t::setup_flex_table_columns(flex_table_t *table) +{ + assert(table); + lua_getfield(lua_state(), -1, "columns"); + if (lua_type(lua_state(), -1) != LUA_TTABLE) { + throw std::runtime_error{ + "No columns defined for table '{}'"_format(table->name())}; + } + + std::size_t num_columns = 0; + lua_pushnil(lua_state()); + while (lua_next(lua_state(), -2) != 0) { + if (!lua_isnumber(lua_state(), -2)) { + throw std::runtime_error{ + "The 'columns' field must contain an array"}; + } + if (!lua_istable(lua_state(), -1)) { + throw std::runtime_error{ + "The entries in the 'columns' array must be tables"}; + } + + char const *const type = + luaX_get_table_string(lua_state(), "type", -1, "Column entry"); + char const *const name = + luaX_get_table_string(lua_state(), "column", -2, "Column entry"); + check_name(name, "column"); + + table->add_column(name, type); + + lua_pop(lua_state(), 3); // column, type, table + ++num_columns; + } + + if (num_columns == 0) { + throw std::runtime_error{ + "No columns defined for table '{}'"_format(table->name())}; + } +} + +int output_flex_t::app_define_table() +{ + luaL_checktype(lua_state(), 1, LUA_TTABLE); + + auto &new_table = create_flex_table(); + setup_id_columns(&new_table); + setup_flex_table_columns(&new_table); + + lua_pushlightuserdata(lua_state(), (void *)(m_tables->size())); + luaL_getmetatable(lua_state(), osm2pgsql_table_name); + lua_setmetatable(lua_state(), -2); + + return 1; +} + +// Check function parameters of all osm2pgsql.table functions and return the +// flex table this function is on. +flex_table_t &output_flex_t::table_func_params(int n) +{ + if (lua_gettop(lua_state()) != n) { + throw std::runtime_error{"Need {} parameter(s)"_format(n)}; + } + + void *user_data = lua_touserdata(lua_state(), 1); + if (user_data == nullptr || !lua_getmetatable(lua_state(), 1)) { + throw std::runtime_error{ + "first parameter must be of type osm2pgsql.table"}; + } + + luaL_getmetatable(lua_state(), osm2pgsql_table_name); + if (!lua_rawequal(lua_state(), -1, -2)) { + throw std::runtime_error{ + "first parameter must be of type osm2pgsql.table"}; + } + lua_pop(lua_state(), 2); + + auto &table = m_tables->at(reinterpret_cast(user_data) - 1); + lua_remove(lua_state(), 1); + return table; +} + +int output_flex_t::table_tostring() +{ + auto const &table = table_func_params(1); + + std::string const str{"osm2pgsql.table[{}]"_format(table.name())}; + lua_pushstring(lua_state(), str.c_str()); + + return 1; +} + +int output_flex_t::table_add_row() +{ + auto &table = table_func_params(2); + luaL_checktype(lua_state(), 1, LUA_TTABLE); + + if (m_context_node) { + if (!table.matches_type(osmium::item_type::node)) { + throw std::runtime_error{ + "Trying to add node to table '{}'"_format(table.name())}; + } + add_row(&table, *m_context_node); + } else if (m_context_way) { + if (!table.matches_type(osmium::item_type::way)) { + throw std::runtime_error{ + "Trying to add way to table '{}'"_format(table.name())}; + } + if (m_in_stage2) { + delete_from_table(&table, osmium::item_type::way, + m_context_way->id()); + } + add_row(&table, *m_context_way); + } else if (m_context_relation) { + if (!table.matches_type(osmium::item_type::relation)) { + throw std::runtime_error{ + "Trying to add relation to table '{}'"_format(table.name())}; + } + if (m_in_stage2) { + delete_from_table(&table, osmium::item_type::relation, + m_context_relation->id()); + } + add_row(&table, *m_context_relation); + } else { + throw std::runtime_error{"The add_row() function can only be called " + "from inside a process function"}; + } + + return 0; +} + +int output_flex_t::table_columns() +{ + auto const &table = table_func_params(1); + + lua_createtable(lua_state(), (int)table.num_columns(), 0); + + int n = 0; + for (auto const &column : table) { + lua_pushinteger(lua_state(), ++n); + lua_newtable(lua_state()); + + luaX_add_table_str(lua_state(), "name", column.name().c_str()); + luaX_add_table_str(lua_state(), "type", column.type_name().c_str()); + luaX_add_table_str(lua_state(), "sql_type", + column.sql_type_name(table.srid()).c_str()); + luaX_add_table_str(lua_state(), "sql_modifiers", + column.sql_modifiers().c_str()); + + lua_rawset(lua_state(), -3); + } + return 1; +} + +int output_flex_t::table_name() +{ + auto const &table = table_func_params(1); + lua_pushstring(lua_state(), table.name().c_str()); + return 1; +} + +int output_flex_t::table_schema() +{ + auto const &table = table_func_params(1); + lua_pushstring(lua_state(), table.schema().c_str()); + return 1; +} + +static std::unique_ptr +get_transform(lua_State *lua_state, flex_table_column_t const &column) +{ + assert(lua_state); + assert(lua_gettop(lua_state) == 1); + + std::unique_ptr transform{}; + + lua_getfield(lua_state, -1, column.name().c_str()); + int const ltype = lua_type(lua_state, -1); + if (ltype != LUA_TTABLE) { + lua_pop(lua_state, 1); // geom field + return transform; + } + + lua_getfield(lua_state, -1, "create"); + char const *create_type = lua_tostring(lua_state, -1); + if (create_type == nullptr) { + throw std::runtime_error{ + "Missing geometry transformation for column '{}'"_format( + column.name())}; + } + + transform = create_geom_transform(create_type); + lua_pop(lua_state, 1); // 'create' field + init_geom_transform(transform.get(), lua_state); + if (!transform->is_compatible_with(column.type())) { + throw std::runtime_error{ + "Geometry transformation is not compatible " + "with column type '{}'"_format(column.type_name())}; + } + + lua_pop(lua_state, 1); // geom field + + return transform; +} + +static geom_transform_t const * +get_default_transform(flex_table_column_t const &column, + osmium::item_type object_type) +{ + static geom_transform_point_t const default_transform_node_to_point{}; + static geom_transform_line_t const default_transform_way_to_line{}; + static geom_transform_area_t const default_transform_way_to_area{}; + + switch (object_type) { + case osmium::item_type::node: + if (column.type() == table_column_type::point) { + return &default_transform_node_to_point; + } + break; + case osmium::item_type::way: + if (column.type() == table_column_type::linestring) { + return &default_transform_way_to_line; + } + if (column.type() == table_column_type::polygon) { + return &default_transform_way_to_area; + } + break; + default: + break; + } + + throw std::runtime_error{ + "Missing geometry transformation for column '{}'"_format( + column.name())}; +} + +geom::osmium_builder_t::wkbs_t +output_flex_t::run_transform(geom_transform_t const *transform, + osmium::Node const &node) +{ + return transform->run(&m_builder, node); +} + +geom::osmium_builder_t::wkbs_t +output_flex_t::run_transform(geom_transform_t const *transform, + osmium::Way const & /*way*/) +{ + if (get_way_nodes() <= 1U) { + return {}; + } + return transform->run(&m_builder, m_context_way); +} + +geom::osmium_builder_t::wkbs_t +output_flex_t::run_transform(geom_transform_t const *transform, + osmium::Relation const &relation) +{ + m_buffer.clear(); + auto const num_ways = + m_mid->rel_way_members_get(relation, nullptr, m_buffer); + + if (num_ways == 0) { + return {}; + } + + for (auto &way : m_buffer.select()) { + m_mid->nodes_get_list(&(way.nodes())); + } + + return transform->run(&m_builder, relation, m_buffer); +} + +template +void output_flex_t::add_row(flex_table_t *table, OBJECT const &object) +{ + assert(table); + + osmid_t const id = table->map_id(object.type(), object.id()); + + if (!table->has_geom_column()) { + write_row(table, object.type(), id, ""); + return; + } + + auto const geom_transform = + get_transform(lua_state(), table->geom_column()); + assert(lua_gettop(lua_state()) == 1); + + geom_transform_t const *transform = geom_transform.get(); + + if (!transform) { + transform = get_default_transform(table->geom_column(), object.type()); + } + + auto const wkbs = run_transform(transform, object); + for (auto const &wkb : wkbs) { + m_expire.from_wkb(wkb.c_str(), id); + write_row(table, object.type(), id, wkb); + } +} + +void output_flex_t::call_process_function(int index, + osmium::OSMObject const &object) +{ + std::lock_guard guard{lua_mutex}; + + assert(lua_gettop(lua_state()) == 3); + + lua_pushvalue(lua_state(), index); // the function to call + push_osm_object_to_lua_stack( + lua_state(), object, + get_options()->extra_attributes); // the single argument + + luaX_set_context(lua_state(), this); + if (lua_pcall(lua_state(), 1, 0, 0)) { + throw std::runtime_error{"Failed to execute lua processing function:" + " {}"_format(lua_tostring(lua_state(), -1))}; + } +} + +void output_flex_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, + std::size_t output_id, std::size_t &added) +{ + fmt::print(stderr, "enqueue_ways: {}/{}\n", id, output_id); + osmid_t const prev = m_ways_pending_tracker.last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.push(pending_job_t(id, output_id)); + } + // already done the job + return; + } + + //make sure we get the one passed in + if (!m_ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) { + job_queue.push(pending_job_t(id, output_id)); + added++; + } + + //grab the first one or bail if its not valid + osmid_t popped = m_ways_pending_tracker.pop_mark(); + if (!id_tracker::is_valid(popped)) { + return; + } + + //get all the ones up to the id that was passed in + while (popped < id) { + if (!m_ways_done_tracker->is_marked(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + popped = m_ways_pending_tracker.pop_mark(); + } + + //make sure to get this one as well and move to the next + if (popped > id) { + if (!m_ways_done_tracker->is_marked(popped) && + id_tracker::is_valid(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + } +} + +void output_flex_t::pending_way(osmid_t id, int exists) +{ + if (!m_has_process_way) { + return; + } + + m_buffer.clear(); + if (!m_mid->ways_get(id, m_buffer)) { + return; + } + + if (exists) { + way_delete(id); + auto const rel_ids = m_mid->relations_using_way(id); + for (auto const id : rel_ids) { + m_rels_pending_tracker.mark(id); + } + } + + auto &way = m_buffer.get(0); + + m_context_way = &way; + call_process_function(2, way); + m_context_way = nullptr; + m_num_way_nodes = std::numeric_limits::max(); + m_buffer.clear(); +} + +void output_flex_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, + std::size_t output_id, std::size_t &added) +{ + if (!m_has_process_relation) { + return; + } + + osmid_t const prev = m_rels_pending_tracker.last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.emplace(id, output_id); + } + // already done the job + return; + } + + //make sure we get the one passed in + if (id_tracker::is_valid(id)) { + job_queue.emplace(id, output_id); + ++added; + } + + //grab the first one or bail if its not valid + osmid_t popped = m_rels_pending_tracker.pop_mark(); + if (!id_tracker::is_valid(popped)) { + return; + } + + //get all the ones up to the id that was passed in + while (popped < id) { + job_queue.emplace(popped, output_id); + ++added; + popped = m_rels_pending_tracker.pop_mark(); + } + + //make sure to get this one as well and move to the next + if (popped > id) { + if (id_tracker::is_valid(popped)) { + job_queue.emplace(popped, output_id); + ++added; + } + } +} + +void output_flex_t::pending_relation(osmid_t id, int exists) +{ + if (!m_has_process_relation) { + return; + } + + // Try to fetch the relation from the DB + // Note that we cannot use the global buffer here because + // we cannot keep a reference to the relation and an autogrow buffer + // might be relocated when more data is added. + if (!m_mid->relations_get(id, m_rels_buffer)) { + return; + } + + // If the flag says this object may exist already, delete it first. + if (exists) { + delete_from_tables(osmium::item_type::relation, id); + } + + auto const &relation = m_rels_buffer.get(0); + + m_context_relation = &relation; + call_process_function(3, relation); + m_context_relation = nullptr; + m_rels_buffer.clear(); +} + +void output_flex_t::commit() +{ + for (auto &table : *m_tables) { + table.commit(); + } +} + +void output_flex_t::stop(osmium::thread::Pool *pool) +{ + for (auto &table : *m_tables) { + pool->submit( + [&]() { table.stop(m_options.slim & !m_options.droptemp); }); + } + + if (m_options.expire_tiles_zoom_min > 0) { + m_expire.output_and_destroy(m_options.expire_tiles_filename.c_str(), + m_options.expire_tiles_zoom_min); + } +} + +void output_flex_t::node_add(osmium::Node const &node) +{ + if (!m_has_process_node) { + return; + } + + m_context_node = &node; + call_process_function(1, node); + m_context_node = nullptr; +} + +void output_flex_t::way_add(osmium::Way *way) +{ + assert(way); + + if (!m_has_process_way) { + return; + } + + m_context_way = way; + call_process_function(2, *way); + m_context_way = nullptr; + m_num_way_nodes = std::numeric_limits::max(); +} + +void output_flex_t::relation_add(osmium::Relation const &relation) +{ + if (!m_has_process_relation) { + return; + } + + m_context_relation = &relation; + call_process_function(3, relation); + m_context_relation = nullptr; +} + +void output_flex_t::delete_from_table(flex_table_t *table, + osmium::item_type type, osmid_t osm_id) +{ + assert(table); + auto const id = table->map_id(type, osm_id); + auto const result = table->get_geom_by_id(type, id); + if (m_expire.from_result(result, id) != 0) { + table->delete_rows_with(type, id); + } +} + +void output_flex_t::delete_from_tables(osmium::item_type type, osmid_t osm_id) +{ + for (auto &table : *m_tables) { + if (table.matches_type(type)) { + delete_from_table(&table, type, osm_id); + } + } +} + +/* Delete is easy, just remove all traces of this object. We don't need to + * worry about finding objects that depend on it, since the same diff must + * contain the change for that also. */ +void output_flex_t::node_delete(osmid_t osm_id) +{ + delete_from_tables(osmium::item_type::node, osm_id); +} + +void output_flex_t::way_delete(osmid_t osm_id) +{ + delete_from_tables(osmium::item_type::way, osm_id); +} + +void output_flex_t::relation_delete(osmid_t osm_id) +{ + delete_from_tables(osmium::item_type::relation, osm_id); +} + +void output_flex_t::node_modify(osmium::Node const &node) +{ + node_delete(node.id()); + node_add(node); +} + +void output_flex_t::way_modify(osmium::Way *way) +{ + way_delete(way->id()); + way_add(way); +} + +void output_flex_t::relation_modify(osmium::Relation const &rel) +{ + relation_delete(rel.id()); + relation_add(rel); +} + +void output_flex_t::init_clone() +{ + for (auto &table : *m_tables) { + table.connect(m_options.database_options.conninfo()); + table.prepare(); + } +} + +void output_flex_t::start() +{ + for (auto &table : *m_tables) { + table.start(m_options.database_options.conninfo()); + } +} + +std::shared_ptr +output_flex_t::clone(std::shared_ptr const &mid, + std::shared_ptr const ©_thread) const +{ + return std::shared_ptr(new output_flex_t{ + mid, *get_options(), copy_thread, true, m_lua_state, m_has_process_node, + m_has_process_way, m_has_process_relation, m_tables, + m_stage2_ways_tracker, m_stage2_rels_tracker}); +} + +output_flex_t::output_flex_t( + std::shared_ptr const &mid, options_t const &o, + std::shared_ptr const ©_thread, bool is_clone, + std::shared_ptr lua_state, bool has_process_node, + bool has_process_way, bool has_process_relation, + std::shared_ptr> tables, + std::shared_ptr ways_tracker, + std::shared_ptr rels_tracker) +: output_t(mid, o), m_tables(std::move(tables)), + m_ways_done_tracker(new id_tracker{}), + m_stage2_ways_tracker(std::move(ways_tracker)), + m_stage2_rels_tracker(std::move(rels_tracker)), m_copy_thread(copy_thread), + m_lua_state(std::move(lua_state)), m_builder(o.projection), + m_expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), + m_buffer(32768, osmium::memory::Buffer::auto_grow::yes), + m_rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_has_process_node(has_process_node), m_has_process_way(has_process_way), + m_has_process_relation(has_process_relation) +{ + assert(copy_thread); + + if (!is_clone) { + init_lua(m_options.style); + } + + for (auto &table : *m_tables) { + table.init(); + } + + if (is_clone) { + init_clone(); + } +} + +static bool prepare_process_function(lua_State *lua_state, char const *name) +{ + lua_getfield(lua_state, 1, name); + + if (lua_type(lua_state, -1) == LUA_TFUNCTION) { + return true; + } + + if (lua_type(lua_state, -1) == LUA_TNIL) { + return false; + } + + throw std::runtime_error{"osm2pgsql.{} must be a function"_format(name)}; +} + +void output_flex_t::init_lua(std::string const &filename) +{ + m_lua_state.reset(luaL_newstate(), + [](lua_State *state) { lua_close(state); }); + + // Set up global lua libs + luaL_openlibs(lua_state()); + + // Set up global "osm2pgsql" object + lua_newtable(lua_state()); + + luaX_add_table_str(lua_state(), "version", get_osm2pgsql_short_version()); + luaX_add_table_int(lua_state(), "srid", + get_options()->projection->target_srs()); + luaX_add_table_str(lua_state(), "mode", + m_options.append ? "append" : "create"); + luaX_add_table_int(lua_state(), "stage", 1); + + luaX_add_table_func(lua_state(), "define_table", + lua_trampoline_app_define_table); + luaX_add_table_func(lua_state(), "mark", lua_trampoline_app_mark); + + lua_setglobal(lua_state(), "osm2pgsql"); + + // Define "osmpgsql.table" metatable + if (luaL_newmetatable(lua_state(), osm2pgsql_table_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed"}; + } + lua_pushvalue(lua_state(), -1); + lua_setfield(lua_state(), -2, "__index"); + luaX_add_table_func(lua_state(), "__tostring", + lua_trampoline_table_tostring); + luaX_add_table_func(lua_state(), "add_row", lua_trampoline_table_add_row); + luaX_add_table_func(lua_state(), "name", lua_trampoline_table_name); + luaX_add_table_func(lua_state(), "schema", lua_trampoline_table_schema); + luaX_add_table_func(lua_state(), "columns", lua_trampoline_table_columns); + + // Clean up stack + lua_settop(lua_state(), 0); + + // Load compiled in init.lua + if (luaL_dostring(lua_state(), lua_init())) { + throw std::runtime_error{"Internal error in Lua setup: {}"_format( + lua_tostring(lua_state(), -1))}; + } + + // Store the "get_bbox" in the "object_metatable". + lua_getglobal(lua_state(), "object_metatable"); + lua_getfield(lua_state(), -1, "__index"); + luaX_add_table_func(lua_state(), "get_bbox", lua_trampoline_app_get_bbox); + lua_settop(lua_state(), 0); + + // Store the global object "object_metatable" defined in the init.lua + // script in the registry and then remove the global object. It will + // later be used as metatable for OSM objects. + lua_pushlightuserdata(lua_state(), (void *)osm2pgsql_object_metatable); + lua_getglobal(lua_state(), "object_metatable"); + lua_settable(lua_state(), LUA_REGISTRYINDEX); + lua_pushnil(lua_state()); + lua_setglobal(lua_state(), "object_metatable"); + + // Load user config file + luaX_set_context(lua_state(), this); + if (luaL_dofile(lua_state(), filename.c_str())) { + throw std::runtime_error{"Error loading lua config: {}"_format( + lua_tostring(lua_state(), -1))}; + } + + // Check whether the process_* functions are available and store them on + // the Lua stack for fast access later + lua_getglobal(lua_state(), "osm2pgsql"); + m_has_process_node = prepare_process_function(lua_state(), "process_node"); + m_has_process_way = prepare_process_function(lua_state(), "process_way"); + m_has_process_relation = + prepare_process_function(lua_state(), "process_relation"); + + lua_remove(lua_state(), 1); // global "osm2pgsql" +} + +std::size_t output_flex_t::pending_count() const +{ + return m_ways_pending_tracker.size() + m_rels_pending_tracker.size(); +} + +void output_flex_t::stage2_proc() +{ + bool const has_marked_ways = m_stage2_ways_tracker->size() > 0; + bool const has_marked_rels = m_stage2_rels_tracker->size() > 0; + + if (!has_marked_ways && !has_marked_rels) { + fmt::print(stderr, "Skipping stage 2 (no marked objects).\n"); + return; + } + + fmt::print(stderr, "Entering stage 2...\n"); + m_in_stage2 = true; + + if (!m_options.append) { + fmt::print(stderr, "Creating id indexes...\n"); + const std::time_t start_time = std::time(nullptr); + + for (auto &table : *m_tables) { + if ((has_marked_ways && + table.matches_type(osmium::item_type::way)) || + (has_marked_rels && + table.matches_type(osmium::item_type::relation))) { + fmt::print(stderr, " Creating id index on table '{}'...\n", + table.name()); + table.create_id_index(); + } + } + + fmt::print(stderr, " Creating id indexes took {} seconds\n"_format( + std::time(nullptr) - start_time)); + } + + lua_gc(lua_state(), LUA_GCCOLLECT, 0); + fmt::print(stderr, "Lua program uses {} MBytes\n", + lua_gc(lua_state(), LUA_GCCOUNT, 0) / 1024); + + lua_getglobal(lua_state(), "osm2pgsql"); + lua_pushinteger(lua_state(), 2); + lua_setfield(lua_state(), -2, "stage"); + lua_pop(lua_state(), 1); // osm2pgsql + + osmid_t id; + + fmt::print(stderr, "Entering stage 2 processing of {} ways...\n"_format( + m_stage2_ways_tracker->size())); + + while (id_tracker::is_valid((id = m_stage2_ways_tracker->pop_mark()))) { + m_buffer.clear(); + if (!m_mid->ways_get(id, m_buffer)) { + continue; + } + auto &way = m_buffer.get(0); + way_add(&way); + } + + fmt::print(stderr, + "Entering stage 2 processing of {} relations...\n"_format( + m_stage2_rels_tracker->size())); + + while (id_tracker::is_valid((id = m_stage2_rels_tracker->pop_mark()))) { + m_rels_buffer.clear(); + if (!m_mid->relations_get(id, m_rels_buffer)) { + continue; + } + auto const &relation = m_rels_buffer.get(0); + relation_add(relation); + } +} + +void output_flex_t::merge_pending_relations(output_t *other) +{ + auto *opgsql = dynamic_cast(other); + if (opgsql) { + osmid_t id; + while (id_tracker::is_valid( + (id = opgsql->m_rels_pending_tracker.pop_mark()))) { + m_rels_pending_tracker.mark(id); + } + } +} + +void output_flex_t::merge_expire_trees(output_t *other) +{ + auto *opgsql = dynamic_cast(other); + if (opgsql) { + m_expire.merge_and_destroy(opgsql->m_expire); + } +} diff --git a/src/output-flex.hpp b/src/output-flex.hpp new file mode 100644 index 000000000..7d16209f3 --- /dev/null +++ b/src/output-flex.hpp @@ -0,0 +1,169 @@ +#ifndef OSM2PGSQL_OUTPUT_FLEX_HPP +#define OSM2PGSQL_OUTPUT_FLEX_HPP + +#include "db-copy.hpp" +#include "expire-tiles.hpp" +#include "flex-table-column.hpp" +#include "flex-table.hpp" +#include "format.hpp" +#include "geom-transform.hpp" +#include "id-tracker.hpp" +#include "osmium-builder.hpp" +#include "output.hpp" +#include "table.hpp" +#include "tagtransform.hpp" + +#include + +extern "C" +{ +#include +} + +#include +#include +#include +#include +#include + +class output_flex_t : public output_t +{ +public: + output_flex_t(std::shared_ptr const &mid, + options_t const &options, + std::shared_ptr const ©_thread, + bool is_clone = false, + std::shared_ptr lua_state = nullptr, + bool has_process_node = false, bool has_process_way = false, + bool has_process_relation = false, + std::shared_ptr> tables = + std::make_shared>(), + std::shared_ptr ways_tracker = + std::make_shared(), + std::shared_ptr rels_tracker = + std::make_shared()); + + output_flex_t(output_flex_t const &) = delete; + output_flex_t &operator=(output_flex_t const &) = delete; + + output_flex_t(output_flex_t &&) = delete; + output_flex_t &operator=(output_flex_t &&) = delete; + + virtual ~output_flex_t() noexcept = default; + + std::shared_ptr + clone(std::shared_ptr const &mid, + std::shared_ptr const ©_thread) const override; + + void start() override; + void stop(osmium::thread::Pool *pool) override; + void commit() override; + + void stage2_proc() override; + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, + std::size_t output_id, std::size_t &added) override; + void pending_way(osmid_t id, int exists) override; + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, + std::size_t output_id, std::size_t &added) override; + void pending_relation(osmid_t id, int exists) override; + + void node_add(osmium::Node const &node) override; + void way_add(osmium::Way *way) override; + void relation_add(osmium::Relation const &rel) override; + + void node_modify(osmium::Node const &node) override; + void way_modify(osmium::Way *way) override; + void relation_modify(osmium::Relation const &rel) override; + + void node_delete(osmid_t id) override; + void way_delete(osmid_t id) override; + void relation_delete(osmid_t id) override; + + std::size_t pending_count() const override; + + void merge_pending_relations(output_t *other) override; + void merge_expire_trees(output_t *other) override; + + int app_define_table(); + int app_mark(); + int app_get_bbox(); + + int table_tostring(); + int table_add_row(); + int table_name(); + int table_schema(); + int table_columns(); + +private: + void init_clone(); + + void call_process_function(int index, osmium::OSMObject const &object); + + void init_lua(std::string const &filename); + + flex_table_t &create_flex_table(); + void setup_id_columns(flex_table_t *table); + void setup_flex_table_columns(flex_table_t *table); + + flex_table_t &table_func_params(int n); + + void write_column(db_copy_mgr_t *copy_mgr, + flex_table_column_t const &column); + void write_row(flex_table_t *table, osmium::item_type id_type, osmid_t id, + std::string const &geom); + + geom::osmium_builder_t::wkbs_t + run_transform(geom_transform_t const *transform, osmium::Node const &node); + + geom::osmium_builder_t::wkbs_t + run_transform(geom_transform_t const *transform, osmium::Way const &way); + + geom::osmium_builder_t::wkbs_t + run_transform(geom_transform_t const *transform, + osmium::Relation const &relation); + + template + void add_row(flex_table_t *table, OBJECT const &object); + + void delete_from_table(flex_table_t *table, osmium::item_type type, + osmid_t osm_id); + void delete_from_tables(osmium::item_type type, osmid_t osm_id); + + std::size_t get_way_nodes(); + + lua_State *lua_state() noexcept { return m_lua_state.get(); } + + std::shared_ptr> m_tables; + + id_tracker m_ways_pending_tracker; + id_tracker m_rels_pending_tracker; + std::shared_ptr m_ways_done_tracker; + + std::shared_ptr m_stage2_ways_tracker; + std::shared_ptr m_stage2_rels_tracker; + + std::shared_ptr m_copy_thread; + + std::shared_ptr m_lua_state; + + geom::osmium_builder_t m_builder; + expire_tiles m_expire; + + osmium::memory::Buffer m_buffer; + osmium::memory::Buffer m_rels_buffer; + + osmium::Node const *m_context_node = nullptr; + osmium::Way *m_context_way = nullptr; + osmium::Relation const *m_context_relation = nullptr; + + std::size_t m_num_way_nodes = std::numeric_limits::max(); + + bool m_in_stage2 = false; + bool m_has_process_node = false; + bool m_has_process_way = false; + bool m_has_process_relation = false; +}; + +#endif // OSM2PGSQL_OUTPUT_FLEX_HPP diff --git a/src/output.cpp b/src/output.cpp index 46c1d6a8c..5536c144d 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -1,3 +1,5 @@ +#include "config.h" + #include "db-copy.hpp" #include "format.hpp" #include "output-gazetteer.hpp" @@ -7,6 +9,13 @@ #include "output.hpp" #include "taginfo-impl.hpp" +#ifdef HAVE_LUA +# include "output-flex.hpp" +# define flex_backend "flex, " +#else +# define flex_backend "" +#endif + #include #include @@ -155,6 +164,12 @@ output_t::create_outputs(std::shared_ptr const &mid, outputs.push_back( std::make_shared(mid, options, copy_thread)); +#ifdef HAVE_LUA + } else if (options.output_backend == "flex") { + outputs.push_back( + std::make_shared(mid, options, copy_thread)); +#endif + } else if (options.output_backend == "gazetteer") { outputs.push_back( std::make_shared(mid, options, copy_thread)); @@ -168,8 +183,8 @@ output_t::create_outputs(std::shared_ptr const &mid, } else { throw std::runtime_error{ "Output backend `{}' not recognised. Should be one " - "of [pgsql, gazetteer, null, multi].\n"_format( - options.output_backend)}; + "of [pgsql, " flex_backend + "gazetteer, null, multi].\n"_format(options.output_backend)}; } return outputs; diff --git a/src/output.hpp b/src/output.hpp index ff35ee257..779a1bf1d 100644 --- a/src/output.hpp +++ b/src/output.hpp @@ -51,6 +51,8 @@ class output_t virtual void stop(osmium::thread::Pool *pool) = 0; virtual void commit() = 0; + virtual void stage2_proc() {} + virtual void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t &added) = 0; virtual void pending_way(osmid_t id, int exists) = 0; diff --git a/src/version.cpp.in b/src/version.cpp.in index 9899089bb..2034fbda0 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -3,3 +3,7 @@ char const *get_osm2pgsql_version() noexcept { return "@PACKAGE_VERSION@@VERSION_FROM_GIT@"; } +char const *get_osm2pgsql_short_version() noexcept { + return "@PACKAGE_VERSION@"; +} + diff --git a/src/version.hpp b/src/version.hpp index 42c94dcda..c6aeffeaa 100644 --- a/src/version.hpp +++ b/src/version.hpp @@ -2,5 +2,6 @@ #define OSM2PGSQL_VERSION_HPP char const *get_osm2pgsql_version() noexcept; +char const *get_osm2pgsql_short_version() noexcept; #endif // OSM2PGSQL_VERSION_HPP diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 90eee6aec..a5176f5fe 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,14 @@ if (HAVE_LUA) set_test(test-output-multi-line-storage) set_test(test-output-multi-poly-trivial) set_test(test-output-multi-tags) + set_test(test-output-flex) + set_test(test-output-flex-area) + set_test(test-output-flex-attr) + set_test(test-output-flex-extra) + set_test(test-output-flex-uni) + set_test(test-output-flex-tablespace LABELS Tablespace) + set_test(test-output-flex-update) + set_test(test-output-flex-validgeom) endif() diff --git a/tests/common-options.hpp b/tests/common-options.hpp index 73178e73a..a4aaa099a 100644 --- a/tests/common-options.hpp +++ b/tests/common-options.hpp @@ -56,6 +56,14 @@ class opt_t return *this; } + opt_t &flex(char const *style) + { + m_opt.output_backend = "flex"; + m_opt.style = TESTDATA_DIR; + m_opt.style += style; + return *this; + } + opt_t &flatnodes() { m_opt.flat_node_file = @@ -77,6 +85,12 @@ class opt_t return *this; } + opt_t &extra_attributes() noexcept + { + m_opt.extra_attributes = true; + return *this; + } + private: options_t m_opt; }; diff --git a/tests/data/test_output_flex.lua b/tests/data/test_output_flex.lua new file mode 100644 index 000000000..1fdd690d1 --- /dev/null +++ b/tests/data/test_output_flex.lua @@ -0,0 +1,139 @@ + +local tables = {} + +tables.point = osm2pgsql.define_node_table('osm2pgsql_test_point', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point' }, +}) + +tables.line = osm2pgsql.define_table{ + name = 'osm2pgsql_test_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'linestring' }, + } +} + +tables.polygon = osm2pgsql.define_table{ + name = 'osm2pgsql_test_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry' }, + { column = 'area', type = 'area' }, + } +} + +tables.route = osm2pgsql.define_table{ + name = 'osm2pgsql_test_route', + ids = { type = 'relation', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'multilinestring' }, + } +} + +function is_empty(some_table) + return next(some_table) == nil +end + +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + tags['source:name'] = nil +end + +function is_polygon(tags) + if tags.aeroway + or tags.amenity + or tags.area + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] + then + return true + else + return false + end +end + +function osm2pgsql.process_node(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + tables.point:add_row({ + tags = data.tags + }) +end + +function osm2pgsql.process_way(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + if is_polygon(data.tags) then + tables.polygon:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'area' } + }) + else + tables.line:add_row({ + tags = data.tags, + name = data.tags.name + }) + end +end + +function osm2pgsql.process_relation(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + if data.tags.type == 'multipolygon' or data.tags.type == 'boundary' then + tables.polygon:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'area', multi = false } + }) + return + end + + if data.tags.type == 'route' then + tables.route:add_row({ + tags = data.tags, + geom = { create = 'line' } + }) + end +end + diff --git a/tests/data/test_output_flex_attr.lua b/tests/data/test_output_flex_attr.lua new file mode 100644 index 000000000..3ee8741d7 --- /dev/null +++ b/tests/data/test_output_flex_attr.lua @@ -0,0 +1,20 @@ + +table = osm2pgsql.define_table{ + name = 'osm2pgsql_test_ways_attr', + ids = { type = 'way', id_column = 'way_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'version', type = 'int4' }, + { column = 'changeset', type = 'int4' }, + { column = 'timestamp', type = 'int4' }, + { column = 'uid', type = 'int4' }, + { column = 'user', type = 'text' }, + { column = 'geom', type = 'linestring' }, + } +} + +function osm2pgsql.process_way(data) + data.geom = { create = 'line' } + table:add_row(data) +end + diff --git a/tests/data/test_output_flex_extra.lua b/tests/data/test_output_flex_extra.lua new file mode 100644 index 000000000..76089a13d --- /dev/null +++ b/tests/data/test_output_flex_extra.lua @@ -0,0 +1,86 @@ + +local tables = {} + +tables.highways = osm2pgsql.define_table{ + name = 'osm2pgsql_test_highways', + ids = { type = 'way', id_column = 'way_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'refs', type = 'text' }, + { column = 'min_x', type = 'real' }, + { column = 'min_y', type = 'real' }, + { column = 'max_x', type = 'real' }, + { column = 'max_y', type = 'real' }, + { column = 'geom', type = 'linestring' }, + } +} + +tables.routes = osm2pgsql.define_table{ + name = 'osm2pgsql_test_routes', + ids = { type = 'relation', id_column = 'rel_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'members', type = 'text' }, + { column = 'geom', type = 'multilinestring' }, + } +} + +local by_way_id = {} + +function osm2pgsql.process_way(object) + if osm2pgsql.stage == 1 then + osm2pgsql.mark_way(object.id) + return + end + + local row = { + tags = object.tags, + geom = { create = 'line' } + } + + row.min_x, row.min_y, row.max_x, row.max_y = object:get_bbox() + + -- if there is any data from relations, add it in + local d = by_way_id[object.id] + if d then + local keys = {} + for k,v in pairs(d.refs) do + keys[#keys + 1] = k + end + + row.refs = table.concat(keys, ',') + -- row.rel_ids = '{' .. table.concat(d.ids, ',') .. '}' + end + + tables.highways:add_row(row) +end + +function osm2pgsql.process_relation(object) + if object.tags.type ~= 'route' then + return + end + + local mlist = {} + for i, member in ipairs(object.members) do + if member.type == 'w' then + osm2pgsql.mark_way(member.ref) + if not by_way_id[member.ref] then + by_way_id[member.ref] = { + ids = {}, + refs = {} + } + end + local d = by_way_id[member.ref] + table.insert(d.ids, object.id) + d.refs[object.tags.ref] = 1 + mlist[#mlist + 1] = member.ref + end + end + + tables.routes:add_row({ + tags = object.tags, + members = table.concat(mlist, ','), + geom = { create = 'line' } + }) +end + diff --git a/tests/data/test_output_flex_uni.lua b/tests/data/test_output_flex_uni.lua new file mode 100644 index 000000000..eb0d576a3 --- /dev/null +++ b/tests/data/test_output_flex_uni.lua @@ -0,0 +1,115 @@ + +local table = osm2pgsql.define_table{ + name = 'osm2pgsql_test_data', + ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry' }, + } +} + +function is_empty(some_table) + return next(some_table) == nil +end + +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + tags['source:name'] = nil +end + +function is_polygon(tags) + if tags.aeroway + or tags.amenity + or tags.area + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] + then + return true + else + return false + end +end + +function osm2pgsql.process_node(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + table:add_row({ + tags = data.tags, + geom = { create = 'point' } + }) +end + +function osm2pgsql.process_way(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + if is_polygon(data.tags) then + table:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'area' } + }) + else + table:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'line' } + }) + end +end + +function osm2pgsql.process_relation(data) + clean_tags(data.tags) + if is_empty(data.tags) then + return + end + + if data.tags.type == 'multipolygon' or data.tags.type == 'boundary' then + table:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'area', multi = false } + }) + return + end + + if data.tags.type == 'route' then + table:add_row({ + tags = data.tags, + name = data.tags.name, + geom = { create = 'line' } + }) + end +end + diff --git a/tests/test-output-flex-area.cpp b/tests/test-output-flex-area.cpp new file mode 100644 index 000000000..444de4558 --- /dev/null +++ b/tests/test-output-flex-area.cpp @@ -0,0 +1,59 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("default projection") +{ + options_t const options = + testing::opt_t().slim().flex("test_output_flex.lua"); + + REQUIRE_NOTHROW(db.run_file(options, "test_output_pgsql_area.osm")); + + auto conn = db.db().connect(); + + REQUIRE(2 == conn.get_count("osm2pgsql_test_polygon")); + conn.assert_double( + 1.23927e+10, + "SELECT area FROM osm2pgsql_test_polygon WHERE name='poly'"); + conn.assert_double( + 9.91828e+10, + "SELECT area FROM osm2pgsql_test_polygon WHERE name='multi'"); +} + +TEST_CASE("latlon projection") +{ + options_t const options = + testing::opt_t().slim().flex("test_output_flex.lua").srs(PROJ_LATLONG); + + REQUIRE_NOTHROW(db.run_file(options, "test_output_pgsql_area.osm")); + + auto conn = db.db().connect(); + + REQUIRE(2 == conn.get_count("osm2pgsql_test_polygon")); + conn.assert_double( + 1, "SELECT area FROM osm2pgsql_test_polygon WHERE name='poly'"); + conn.assert_double( + 8, "SELECT area FROM osm2pgsql_test_polygon WHERE name='multi'"); +} + +TEST_CASE("latlon projection with way area reprojection") +{ + options_t options = + testing::opt_t().slim().flex("test_output_flex.lua").srs(PROJ_LATLONG); + options.reproject_area = true; + + REQUIRE_NOTHROW(db.run_file(options, "test_output_pgsql_area.osm")); + + auto conn = db.db().connect(); + + REQUIRE(2 == conn.get_count("osm2pgsql_test_polygon")); + conn.assert_double( + 1.23927e+10, + "SELECT area FROM osm2pgsql_test_polygon WHERE name='poly'"); + conn.assert_double( + 9.91828e+10, + "SELECT area FROM osm2pgsql_test_polygon WHERE name='multi'"); +} diff --git a/tests/test-output-flex-attr.cpp b/tests/test-output-flex-attr.cpp new file mode 100644 index 000000000..bec5b6577 --- /dev/null +++ b/tests/test-output-flex-attr.cpp @@ -0,0 +1,82 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("no extra_attributes") +{ + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex_attr.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest " + "Thighway=primary Nn10,n11,n12\n")); + + auto conn = db.db().connect(); + + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", + "tags->'highway' = 'primary'")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "version = 1")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "changeset = 31")); + CHECK(0 == + conn.get_count("osm2pgsql_test_ways_attr", "timestamp = 1578832496")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "uid = 17")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "\"user\" = 'test'")); + + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_attr.lua"), + "n10 v2 dV x11.0 y11.0\n")); + + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", + "tags->'highway' = 'primary'")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "version = 1")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "changeset = 31")); + CHECK(0 == + conn.get_count("osm2pgsql_test_ways_attr", "timestamp = 1578832496")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "uid = 17")); + CHECK(0 == conn.get_count("osm2pgsql_test_ways_attr", "\"user\" = 'test'")); +} + +TEST_CASE("with extra_attributes") +{ + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().extra_attributes().slim().flex( + "test_output_flex_attr.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest " + "Thighway=primary Nn10,n11,n12\n")); + + auto conn = db.db().connect(); + + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "version = 1")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "changeset = 31")); + CHECK(1 == + conn.get_count("osm2pgsql_test_ways_attr", "timestamp = 1578832496")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "uid = 17")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "\"user\" = 'test'")); + + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().extra_attributes().slim().append().flex( + "test_output_flex_attr.lua"), + "n10 v2 dV x11.0 y11.0\n")); + + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "version = 1")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "changeset = 31")); + CHECK(1 == + conn.get_count("osm2pgsql_test_ways_attr", "timestamp = 1578832496")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "uid = 17")); + CHECK(1 == conn.get_count("osm2pgsql_test_ways_attr", "\"user\" = 'test'")); +} diff --git a/tests/test-output-flex-extra.cpp b/tests/test-output-flex-extra.cpp new file mode 100644 index 000000000..ed2330c60 --- /dev/null +++ b/tests/test-output-flex-extra.cpp @@ -0,0 +1,194 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("nodes and ways") +{ + testing::opt_t options = testing::opt_t() + .slim() + .flex("test_output_flex_extra.lua") + .srs(PROJ_LATLONG); + + REQUIRE_NOTHROW(db.run_import(options, + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "n14 v1 dV x10.3 y10.0\n" + "n15 v1 dV x10.4 y10.0\n" + "w20 v1 dV Thighway=primary Nn10,n11,n12\n" + "w21 v1 dV Thighway=secondary Nn12,n13\n")); + + auto conn = db.db().connect(); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways")); + CHECK(0 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + CHECK(1 == conn.get_count( + "osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10 10,10 10.2,10.2 10.2)'")); + CHECK(1 == + conn.get_count("osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10.2 10.2,10.2 10)'")); + + REQUIRE_NOTHROW(db.run_import(options.append(), "n11 v2 dV x10.0 y10.3\n")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + CHECK(1 == conn.get_count( + "osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10 10,10 10.3,10.2 10.2)'")); + CHECK(1 == + conn.get_count("osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10.2 10.2,10.2 10)'")); + + REQUIRE_NOTHROW(db.run_import( + options.append(), + "n12 v2 dD\n" + "w20 v2 dV Thighway=primary Nn10,n11\n" + "w21 v2 dV Thighway=secondary Nn13\n")); // single node in way! + + CHECK(1 == conn.get_count("osm2pgsql_test_highways")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(0 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10 10,10 10.3)'")); + + REQUIRE_NOTHROW(db.run_import( + options.append(), "w21 v2 dV Thighway=secondary Nn13,n14,n15\n")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10 10,10 10.3)'")); + CHECK(1 == conn.get_count( + "osm2pgsql_test_highways", + "ST_AsText(geom) = 'LINESTRING(10.2 10,10.3 10,10.4 10)'")); +} + +TEST_CASE("relation data on ways") +{ + testing::opt_t options = testing::opt_t() + .slim() + .flex("test_output_flex_extra.lua") + .srs(PROJ_LATLONG); + + // create database with three ways and a relation on two of them + REQUIRE_NOTHROW( + db.run_import(options, "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "n14 v1 dV x10.3 y10.0\n" + "n15 v1 dV x10.4 y10.0\n" + "w20 v1 dV Thighway=primary Nn10,n11,n12\n" + "w21 v1 dV Thighway=secondary Nn12,n13\n" + "w22 v1 dV Thighway=secondary Nn13,n14,n15\n" + "r30 v1 dV Ttype=route,ref=X11 Mw20@,w21@\n")); + + auto conn = db.db().connect(); + + CHECK(3 == conn.get_count("osm2pgsql_test_highways")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(2 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways", "refs = 'X11'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", "refs IS NULL")); + + CHECK(1 == conn.get_count( + "osm2pgsql_test_highways", + "abs(min_x - 10.0) < 0.01 AND abs(min_y - 10.0) < 0.01 AND " + "abs(max_x - 10.2) < 0.01 AND abs(max_y - 10.2) < 0.01")); + + CHECK(1 == conn.get_count("osm2pgsql_test_routes", "members = '20,21'")); + + // move node in way in the relation + REQUIRE_NOTHROW(db.run_import(options.append(), "n11 v2 dV x10.0 y10.1\n")); + + CHECK(3 == conn.get_count("osm2pgsql_test_highways")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(2 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways", "refs = 'X11'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", "refs IS NULL")); + + CHECK(1 == conn.get_count( + "osm2pgsql_test_highways", + "abs(min_x - 10.0) < 0.01 AND abs(min_y - 10.0) < 0.01 AND " + "abs(max_x - 10.2) < 0.01 AND abs(max_y - 10.2) < 0.01")); + + CHECK(1 == conn.get_count("osm2pgsql_test_routes", "members = '20,21'")); + + // add the third way to the relation + REQUIRE_NOTHROW(db.run_import( + options.append(), "r30 v2 dV Ttype=route,ref=X11 Mw20@,w21@,w22@\n")); + + CHECK(3 == conn.get_count("osm2pgsql_test_highways")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(2 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + + CHECK(3 == conn.get_count("osm2pgsql_test_highways", "refs = 'X11'")); + CHECK(0 == conn.get_count("osm2pgsql_test_highways", "refs IS NULL")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes", "members = '20,21,22'")); + + // remove the second way from the relation and delete it + REQUIRE_NOTHROW(db.run_import( + options.append(), "w21 v2 dD\n" + "r30 v3 dV Ttype=route,ref=X11 Mw20@,w22@\n")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways", "refs = 'X11'")); + CHECK(0 == conn.get_count("osm2pgsql_test_highways", "refs IS NULL")); + CHECK(1 == conn.get_count("osm2pgsql_test_routes", "members = '20,22'")); + + // delete the relation, leaving two ways + REQUIRE_NOTHROW(db.run_import(options.append(), "r30 v4 dD\n")); + + CHECK(2 == conn.get_count("osm2pgsql_test_highways")); + CHECK(0 == conn.get_count("osm2pgsql_test_routes")); + + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'primary'")); + CHECK(1 == conn.get_count("osm2pgsql_test_highways", + "tags->'highway' = 'secondary'")); + + CHECK(0 == conn.get_count("osm2pgsql_test_highways", "refs = 'X11'")); + CHECK(2 == conn.get_count("osm2pgsql_test_highways", "refs IS NULL")); +} diff --git a/tests/test-output-flex-tablespace.cpp b/tests/test-output-flex-tablespace.cpp new file mode 100644 index 000000000..194d98941 --- /dev/null +++ b/tests/test-output-flex-tablespace.cpp @@ -0,0 +1,36 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +static void require_tables(pg::conn_t const &conn) +{ + conn.require_has_table("osm2pgsql_test_point"); + conn.require_has_table("osm2pgsql_test_line"); + conn.require_has_table("osm2pgsql_test_polygon"); +} + +TEST_CASE("simple import with tables spaces") +{ + { + auto conn = db.db().connect(); + REQUIRE(1 == + conn.get_count("pg_tablespace", "spcname = 'tablespacetest'")); + } + + options_t options = testing::opt_t().slim().flex("test_output_flex.lua"); + options.tblsslim_index = "tablespacetest"; + options.tblsslim_data = "tablespacetest"; + + REQUIRE_NOTHROW(db.run_file(options, "liechtenstein-2013-08-03.osm.pbf")); + + auto conn = db.db().connect(); + require_tables(conn); + + REQUIRE(1362 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(2932 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(4136 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(35 == conn.get_count("osm2pgsql_test_route")); +} diff --git a/tests/test-output-flex-uni.cpp b/tests/test-output-flex-uni.cpp new file mode 100644 index 000000000..1fa360c5b --- /dev/null +++ b/tests/test-output-flex-uni.cpp @@ -0,0 +1,248 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("updating a node") +{ + // import a node... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex_uni.lua"), + "n10 v1 dV x10 y10\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + + // give the node a tag... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "n10 v2 dV x10 y10 Tamenity=restaurant\n")); + + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", + "osm_type = 'N' AND osm_id = 10 AND " + "tags->'amenity' = 'restaurant'")); + + SECTION("remove the tag from node") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "n10 v3 dV x10 y10\n")); + } + + SECTION("delete the node") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "n10 v3 dD\n")); + } + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); +} + +TEST_CASE("updating a way") +{ + // import a simple way... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex_uni.lua"), + "n10 v1 dV x10.0 y10.1\n" + "n11 v1 dV x10.1 y10.2\n" + "w20 v1 dV Thighway=primary Nn10,n11\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE( + 1 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'highway' = 'primary' " + "AND ST_NumPoints(geom) = 2")); + + // now change the way itself... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v2 dV Thighway=secondary Nn10,n11\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE(1 == conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'highway' = " + "'secondary' AND ST_NumPoints(geom) = 2")); + + // now change a node in the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "n10 v2 dV x10.0 y10.3\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE(1 == conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'highway' = " + "'secondary' AND ST_NumPoints(geom) = 2")); + + // now add a node to the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "n12 v1 dV x10.2 y10.1\n" + "w20 v3 dV Thighway=residential Nn10,n11,n12\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE(1 == conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'highway' = " + "'residential' AND ST_NumPoints(geom) = 3")); + + // now delete the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v4 dD\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); +} + +TEST_CASE("ways as linestrings and polygons") +{ + // import a simple way... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex_uni.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "w20 v1 dV Tbuilding=yes Nn10,n11,n12,n13,n14,n10\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type != 'W'")); + // REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + // REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE( + 1 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND " + "tags->'highway' = 'secondary' AND " + "ST_GeometryType(geom) = 'ST_LineString'")); + + // now change the way tags... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v2 dV Thighway=secondary Nn10,n11,n12,n13,n14,n10\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type != 'W'")); + REQUIRE( + 0 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND " + "tags->'highway' = 'secondary' AND " + "ST_GeometryType(geom) = 'ST_LineString'")); + + // now remove a node from the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v3 dV Thighway=secondary Nn10,n11,n12,n13,n14\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type != 'W'")); + REQUIRE( + 0 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND " + "tags->'highway' = 'secondary' AND " + "ST_GeometryType(geom) = 'ST_LineString'")); + + // now change the tag back to an area tag (but the way is not closed)... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v4 dV Tbuilding=yes Nn10,n11,n12,n13,n14\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data")); + + // now close the way again + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "w20 v5 dV Tbuilding=yes Nn10,n11,n12,n13,n14,n10\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type != 'W'")); + REQUIRE( + 1 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'W' AND osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); +} + +TEST_CASE("multipolygons") +{ + // import a simple multipolygon relation... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex_uni.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "w20 v1 dV Nn10,n11,n12,n13,n14,n10\n" + "r30 v1 dV Ttype=multipolygon,building=yes Mw20@\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'R'")); + REQUIRE( + 1 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'R' AND osm_id = 30 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + + // change tags on that relation... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "r30 v2 dV Ttype=multipolygon,building=yes,name=Shed Mw20@\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'N'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_data", "osm_type = 'W'")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_data", "osm_type = 'R'")); + REQUIRE( + 1 == + conn.get_count( + "osm2pgsql_test_data", + "osm_type = 'R' AND osm_id = 30 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + + SECTION("remove relation") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "r30 v3 dD\n")); + } + + SECTION("remove multipolygon tag") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex_uni.lua"), + "r30 v3 dV Tbuilding=yes,name=Shed Mw20@\n")); + } + + REQUIRE(0 == conn.get_count("osm2pgsql_test_data")); +} diff --git a/tests/test-output-flex-update.cpp b/tests/test-output-flex-update.cpp new file mode 100644 index 000000000..a9e23778a --- /dev/null +++ b/tests/test-output-flex-update.cpp @@ -0,0 +1,224 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("updating a node") +{ + // import a node... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex.lua"), + "n10 v1 dV x10 y10\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + + // give the node a tag... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "n10 v2 dV x10 y10 Tamenity=restaurant\n")); + + REQUIRE(1 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == + conn.get_count("osm2pgsql_test_point", + "node_id = 10 AND tags->'amenity' = 'restaurant'")); + + SECTION("remove the tag from node") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "n10 v3 dV x10 y10\n")); + } + + SECTION("delete the node") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "n10 v3 dD\n")); + } + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); +} + +TEST_CASE("updating a way") +{ + // import a simple way... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex.lua"), + "n10 v1 dV x10.0 y10.1\n" + "n11 v1 dV x10.1 y10.2\n" + "w20 v1 dV Thighway=primary Nn10,n11\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = 'primary' " + "AND ST_NumPoints(geom) = 2")); + + // now change the way itself... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v2 dV Thighway=secondary Nn10,n11\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = " + "'secondary' AND ST_NumPoints(geom) = 2")); + + // now change a node in the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "n10 v2 dV x10.0 y10.3\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = " + "'secondary' AND ST_NumPoints(geom) = 2")); + + // now add a node to the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "n12 v1 dV x10.2 y10.1\n" + "w20 v3 dV Thighway=residential Nn10,n11,n12\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = " + "'residential' AND ST_NumPoints(geom) = 3")); + + // now delete the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v4 dD\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); +} + +TEST_CASE("ways as linestrings and polygons") +{ + // import a simple way... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "w20 v1 dV Tbuilding=yes Nn10,n11,n12,n13,n14,n10\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon", + "osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + + // now change the way tags... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v2 dV Thighway=secondary Nn10,n11,n12,n13,n14,n10\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == + conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = 'secondary' AND " + "ST_GeometryType(geom) = 'ST_LineString'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon")); + + // now remove a node from the way... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v3 dV Thighway=secondary Nn10,n11,n12,n13,n14\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == + conn.get_count("osm2pgsql_test_line", + "osm_id = 20 AND tags->'highway' = 'secondary' AND " + "ST_GeometryType(geom) = 'ST_LineString'")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon")); + + // now change the tag back to an area tag (but the way is not closed)... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v4 dV Tbuilding=yes Nn10,n11,n12,n13,n14\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon")); + + // now close the way again + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "w20 v5 dV Tbuilding=yes Nn10,n11,n12,n13,n14,n10\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon", + "osm_id = 20 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); +} + +TEST_CASE("multipolygons") +{ + // import a simple multipolygon relation... + REQUIRE_NOTHROW( + db.run_import(testing::opt_t().slim().flex("test_output_flex.lua"), + "n10 v1 dV x10.0 y10.0\n" + "n11 v1 dV x10.0 y10.2\n" + "n12 v1 dV x10.2 y10.2\n" + "n13 v1 dV x10.2 y10.0\n" + "w20 v1 dV Nn10,n11,n12,n13,n14,n10\n" + "r30 v1 dV Ttype=multipolygon,building=yes Mw20@\n")); + + auto conn = db.db().connect(); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon", + "osm_id = -30 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + + // change tags on that relation... + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "r30 v2 dV Ttype=multipolygon,building=yes,name=Shed Mw20@\n")); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon", + "osm_id = -30 AND tags->'building' = 'yes' AND " + "ST_GeometryType(geom) = 'ST_Polygon'")); + + SECTION("remove relation") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "r30 v3 dD\n")); + } + + SECTION("remove multipolygon tag") + { + REQUIRE_NOTHROW(db.run_import( + testing::opt_t().slim().append().flex("test_output_flex.lua"), + "r30 v3 dV Tbuilding=yes,name=Shed Mw20@\n")); + } + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon")); +} diff --git a/tests/test-output-flex-validgeom.cpp b/tests/test-output-flex-validgeom.cpp new file mode 100644 index 000000000..5800dbc08 --- /dev/null +++ b/tests/test-output-flex-validgeom.cpp @@ -0,0 +1,25 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +TEST_CASE("no invalid geometries") +{ + REQUIRE_NOTHROW( + db.run_file(testing::opt_t().slim().flex("test_output_flex.lua"), + "test_output_pgsql_validgeom.osm")); + + auto conn = db.db().connect(); + + conn.require_has_table("osm2pgsql_test_point"); + conn.require_has_table("osm2pgsql_test_line"); + conn.require_has_table("osm2pgsql_test_polygon"); + conn.require_has_table("osm2pgsql_test_route"); + + REQUIRE(12 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(0 == + conn.get_count("osm2pgsql_test_polygon", "NOT ST_IsValid(geom)")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon", "ST_IsEmpty(geom)")); +} diff --git a/tests/test-output-flex.cpp b/tests/test-output-flex.cpp new file mode 100644 index 000000000..e387247a7 --- /dev/null +++ b/tests/test-output-flex.cpp @@ -0,0 +1,132 @@ +#include + +#include "common-import.hpp" +#include "common-options.hpp" + +static testing::db::import_t db; + +static void require_tables(pg::conn_t const &conn) +{ + conn.require_has_table("osm2pgsql_test_point"); + conn.require_has_table("osm2pgsql_test_line"); + conn.require_has_table("osm2pgsql_test_polygon"); + conn.require_has_table("osm2pgsql_test_route"); +} + +TEST_CASE("liechtenstein slim regression simple") +{ + REQUIRE_NOTHROW( + db.run_file(testing::opt_t().slim().flex("test_output_flex.lua"), + "liechtenstein-2013-08-03.osm.pbf")); + + auto conn = db.db().connect(); + require_tables(conn); + + CHECK(1362 == conn.get_count("osm2pgsql_test_point")); + CHECK(2932 == conn.get_count("osm2pgsql_test_line")); + CHECK(4136 == conn.get_count("osm2pgsql_test_polygon")); + CHECK(35 == conn.get_count("osm2pgsql_test_route")); + + // Check size of lines + conn.assert_double( + 1696.04, + "SELECT ST_Length(geom) FROM osm2pgsql_test_line WHERE osm_id = 1101"); + conn.assert_double(1151.26, + "SELECT ST_Length(ST_Transform(geom,4326)::geography) " + "FROM osm2pgsql_test_line WHERE osm_id = 1101"); + + conn.assert_double( + 311.289, "SELECT area FROM osm2pgsql_test_polygon WHERE osm_id = 3265"); + conn.assert_double( + 311.289, + "SELECT ST_Area(geom) FROM osm2pgsql_test_polygon WHERE osm_id = 3265"); + conn.assert_double( + 143.845, "SELECT ST_Area(ST_Transform(geom,4326)::geography) FROM " + "osm2pgsql_test_polygon WHERE osm_id = 3265"); + + // Check a point's location + REQUIRE(1 == conn.get_count("osm2pgsql_test_point", + "ST_DWithin(geom, 'SRID=3857;POINT(1062645.12 " + "5972593.4)'::geometry, 0.1)")); +} + +TEST_CASE("liechtenstein slim latlon") +{ + REQUIRE_NOTHROW(db.run_file( + testing::opt_t().slim().flex("test_output_flex.lua").srs(PROJ_LATLONG), + "liechtenstein-2013-08-03.osm.pbf")); + + auto conn = db.db().connect(); + require_tables(conn); + + REQUIRE(1362 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(2932 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(4136 == conn.get_count("osm2pgsql_test_polygon")); + + // Check size of lines + conn.assert_double( + 0.0105343, + "SELECT ST_Length(geom) FROM osm2pgsql_test_line WHERE osm_id = 1101"); + conn.assert_double(1151.26, + "SELECT ST_Length(ST_Transform(geom,4326)::geography) " + "FROM osm2pgsql_test_line WHERE osm_id = 1101"); + + conn.assert_double( + 1.70718e-08, + "SELECT area FROM osm2pgsql_test_polygon WHERE osm_id = 3265"); + conn.assert_double( + 1.70718e-08, + "SELECT ST_Area(geom) FROM osm2pgsql_test_polygon WHERE osm_id = 3265"); + conn.assert_double( + 143.845, "SELECT ST_Area(ST_Transform(geom,4326)::geography) FROM " + "osm2pgsql_test_polygon WHERE osm_id = 3265"); + + // Check a point's location + REQUIRE(1 == conn.get_count("osm2pgsql_test_point", + "ST_DWithin(geom, 'SRID=4326;POINT(9.5459035 " + "47.1866494)'::geometry, 0.00001)")); +} + +TEST_CASE("way area slim flatnode") +{ + REQUIRE_NOTHROW(db.run_file( + testing::opt_t().slim().flex("test_output_flex.lua").flatnodes(), + "test_output_pgsql_way_area.osm")); + + auto conn = db.db().connect(); + require_tables(conn); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_polygon")); +} + +TEST_CASE("route relation slim flatnode") +{ + REQUIRE_NOTHROW(db.run_file( + testing::opt_t().slim().flex("test_output_flex.lua").flatnodes(), + "test_output_pgsql_route_rel.osm")); + + auto conn = db.db().connect(); + require_tables(conn); + + REQUIRE(0 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(0 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(1 == conn.get_count("osm2pgsql_test_route")); +} + +TEST_CASE("liechtenstein slim bz2 parsing regression") +{ + REQUIRE_NOTHROW( + db.run_file(testing::opt_t().slim().flex("test_output_flex.lua"), + "liechtenstein-2013-08-03.osm.bz2")); + + auto conn = db.db().connect(); + require_tables(conn); + + REQUIRE(1362 == conn.get_count("osm2pgsql_test_point")); + REQUIRE(2932 == conn.get_count("osm2pgsql_test_line")); + REQUIRE(4136 == conn.get_count("osm2pgsql_test_polygon")); + REQUIRE(35 == conn.get_count("osm2pgsql_test_route")); +}