Emit one index statement per line, add -0 option

.github/workflows/ci.yml

@@ -40,6 +40,15 @@ jobs:
       run: '! grep "class:" project.mml > /dev/null'
     - name: Validate with CartoCSS
       run: 'carto project.mml --api 3.0.22 --quiet > /dev/null'
+    - name: Check that index script works without errors
+      run: |
+        scripts/indexes.py --fillfactor 100 > /dev/null &&
+        scripts/indexes.py --concurrent > /dev/null &&
+        scripts/indexes.py --reindex > /dev/null &&
+        scripts/indexes.py --concurrent --reindex > /dev/null &&
+        scripts/indexes.py --notexist > /dev/null &&
+        scripts/indexes.py --null > /dev/null &&
+        scripts/indexes.py --null > /dev/null
   import:
     needs: [syntax]
     runs-on: ubuntu-22.04

INSTALL.md

@@ -7,20 +7,20 @@ You need OpenStreetMap data loaded into a PostGIS database (see below for [depen
 
 Start by creating a database
 
-```
+```sh
 sudo -u postgres createuser -s $USER
 createdb gis
 ```
 
 Enable PostGIS and hstore extensions with
 
-```
+```sh
 psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;'
 ```
 
 then grab some OSM data. It's probably easiest to grab an PBF of OSM data from [Geofabrik](https://download.geofabrik.de/). Once you've done that, import with osm2pgsql:
 
-```
+```sh
 osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf
 ```
 
@@ -35,18 +35,24 @@ Disabling JIT is **essential** for use with Kosmtik and other style development
 JIT can be disabled with `psql -d gis -c 'ALTER SYSTEM SET jit=off;' -c 'SELECT pg_reload_conf();'` or any other means of adjusting the PostgreSQL config.
 
 ### Custom indexes
-Custom indexes are required for rendering performance and are essential on full planet databases.
+Custom indexes are required for rendering performance and are essential on full planet databases. These are generated by the `script/indexes.py` script, see `script/indexes.py --help` for various advanced options, but the command below will work to create the indexes on a new installation:
 
-```
+```sh
 psql -d gis -f indexes.sql
 ```
 
+The indexes can be created in parallel with
+
+```sh
+scripts/indexes.py -0 | xargs -0 -P0 -I{} psql -d gis -c "{}"
+```
+
 ## Scripted download
 Some features are rendered using preprocessed shapefiles.
 
 To download them and import them into the database you can run the following script
 
-```
+```sh
 scripts/get-external-data.py
 ```
 

indexes.sql

@@ -1,53 +1,18 @@
 -- These are indexes for rendering performance with OpenStreetMap Carto.
 -- This file is generated with scripts/indexes.py
-
-CREATE INDEX planet_osm_line_ferry
-  ON planet_osm_line USING GIST (way)
-  WHERE route = 'ferry' AND osm_id > 0;
-CREATE INDEX planet_osm_line_label
-  ON planet_osm_line USING GIST (way)
-  WHERE name IS NOT NULL OR ref IS NOT NULL;
-CREATE INDEX planet_osm_line_river
-  ON planet_osm_line USING GIST (way)
-  WHERE waterway = 'river';
-CREATE INDEX planet_osm_line_waterway
-  ON planet_osm_line USING GIST (way)
-  WHERE waterway IN ('river', 'canal', 'stream', 'drain', 'ditch');
-CREATE INDEX planet_osm_point_place
-  ON planet_osm_point USING GIST (way)
-  WHERE place IS NOT NULL AND name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_admin
-  ON planet_osm_polygon USING GIST (ST_PointOnSurface(way))
-  WHERE name IS NOT NULL AND boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
-CREATE INDEX planet_osm_polygon_military
-  ON planet_osm_polygon USING GIST (way)
-  WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
-CREATE INDEX planet_osm_polygon_name
-  ON planet_osm_polygon USING GIST (ST_PointOnSurface(way))
-  WHERE name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_name_z6
-  ON planet_osm_polygon USING GIST (ST_PointOnSurface(way))
-  WHERE name IS NOT NULL AND way_area > 5980000;
-CREATE INDEX planet_osm_polygon_nobuilding
-  ON planet_osm_polygon USING GIST (way)
-  WHERE building IS NULL;
-CREATE INDEX planet_osm_polygon_water
-  ON planet_osm_polygon USING GIST (way)
-  WHERE waterway IN ('dock', 'riverbank', 'canal')
-    OR landuse IN ('reservoir', 'basin')
-    OR "natural" IN ('water', 'glacier');
-CREATE INDEX planet_osm_polygon_way_area_z10
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 23300;
-CREATE INDEX planet_osm_polygon_way_area_z6
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 5980000;
-CREATE INDEX planet_osm_roads_admin
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative';
-CREATE INDEX planet_osm_roads_admin_low
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
-CREATE INDEX planet_osm_roads_roads_ref
-  ON planet_osm_roads USING GIST (way)
-  WHERE highway IS NOT NULL AND ref IS NOT NULL;
+CREATE INDEX planet_osm_line_ferry ON planet_osm_line USING GIST (way) WHERE route = 'ferry' AND osm_id > 0;
+CREATE INDEX planet_osm_line_label ON planet_osm_line USING GIST (way) WHERE name IS NOT NULL OR ref IS NOT NULL;
+CREATE INDEX planet_osm_line_river ON planet_osm_line USING GIST (way) WHERE waterway = 'river';
+CREATE INDEX planet_osm_line_waterway ON planet_osm_line USING GIST (way) WHERE waterway IN ('river', 'canal', 'stream', 'drain', 'ditch');
+CREATE INDEX planet_osm_point_place ON planet_osm_point USING GIST (way) WHERE place IS NOT NULL AND name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_admin ON planet_osm_polygon USING GIST (ST_PointOnSurface(way)) WHERE name IS NOT NULL AND boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
+CREATE INDEX planet_osm_polygon_military ON planet_osm_polygon USING GIST (way) WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
+CREATE INDEX planet_osm_polygon_name ON planet_osm_polygon USING GIST (ST_PointOnSurface(way)) WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_name_z6 ON planet_osm_polygon USING GIST (ST_PointOnSurface(way)) WHERE name IS NOT NULL AND way_area > 5980000;
+CREATE INDEX planet_osm_polygon_nobuilding ON planet_osm_polygon USING GIST (way) WHERE building IS NULL;
+CREATE INDEX planet_osm_polygon_water ON planet_osm_polygon USING GIST (way) WHERE waterway IN ('dock', 'riverbank', 'canal') OR landuse IN ('reservoir', 'basin') OR "natural" IN ('water', 'glacier');
+CREATE INDEX planet_osm_polygon_way_area_z10 ON planet_osm_polygon USING GIST (way) WHERE way_area > 23300;
+CREATE INDEX planet_osm_polygon_way_area_z6 ON planet_osm_polygon USING GIST (way) WHERE way_area > 5980000;
+CREATE INDEX planet_osm_roads_admin ON planet_osm_roads USING GIST (way) WHERE boundary = 'administrative';
+CREATE INDEX planet_osm_roads_admin_low ON planet_osm_roads USING GIST (way) WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
+CREATE INDEX planet_osm_roads_roads_ref ON planet_osm_roads USING GIST (way) WHERE highway IS NOT NULL AND ref IS NOT NULL;

indexes.yml

@@ -32,10 +32,7 @@ polygon:
     where: (landuse = 'military' OR military = 'danger_area') AND building IS NULL
   water:
     # The indentation here makes sense in the SQL output
-    where: |-
-      waterway IN ('dock', 'riverbank', 'canal')
-          OR landuse IN ('reservoir', 'basin')
-          OR "natural" IN ('water', 'glacier')
+    where: waterway IN ('dock', 'riverbank', 'canal') OR landuse IN ('reservoir', 'basin') OR "natural" IN ('water', 'glacier')
   way_area_z6:
     where: way_area > 5980000
   way_area_z10:

scripts/indexes.py

@@ -8,63 +8,61 @@
 
 import argparse, sys, os, yaml
 
+parser = argparse.ArgumentParser(description='Generates custom index statements')
+parser.add_argument('--concurrent', dest='concurrent', help='Generate indexes CONCURRENTLY', action='store_true', default=False)
+parser.add_argument('--fillfactor', help='Custom fillfactor to use')
+parser.add_argument('--notexist', help='Use IF NOT EXISTS (requires 9.5)', action='store_true', default=False)
+parser.add_argument('--osm2pgsql', help='Include indexes normally built by osm2pgsql', action='store_true', default=False)
+parser.add_argument('--reindex', help='Rebuild existing indexes', action='store_true', default=False)
+parser.add_argument('--null', '-0', help='SQL statements are terminated by a null character instead of whitespace for use with xargs', action='store_true', default=False)
+args = parser.parse_args()
+
+separator = '\0' if args.null else '\n'
+
 def index_statement(table, name, function, conditions=None, concurrent=False,notexist=False, fillfactor=None):
     options = ' CONCURRENTLY' if concurrent else ''
     options += ' IF NOT EXISTS' if notexist else ''
-    storage = '' if fillfactor is None else '\n  WITH (fillfactor={})'.format(fillfactor)
-    where = '' if conditions is None else '\n  WHERE {}'.format(conditions)
-    return ('CREATE INDEX{options} {table}_{name}\n' +
-            '  ON {table} USING GIST ({function})' +
-            '{storage}' +
-            '{where};\n').format(table="planet_osm_"+table, name=name, function=function,
-                storage=storage, options=options, where=where)
+    storage = '' if fillfactor is None else f' WITH (fillfactor={fillfactor})'
+    where = '' if conditions is None else f' WHERE {conditions}'
 
-def parse(cb):
+    return f'CREATE INDEX{options} planet_osm_{table}_{name} ON planet_osm_{table} USING GIST ({function}){storage}{where};'.replace('\n', ' ')
+
+def parse(index_function):
     with open(os.path.join(os.path.dirname(__file__), '../indexes.yml')) as yaml_file:
         indexes = yaml.safe_load(yaml_file)
 
     for table, data in sorted(indexes.items()):
         for name, definition in sorted(data.items()):
-            cb(table, name, definition.get("function", "way"), definition["where"])
+            print(index_function(table, name, definition.get("function", "way"), definition["where"]), end=separator)
 
 # The same as parse, but for osm2pgsql-built indexes
-def osm2pgsql_parse(cb):
-    cb('point', 'index', None)
-    cb('line', 'index', None)
-    cb('polygon', 'index', None)
-    cb('roads', 'index', None)
-
-parser = argparse.ArgumentParser(description='Generates custom index statements')
-parser.add_argument('--concurrent', dest='concurrent', help='Generate indexes CONCURRENTLY', action='store_true', default=False)
-parser.add_argument('--fillfactor', help='Custom fillfactor to use')
-parser.add_argument('--notexist', help='Use IF NOT EXISTS (requires 9.5)', action='store_true', default=False)
-parser.add_argument('--osm2pgsql', help='Include indexes normally built by osm2pgsql', action='store_true', default=False)
-parser.add_argument('--reindex', help='Rebuild existing indexes', action='store_true', default=False)
-args = parser.parse_args()
+def osm2pgsql_parse(index_function):
+    print(index_function('point', 'index', None, None), end=separator)
+    print(index_function('line', 'index', None, None), end=separator)
+    print(index_function('polygon', 'index', None, None), end=separator)
+    print(index_function('roads', 'index', None, None), end=separator)
 
-def cb (table, name, function, where):
-    print(index_statement(table, name, function, where, args.concurrent, args.notexist, args.fillfactor), end='')
+def generate_statement(table, name, function, where):
+    return index_statement(table, name, function, where, args.concurrent, args.notexist, args.fillfactor)
 
-def reindex_cb(table, name, function, where):
+def generate_reindex_statement(table, name, function, where):
     if not args.concurrent:
-        print('REINDEX planet_osm_{table}_{name};'.format(table=table, name=name))
+        return f'REINDEX planet_osm_{table}_{name};'
     else:
         # Rebuilding indexes concurrently requires making a new index, dropping the old one, and renaming.
-        print('ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old;'.format(table=table, name=name))
-        cb(table, name, function, where)
-        print('DROP INDEX planet_osm_{table}_{name}_old;\n'.format(table=table, name=name))
+        return f'ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old; {generate_statement(table, name, function, where)} + DROP INDEX planet_osm_{table}_{name}_old;'
 
-print(('-- These are indexes for rendering performance with OpenStreetMap Carto.\n'+
-      '-- This file is generated with {}\n').format(' '.join(sys.argv)))
+print('-- These are indexes for rendering performance with OpenStreetMap Carto.', end=separator)
+print('-- This file is generated with {}'.format(' '.join(sys.argv)), end=separator)
 
 if not args.reindex:
-    parse(cb)
+    parse(generate_statement)
 else:
-    parse(reindex_cb)
+    parse(generate_reindex_statement)
 
 if args.osm2pgsql:
-    print('\n-- These indexes are normally built by osm2pgsql')
+    print('-- These indexes are normally built by osm2pgsql', end=separator)
     if not args.reindex:
-        osm2pgsql_parse(cb)
+        osm2pgsql_parse(generate_statement)
     else:
-        osm2pgsql_parse(reindex_cb)
+        osm2pgsql_parse(generate_reindex_statement)