diff --git a/.editorconfig b/.editorconfig index 868b9cccef..d5298b2f39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,3 +18,7 @@ indent_size = 4 [*.svg] indent_style = undef indent_size = undef + +# Lastmod files are used by the scripts and should not contain newlines +[*.lastmod] +insert_final_newline = false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 756a79de1b..30bb795de0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install osm2pgsql and database - run: sudo apt-get install -qq --no-install-recommends osm2pgsql postgresql-14-postgis-3 + run: sudo apt-get install -qq --no-install-recommends osm2pgsql postgresql-14-postgis-3 gdal-bin - name: Wait for database run : sudo pg_ctlcluster 14 main start; until pg_isready; do sleep 0.5; done - name: Setup database @@ -45,3 +45,7 @@ jobs: osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis -r xml <(echo '') - name: Create indexes run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql + - name: Load empty shapefiles + run: scripts/get-external-data.py --no-update --cache -D scripts/empty_files + - name: Test queries are valid + run: scripts/test-queries.py project.mml diff --git a/scripts/empty_files/antarctica-icesheet-outlines-3857.zip b/scripts/empty_files/antarctica-icesheet-outlines-3857.zip new file mode 100644 index 0000000000..89c5f5caa9 Binary files /dev/null and b/scripts/empty_files/antarctica-icesheet-outlines-3857.zip differ diff --git a/scripts/empty_files/antarctica-icesheet-outlines-3857.zip.lastmod b/scripts/empty_files/antarctica-icesheet-outlines-3857.zip.lastmod new file mode 100644 index 0000000000..46e2b50b44 --- /dev/null +++ b/scripts/empty_files/antarctica-icesheet-outlines-3857.zip.lastmod @@ -0,0 +1 @@ +Thu, 01 Jan 1970 00:00:00 GMT \ No newline at end of file diff --git a/scripts/empty_files/antarctica-icesheet-polygons-3857.zip b/scripts/empty_files/antarctica-icesheet-polygons-3857.zip new file mode 100644 index 0000000000..30b4d2ef97 Binary files /dev/null and b/scripts/empty_files/antarctica-icesheet-polygons-3857.zip differ diff --git a/scripts/empty_files/antarctica-icesheet-polygons-3857.zip.lastmod b/scripts/empty_files/antarctica-icesheet-polygons-3857.zip.lastmod new file mode 100644 index 0000000000..46e2b50b44 --- /dev/null +++ b/scripts/empty_files/antarctica-icesheet-polygons-3857.zip.lastmod @@ -0,0 +1 @@ +Thu, 01 Jan 1970 00:00:00 GMT \ No newline at end of file diff --git a/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip b/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip new file mode 100644 index 0000000000..00b5d2f3b5 Binary files /dev/null and b/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip differ diff --git a/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip.lastmod b/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip.lastmod new file mode 100644 index 0000000000..46e2b50b44 --- /dev/null +++ b/scripts/empty_files/ne_110m_admin_0_boundary_lines_land.zip.lastmod @@ -0,0 +1 @@ +Thu, 01 Jan 1970 00:00:00 GMT \ No newline at end of file diff --git a/scripts/empty_files/simplified-water-polygons-split-3857.zip b/scripts/empty_files/simplified-water-polygons-split-3857.zip new file mode 100644 index 0000000000..6e48430f12 Binary files /dev/null and b/scripts/empty_files/simplified-water-polygons-split-3857.zip differ diff --git a/scripts/empty_files/simplified-water-polygons-split-3857.zip.lastmod b/scripts/empty_files/simplified-water-polygons-split-3857.zip.lastmod new file mode 100644 index 0000000000..46e2b50b44 --- /dev/null +++ b/scripts/empty_files/simplified-water-polygons-split-3857.zip.lastmod @@ -0,0 +1 @@ +Thu, 01 Jan 1970 00:00:00 GMT \ No newline at end of file diff --git a/scripts/empty_files/water-polygons-split-3857.zip b/scripts/empty_files/water-polygons-split-3857.zip new file mode 100644 index 0000000000..6d586e7a7a Binary files /dev/null and b/scripts/empty_files/water-polygons-split-3857.zip differ diff --git a/scripts/empty_files/water-polygons-split-3857.zip.lastmod b/scripts/empty_files/water-polygons-split-3857.zip.lastmod new file mode 100644 index 0000000000..46e2b50b44 --- /dev/null +++ b/scripts/empty_files/water-polygons-split-3857.zip.lastmod @@ -0,0 +1 @@ +Thu, 01 Jan 1970 00:00:00 GMT \ No newline at end of file diff --git a/scripts/test-queries.py b/scripts/test-queries.py new file mode 100755 index 0000000000..64432987f8 --- /dev/null +++ b/scripts/test-queries.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""" +Tests the queries in a CartoCSS MML file against a database. + +Ignore database settings in the MML file. +""" +import argparse +import logging +import psycopg2 +import yaml + +EMPTY_POLYGON="""ST_SetSRID('POLYGON EMPTY'::geometry, 3857)""" + +class ResultHasRowsError(Exception): + """Class for layers that have """ + pass + +def testLayer(layer, curs): + datasource = layer["Datasource"] + if datasource["type"] != "postgis": + logging.warning("Layer type {} not supported".format(layer["type"])) + return + geometry_field=datasource["geometry_field"] + + full_query = """ + SELECT "{}" FROM + {} + """.format(geometry_field, datasource["table"]) + if "!bbox!" not in full_query: + full_query += "\nWHERE {} && !bbox!".format(geometry_field) + + query = full_query.replace("!bbox!", EMPTY_POLYGON).replace("!pixel_width!", "0").replace("!pixel_height!", "0").replace("!scale_denominator!", "0") + logging.debug("Running query {}".format(query)) + curs.execute(query) + if curs.rowcount != 0: + raise ResultHasRowsError(query) + +def main(): + # parse options + parser = argparse.ArgumentParser( + description="Test CartoCSS project queries against a database") + + + parser.add_argument("-d", "--database", action="store", default="gis", + help="Override database name to connect to") + parser.add_argument("-H", "--host", action="store", + help="Override database server host or socket directory") + parser.add_argument("-p", "--port", action="store", + help="Override database server port") + parser.add_argument("-U", "--username", action="store", + help="Override database user name") + parser.add_argument("-v", "--verbose", action="store_true", + help="Be more verbose. Overrides -q") + parser.add_argument("-q", "--quiet", action="store_true", + help="Only report serious problems") + parser.add_argument("-w", "--password", action="store", + help="Override database password") + + parser.add_argument('project', type=argparse.FileType('r'), + help="CartoCSS MML file") + opts = parser.parse_args() + + if opts.verbose: + logging.basicConfig(level=logging.DEBUG) + elif opts.quiet: + logging.basicConfig(level=logging.WARNING) + else: + logging.basicConfig(level=logging.INFO) + + mml=yaml.safe_load(opts.project) + logging.debug("Loaded MML") + + with psycopg2.connect(database=opts.database, host=opts.host, + port=opts.port, user=opts.username, + password=opts.password).cursor() as curs: + curs.execute("SET default_transaction_read_only = TRUE;") + for layer in mml["Layer"]: + testLayer(layer, curs) + + + + +if __name__ == '__main__': + main()