Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test MML queries are valid in CI #4614

Merged
merged 3 commits into from
Sep 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,7 @@ indent_size = 4
[*.svg]
indent_style = undef
indent_size = undef

# Lastmod files are used by the scripts and should not contain newlines
[*.lastmod]
insert_final_newline = false
6 changes: 5 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Install osm2pgsql and database
run: sudo apt-get install -qq --no-install-recommends osm2pgsql postgresql-14-postgis-3
run: sudo apt-get install -qq --no-install-recommends osm2pgsql postgresql-14-postgis-3 gdal-bin
- name: Wait for database
run : sudo pg_ctlcluster 14 main start; until pg_isready; do sleep 0.5; done
- name: Setup database
Expand All @@ -45,3 +45,7 @@ jobs:
osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis -r xml <(echo '<osm version="0.6"/>')
- name: Create indexes
run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql
- name: Load empty shapefiles
run: scripts/get-external-data.py --no-update --cache -D scripts/empty_files
- name: Test queries are valid
run: scripts/test-queries.py project.mml
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thu, 01 Jan 1970 00:00:00 GMT
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thu, 01 Jan 1970 00:00:00 GMT
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thu, 01 Jan 1970 00:00:00 GMT
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thu, 01 Jan 1970 00:00:00 GMT
Binary file added scripts/empty_files/water-polygons-split-3857.zip
Binary file not shown.
1 change: 1 addition & 0 deletions scripts/empty_files/water-polygons-split-3857.zip.lastmod
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thu, 01 Jan 1970 00:00:00 GMT
84 changes: 84 additions & 0 deletions scripts/test-queries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/usr/bin/env python3
""""
Tests the queries in a CartoCSS MML file against a database.

Ignore database settings in the MML file.
"""
import argparse
import logging
import psycopg2
import yaml

EMPTY_POLYGON="""ST_SetSRID('POLYGON EMPTY'::geometry, 3857)"""

class ResultHasRowsError(Exception):
"""Class for layers that have """
pass

def testLayer(layer, curs):
datasource = layer["Datasource"]
if datasource["type"] != "postgis":
logging.warning("Layer type {} not supported".format(layer["type"]))
return
geometry_field=datasource["geometry_field"]

full_query = """
SELECT "{}" FROM
{}
""".format(geometry_field, datasource["table"])
if "!bbox!" not in full_query:
full_query += "\nWHERE {} && !bbox!".format(geometry_field)

query = full_query.replace("!bbox!", EMPTY_POLYGON).replace("!pixel_width!", "0").replace("!pixel_height!", "0").replace("!scale_denominator!", "0")
logging.debug("Running query {}".format(query))
curs.execute(query)
if curs.rowcount != 0:
raise ResultHasRowsError(query)

def main():
# parse options
parser = argparse.ArgumentParser(
description="Test CartoCSS project queries against a database")


parser.add_argument("-d", "--database", action="store", default="gis",
help="Override database name to connect to")
parser.add_argument("-H", "--host", action="store",
help="Override database server host or socket directory")
parser.add_argument("-p", "--port", action="store",
help="Override database server port")
parser.add_argument("-U", "--username", action="store",
help="Override database user name")
parser.add_argument("-v", "--verbose", action="store_true",
help="Be more verbose. Overrides -q")
parser.add_argument("-q", "--quiet", action="store_true",
help="Only report serious problems")
parser.add_argument("-w", "--password", action="store",
help="Override database password")

parser.add_argument('project', type=argparse.FileType('r'),
help="CartoCSS MML file")
opts = parser.parse_args()

if opts.verbose:
logging.basicConfig(level=logging.DEBUG)
elif opts.quiet:
logging.basicConfig(level=logging.WARNING)
else:
logging.basicConfig(level=logging.INFO)

mml=yaml.safe_load(opts.project)
logging.debug("Loaded MML")

with psycopg2.connect(database=opts.database, host=opts.host,
port=opts.port, user=opts.username,
password=opts.password).cursor() as curs:
curs.execute("SET default_transaction_read_only = TRUE;")
for layer in mml["Layer"]:
testLayer(layer, curs)




if __name__ == '__main__':
main()