Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

make calculating spatial/datetime extents optional #143

Merged
merged 3 commits into from
Nov 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/src/user_guide/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ class: `tipg.settings.DatabaseSettings`
prefix: **`TIPG_DB_`**

- **SCHEMAS** (list of string): Named schemas, `tipg` can look for `Tables` or `Functions`. Default is `["public"]`
- **SPATIAL_EXTENT** (bool): Calculate spatial extent of records. Default is `True`.
- **DATETIME_EXTENT** (bool): Calculate temporal extent of records. Default is `True`.

#### `Tables`

Expand Down
54 changes: 54 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ async def lifespan(app: FastAPI):
exclude_functions=db_settings.exclude_functions,
exclude_function_schemas=db_settings.exclude_function_schemas,
spatial=db_settings.only_spatial_tables,
spatial_extent=db_settings.spatial_extent,
datetime_extent=db_settings.datetime_extent,
)
yield
await close_db_connection(app)
Expand Down Expand Up @@ -333,6 +335,58 @@ def app_myschema_public(database_url, monkeypatch):
yield client


@pytest.fixture
def app_no_extents(database_url, monkeypatch):
"""Create APP with tables from `myschema` and `public` schema but without
calculating the spatial/datetime extents.

Available tables should come from `myschema` and `public` and functions from `pg_temp`.
"""
postgres_settings = PostgresSettings(database_url=database_url)
db_settings = DatabaseSettings(
schemas=["myschema", "public"],
spatial_extent=False,
datetime_extent=False,
only_spatial_tables=False,
)
sql_settings = CustomSQLSettings(custom_sql_directory=SQL_FUNCTIONS_DIRECTORY)

app = create_tipg_app(
postgres_settings=postgres_settings,
db_settings=db_settings,
sql_settings=sql_settings,
)

with TestClient(app) as client:
yield client


@pytest.fixture
def app_no_spatial_extent(database_url, monkeypatch):
"""Create APP with tables from `myschema` and `public` schema but without
calculating the spatial/datetime extents.

Available tables should come from `myschema` and `public` and functions from `pg_temp`.
"""
postgres_settings = PostgresSettings(database_url=database_url)
db_settings = DatabaseSettings(
schemas=["myschema", "public"],
spatial_extent=False,
datetime_extent=True,
only_spatial_tables=False,
)
sql_settings = CustomSQLSettings(custom_sql_directory=SQL_FUNCTIONS_DIRECTORY)

app = create_tipg_app(
postgres_settings=postgres_settings,
db_settings=db_settings,
sql_settings=sql_settings,
)

with TestClient(app) as client:
yield client


@pytest.fixture
def app_myschema_public_functions(database_url, monkeypatch):
"""Create APP with only tables from `myschema` schema and functions from `public` schema.
Expand Down
46 changes: 46 additions & 0 deletions tests/routes/test_collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,3 +281,49 @@ def test_collections_empty(app_empty):
assert response.status_code == 200
body = response.json()
assert not body["collections"]


def test_collections_no_extents(app_no_extents):
"""Test /collections endpoint."""
response = app_no_extents.get("/collections/public.landsat_wrs")
assert response.status_code == 200
body = response.json()
assert body["crs"] == ["http://www.opengis.net/def/crs/OGC/1.3/CRS84"]
assert body["extent"].get("spatial").get("bbox") == [
[
-180,
-90,
180,
90,
]
] # default value
assert not body["extent"].get("temporal")

# check a table with datetime column
response = app_no_extents.get("/collections/public.nongeo_data")
assert response.status_code == 200
body = response.json()
assert not body.get("extent")


def test_collections_no_spatial_extent(app_no_spatial_extent):
"""Test /collections endpoint."""
response = app_no_spatial_extent.get("/collections/public.canada")
assert response.status_code == 200
body = response.json()
assert body["crs"] == ["http://www.opengis.net/def/crs/OGC/1.3/CRS84"]
assert body["extent"].get("spatial").get("bbox") == [
[
-180,
-90,
180,
90,
]
]

# check a table with datetime column
response = app_no_spatial_extent.get("/collections/public.nongeo_data")
assert response.status_code == 200
body = response.json()
assert not body["extent"].get("spatial")
assert body["extent"].get("temporal")
8 changes: 7 additions & 1 deletion tipg/collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,8 @@ async def get_collection_index( # noqa: C901
exclude_functions: Optional[List[str]] = None,
exclude_function_schemas: Optional[List[str]] = None,
spatial: bool = True,
spatial_extent: bool = True,
datetime_extent: bool = True,
) -> Catalog:
"""Fetch Table and Functions index."""
schemas = schemas or ["public"]
Expand All @@ -920,7 +922,9 @@ async def get_collection_index( # noqa: C901
:functions,
:exclude_functions,
:exclude_function_schemas,
:spatial
:spatial,
:spatial_extent,
:datetime_extent
);
""" # noqa: W605

Expand All @@ -935,6 +939,8 @@ async def get_collection_index( # noqa: C901
exclude_functions=exclude_functions,
exclude_function_schemas=exclude_function_schemas,
spatial=spatial,
spatial_extent=spatial_extent,
datetime_extent=datetime_extent,
)

catalog: Dict[str, Collection] = {}
Expand Down
2 changes: 2 additions & 0 deletions tipg/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ async def lifespan(app: FastAPI):
exclude_functions=db_settings.exclude_functions,
exclude_function_schemas=db_settings.exclude_function_schemas,
spatial=db_settings.only_spatial_tables,
spatial_extent=db_settings.spatial_extent,
datetime_extent=db_settings.datetime_extent,
)

yield
Expand Down
3 changes: 2 additions & 1 deletion tipg/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ class TableSettings(BaseSettings):

fallback_key_names: List[str] = ["ogc_fid", "id", "pkey", "gid"]
table_config: Dict[str, TableConfig] = {}
datetime_extent: bool = True

model_config = {
"env_prefix": "TIPG_",
Expand Down Expand Up @@ -158,6 +157,8 @@ class DatabaseSettings(BaseSettings):
functions: Optional[List[str]] = None
exclude_functions: Optional[List[str]] = None
exclude_function_schemas: Optional[List[str]] = None
datetime_extent: bool = True
spatial_extent: bool = True

only_spatial_tables: bool = True

Expand Down
60 changes: 35 additions & 25 deletions tipg/sql/dbcatalog.sql
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ CREATE OR REPLACE FUNCTION pg_temp.tipg_pk(
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION pg_temp.tipg_properties(
att pg_attribute
att pg_attribute,
spatial_extent boolean,
datetime_extent boolean
) RETURNS jsonb AS $$
DECLARE
attname text := att.attname;
Expand All @@ -48,7 +50,7 @@ DECLARE
bounds_geom geometry;
bounds float[];
BEGIN
IF atttype IN ('timestamp', 'timestamptz') THEN
IF atttype IN ('timestamp', 'timestamptz') AND datetime_extent THEN
EXECUTE FORMAT(
$q$
SELECT to_json(min(%I::timestamptz)), to_json(max(%I::timestamptz))
Expand All @@ -62,28 +64,30 @@ BEGIN
geometry_type := postgis_typmod_type(att.atttypmod);
srid = coalesce(nullif(postgis_typmod_srid(att.atttypmod),0), 4326);

SELECT schemaname, relname, n_live_tup, n_mod_since_analyze
INTO _schemaname, _relname, _n_live_tup, _n_mod_since_analyze
FROM pg_stat_user_tables
WHERE relid = att.attrelid;
IF spatial_extent THEN
SELECT schemaname, relname, n_live_tup, n_mod_since_analyze
INTO _schemaname, _relname, _n_live_tup, _n_mod_since_analyze
FROM pg_stat_user_tables
WHERE relid = att.attrelid;

IF _n_live_tup > 0 AND _n_mod_since_analyze = 0 THEN
bounds_geom := st_setsrid(st_estimatedextent(_schemaname, _relname, attname), srid);
END IF;
IF _n_live_tup > 0 AND _n_mod_since_analyze = 0 THEN
bounds_geom := st_setsrid(st_estimatedextent(_schemaname, _relname, attname), srid);
END IF;

IF bounds_geom IS NULL THEN
IF atttype = 'geography' THEN
EXECUTE format('SELECT ST_SetSRID(ST_Extent(%I::geometry), %L) FROM %s', attname, srid, att.attrelid::regclass::text) INTO bounds_geom;
ELSE
EXECUTE format('SELECT ST_SetSRID(ST_Extent(%I), %L) FROM %s', attname, srid, att.attrelid::regclass::text) INTO bounds_geom;
IF bounds_geom IS NULL THEN
IF atttype = 'geography' THEN
EXECUTE format('SELECT ST_SetSRID(ST_Extent(%I::geometry), %L) FROM %s', attname, srid, att.attrelid::regclass::text) INTO bounds_geom;
ELSE
EXECUTE format('SELECT ST_SetSRID(ST_Extent(%I), %L) FROM %s', attname, srid, att.attrelid::regclass::text) INTO bounds_geom;
END IF;
END IF;
END IF;

IF bounds_geom IS NOT NULL THEN
IF srid != 4326 THEN
bounds_geom := st_transform(bounds_geom, 4326);
IF bounds_geom IS NOT NULL THEN
IF srid != 4326 THEN
bounds_geom := st_transform(bounds_geom, 4326);
END IF;
bounds = ARRAY[ st_xmin(bounds_geom), st_ymin(bounds_geom), st_xmax(bounds_geom), st_ymax(bounds_geom) ];
END IF;
bounds = ARRAY[ st_xmin(bounds_geom), st_ymin(bounds_geom), st_xmax(bounds_geom), st_ymax(bounds_geom) ];
END IF;
END IF;

Expand All @@ -101,11 +105,13 @@ END;
$$ LANGUAGE PLPGSQL;

CREATE OR REPLACE FUNCTION pg_temp.tipg_tproperties(
c pg_class
c pg_class,
spatial_extent boolean,
datetime_extent boolean
) RETURNS jsonb AS $$
WITH t AS (
SELECT
jsonb_agg(pg_temp.tipg_properties(a)) as properties
jsonb_agg(pg_temp.tipg_properties(a, spatial_extent, datetime_extent)) as properties
FROM
pg_attribute a
WHERE
Expand All @@ -123,9 +129,11 @@ CREATE OR REPLACE FUNCTION pg_temp.tipg_tproperties(
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION pg_temp.tipg_tproperties(
tabl text
tabl text,
spatial_extent boolean,
datetime_extent boolean
) RETURNS jsonb AS $$
SELECT pg_temp.tipg_tproperties(pg_class) FROM pg_class WHERE oid=tabl::regclass;
SELECT pg_temp.tipg_tproperties(pg_class, spatial_extent, datetime_extent) FROM pg_class WHERE oid=tabl::regclass;
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION pg_temp.tipg_fun_defaults(defaults pg_node_tree) RETURNS text[] AS $$
Expand Down Expand Up @@ -235,11 +243,13 @@ CREATE OR REPLACE FUNCTION pg_temp.tipg_catalog(
functions text[] DEFAULT NULL,
exclude_functions text[] DEFAULT NULL,
exclude_function_schemas text[] DEFAULT NULL,
spatial boolean DEFAULT FALSE
spatial boolean DEFAULT FALSE,
spatial_extent boolean DEFAULT TRUE,
datetime_extent boolean DEFAULT TRUE
) RETURNS SETOF jsonb AS $$
WITH a AS (
SELECT
pg_temp.tipg_tproperties(c) as meta
pg_temp.tipg_tproperties(c, spatial_extent, datetime_extent) as meta
FROM pg_class c, pg_temp.tipg_get_schemas(schemas,exclude_table_schemas) s
WHERE
c.relnamespace=s
Expand Down
Loading