Skip to content

Commit

Permalink
Merge pull request #6627 from OSGeo/backport-6623-to-release/3.6
Browse files Browse the repository at this point in the history 
[Backport release/3.6] GPKG: add compatibility with GPKG 1.0 gpkg_data_column_constraints table
  • Loading branch information
@rouault
rouault authored Nov 6, 2022
2 parents f987d20 + bb05d40 commit b7f4bb1
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 29 deletions.
48 changes: 48 additions & 0 deletions autotest/ogr/ogr_gpkg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6063,6 +6063,54 @@ def test_ogr_gpkg_field_domains_errors():
gdal.Unlink(filename)


###############################################################################
# Test gpkg_data_column_constraints of GPKG 1.0


def test_ogr_gpkg_field_domain_gpkg_1_0():

filename = "/vsimem/test.gpkg"

ds = gdal.GetDriverByName("GPKG").Create(
filename, 0, 0, 0, gdal.GDT_Unknown, options=["VERSION=1.0"]
)
ds.CreateLayer("test")
assert ds.AddFieldDomain(
ogr.CreateRangeFieldDomain(
"range_domain_int",
"my desc",
ogr.OFTReal,
ogr.OFSTNone,
1.5,
True,
2.5,
False,
)
)
ds = None

assert validate(filename)

ds = gdal.OpenEx(filename, gdal.OF_VECTOR)

gdal.ErrorReset()
domain = ds.GetFieldDomain("range_domain_int")
assert gdal.GetLastErrorMsg() == ""
assert domain is not None
assert domain.GetName() == "range_domain_int"
assert domain.GetDescription() == "my desc"
assert domain.GetDomainType() == ogr.OFDT_RANGE
assert domain.GetFieldType() == ogr.OFTReal
assert domain.GetMinAsDouble() == 1.5
assert domain.IsMinInclusive()
assert domain.GetMaxAsDouble() == 2.5
assert not domain.IsMaxInclusive()

ds = None

gdal.Unlink(filename)


###############################################################################
# Test attribute and spatial views

Expand Down
1 change: 1 addition & 0 deletions ogr/ogrsf_frmts/gpkg/ogr_geopackage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ class GDALGeoPackageDataset final : public OGRSQLiteBaseDataSource, public GDALG

bool HasDataColumnsTable() const;
bool HasDataColumnConstraintsTable() const;
bool HasDataColumnConstraintsTableGPKG_1_0() const;
bool CreateColumnsTableAndColumnConstraintsTablesIfNecessary();
bool HasGpkgextRelationsTable() const;
bool HasQGISLayerStyles() const;
Expand Down
67 changes: 54 additions & 13 deletions ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3497,6 +3497,29 @@ bool GDALGeoPackageDataset::HasDataColumnConstraintsTable() const
return nCount == 1;
}

/************************************************************************/
/* HasDataColumnConstraintsTableGPKG_1_0() */
/************************************************************************/

bool GDALGeoPackageDataset::HasDataColumnConstraintsTableGPKG_1_0() const
{
if( m_nApplicationId != GP10_APPLICATION_ID )
return false;
// In GPKG 1.0, the columns were named minIsInclusive, maxIsInclusive
// They were changed in 1.1 to min_is_inclusive, max_is_inclusive
bool bRet = false;
sqlite3_stmt* hSQLStmt = nullptr;
int rc = sqlite3_prepare_v2( hDB,
"SELECT minIsInclusive, maxIsInclusive FROM gpkg_data_column_constraints", -1,
&hSQLStmt, nullptr );
if( rc == SQLITE_OK )
{
bRet = true;
sqlite3_finalize(hSQLStmt);
}
return bRet;
}

/************************************************************************/
/* CreateColumnsTableAndColumnConstraintsTablesIfNecessary() */
/************************************************************************/
Expand Down Expand Up @@ -3525,18 +3548,21 @@ bool GDALGeoPackageDataset::CreateColumnsTableAndColumnConstraintsTablesIfNecess
}
if( !HasDataColumnConstraintsTable() )
{
if( OGRERR_NONE != SQLCommand(GetDB(),
"CREATE TABLE gpkg_data_column_constraints ("
const char* min_is_inclusive = m_nApplicationId != GP10_APPLICATION_ID ? "min_is_inclusive": "minIsInclusive";
const char* max_is_inclusive = m_nApplicationId != GP10_APPLICATION_ID ? "max_is_inclusive": "maxIsInclusive";

const std::string osSQL(CPLSPrintf("CREATE TABLE gpkg_data_column_constraints ("
"constraint_name TEXT NOT NULL,"
"constraint_type TEXT NOT NULL,"
"value TEXT,"
"min NUMERIC,"
"min_is_inclusive BOOLEAN,"
"%s BOOLEAN,"
"max NUMERIC,"
"max_is_inclusive BOOLEAN,"
"%s BOOLEAN,"
"description TEXT,"
"CONSTRAINT gdcc_ntv UNIQUE (constraint_name, "
"constraint_type, value));") )
"constraint_type, value));", min_is_inclusive, max_is_inclusive));
if( OGRERR_NONE != SQLCommand(GetDB(), osSQL.c_str()) )
{
return false;
}
Expand Down Expand Up @@ -8045,21 +8071,26 @@ const OGRFieldDomain* GDALGeoPackageDataset::GetFieldDomain(const std::string& n
if( !HasDataColumnConstraintsTable() )
return nullptr;

const bool bIsGPKG10 = HasDataColumnConstraintsTableGPKG_1_0();
const char* min_is_inclusive = bIsGPKG10 ? "minIsInclusive" : "min_is_inclusive";
const char* max_is_inclusive = bIsGPKG10 ? "maxIsInclusive" : "max_is_inclusive";

std::unique_ptr<SQLResult> oResultTable;
// Note: for coded domains, we use a little trick by using a dummy
// _{domainname}_domain_description enum that has a single entry whose
// description is the description of the main domain.
{
char* pszSQL = sqlite3_mprintf(
"SELECT constraint_type, value, min, min_is_inclusive, "
"max, max_is_inclusive, description, constraint_name "
"SELECT constraint_type, value, min, %s, "
"max, %s, description, constraint_name "
"FROM gpkg_data_column_constraints "
"WHERE constraint_name IN ('%q', '_%q_domain_description') "
"AND length(constraint_type) < 100 " // to avoid denial of service
"AND (value IS NULL OR length(value) < 10000) " // to avoid denial of service
"AND (description IS NULL OR length(description) < 10000) " // to avoid denial of service
"ORDER BY value "
"LIMIT 10000", // to avoid denial of service
min_is_inclusive, max_is_inclusive,
name.c_str(), name.c_str());
oResultTable = SQLQuery(hDB, pszSQL);
sqlite3_free(pszSQL);
Expand Down Expand Up @@ -8347,6 +8378,10 @@ bool GDALGeoPackageDataset::AddFieldDomain(std::unique_ptr<OGRFieldDomain>&& dom
if( !CreateColumnsTableAndColumnConstraintsTablesIfNecessary() )
return false;

const bool bIsGPKG10 = HasDataColumnConstraintsTableGPKG_1_0();
const char* min_is_inclusive = bIsGPKG10 ? "minIsInclusive" : "min_is_inclusive";
const char* max_is_inclusive = bIsGPKG10 ? "maxIsInclusive" : "max_is_inclusive";

const auto& osDescription = domain->GetDescription();
switch( domain->GetDomainType() )
{
Expand All @@ -8363,10 +8398,12 @@ bool GDALGeoPackageDataset::AddFieldDomain(std::unique_ptr<OGRFieldDomain>&& dom
char* pszSQL = sqlite3_mprintf(
"INSERT INTO gpkg_data_column_constraints ("
"constraint_name, constraint_type, value, "
"min, min_is_inclusive, max, max_is_inclusive, "
"min, %s, max, %s, "
"description) VALUES ("
"'_%q_domain_description', 'enum', '', NULL, NULL, NULL, "
"NULL, %Q)",
min_is_inclusive,
max_is_inclusive,
domainName.c_str(),
osDescription.c_str());
CPL_IGNORE_RET_VAL(SQLCommand(hDB, pszSQL));
Expand All @@ -8378,9 +8415,11 @@ bool GDALGeoPackageDataset::AddFieldDomain(std::unique_ptr<OGRFieldDomain>&& dom
char* pszSQL = sqlite3_mprintf(
"INSERT INTO gpkg_data_column_constraints ("
"constraint_name, constraint_type, value, "
"min, min_is_inclusive, max, max_is_inclusive, "
"min, %s, max, %s, "
"description) VALUES ("
"'%q', 'enum', '%q', NULL, NULL, NULL, NULL, %Q)",
min_is_inclusive,
max_is_inclusive,
domainName.c_str(),
enumeration[i].pszCode,
enumeration[i].pszValue);
Expand Down Expand Up @@ -8435,11 +8474,11 @@ bool GDALGeoPackageDataset::AddFieldDomain(std::unique_ptr<OGRFieldDomain>&& dom
}

sqlite3_stmt* hInsertStmt = nullptr;
const char* pszSQL = "INSERT INTO gpkg_data_column_constraints ("
const char* pszSQL = CPLSPrintf("INSERT INTO gpkg_data_column_constraints ("
"constraint_name, constraint_type, value, "
"min, min_is_inclusive, max, max_is_inclusive, "
"min, %s, max, %s, "
"description) VALUES ("
"?, 'range', NULL, ?, ?, ?, ?, ?)";
"?, 'range', NULL, ?, ?, ?, ?, ?)", min_is_inclusive, max_is_inclusive);
if ( sqlite3_prepare_v2(hDB, pszSQL, -1, &hInsertStmt, nullptr)
!= SQLITE_OK )
{
Expand Down Expand Up @@ -8482,9 +8521,11 @@ bool GDALGeoPackageDataset::AddFieldDomain(std::unique_ptr<OGRFieldDomain>&& dom
char* pszSQL = sqlite3_mprintf(
"INSERT INTO gpkg_data_column_constraints ("
"constraint_name, constraint_type, value, "
"min, min_is_inclusive, max, max_is_inclusive, "
"min, %s, max, %s, "
"description) VALUES ("
"'%q', 'glob', '%q', NULL, NULL, NULL, NULL, %Q)",
min_is_inclusive,
max_is_inclusive,
domainName.c_str(),
poGlobDomain->GetGlob().c_str(),
osDescription.empty() ? nullptr : osDescription.c_str());
Expand Down
52 changes: 36 additions & 16 deletions swig/python/gdal-utils/osgeo_utils/samples/validate_gpkg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,7 @@ def _check_vector_user_table(self, c, table_name):
"SELECT %s FROM %s " % (_esc_id(geom_column_name), _esc_id(table_name))
)
found_geom_types = set()
warning_messages = set()
for (blob,) in c.fetchall():
if blob is None:
continue
Expand All @@ -788,15 +789,14 @@ def _check_vector_user_table(self, c, table_name):
)
endian_prefix = ">" if big_endian else "<"
geom_srs_id = struct.unpack((endian_prefix + "I") * 1, blob[4:8])[0]
self._assert(
srs_id == geom_srs_id,
33,
(
if srs_id != geom_srs_id:
warning_msg = (
"table %s has geometries with SRID %d, "
+ "whereas only %d is expected"
)
% (table_name, geom_srs_id, srs_id),
)
) % (table_name, geom_srs_id, srs_id)
if warning_msg not in warning_messages:
warning_messages.add(warning_msg)
self._assert(False, 33, warning_msg)

self._assert(
not (empty_flag and env_ind != 0), 152, "Invalid empty geometry"
Expand Down Expand Up @@ -878,8 +878,8 @@ def _check_vector_user_table(self, c, table_name):
self._assert(
not found_geom_types or found_geom_types == set([geometry_type_name]),
32,
"in table %s, found geometry types %s"
% (table_name, str(found_geom_types)),
"in table %s, found geometry types %s whereas %s was expected"
% (table_name, str(found_geom_types), geometry_type_name),
)
elif geometry_type_name == "GEOMETRYCOLLECTION":
self._assert(
Expand Down Expand Up @@ -2006,6 +2006,8 @@ def _check_gpkg_extensions(self, c):
]
for geom_name in GPKGChecker.EXT_GEOM_TYPES:
KNOWN_EXTENSIONS += ["gpkg_geom_" + geom_name]
if self.version < (1, 2):
KNOWN_EXTENSIONS += ["gpkg_geometry_type_trigger", "gpkg_srs_id_trigger"]

for (extension_name,) in rows:

Expand Down Expand Up @@ -2440,14 +2442,24 @@ def _check_schema(self, c):

c.execute("PRAGMA table_info(gpkg_data_column_constraints)")
columns = c.fetchall()

# GPKG 1.1 uses min_is_inclusive/max_is_inclusive but GPKG 1.0 had
# minIsInclusive/maxIsInclusive
min_is_inclusive = (
"min_is_inclusive" if self.version >= (1, 1) else "minIsInclusive"
)
max_is_inclusive = (
"max_is_inclusive" if self.version >= (1, 1) else "maxIsInclusive"
)

expected_columns = [
(0, "constraint_name", "TEXT", 1, None, 0),
(1, "constraint_type", "TEXT", 1, None, 0),
(2, "value", "TEXT", 0, None, 0),
(3, "min", "NUMERIC", 0, None, 0),
(4, "min_is_inclusive", "BOOLEAN", 0, None, 0),
(4, min_is_inclusive, "BOOLEAN", 0, None, 0),
(5, "max", "NUMERIC", 0, None, 0),
(6, "max_is_inclusive", "BOOLEAN", 0, None, 0),
(6, max_is_inclusive, "BOOLEAN", 0, None, 0),
(7, "description", "TEXT", 0, None, 0),
]
self._check_structure(
Expand Down Expand Up @@ -2531,7 +2543,7 @@ def _check_schema(self, c):

c.execute(
"SELECT 1 FROM gpkg_data_column_constraints WHERE "
+ "constraint_type = 'range' AND min_is_inclusive NOT IN (0,1)"
+ f"constraint_type = 'range' AND {min_is_inclusive} NOT IN (0,1)"
)
if c.fetchone() is not None:
self._assert(
Expand All @@ -2544,7 +2556,7 @@ def _check_schema(self, c):

c.execute(
"SELECT 1 FROM gpkg_data_column_constraints WHERE "
+ "constraint_type = 'range' AND max_is_inclusive NOT IN (0,1)"
+ f"constraint_type = 'range' AND {max_is_inclusive} NOT IN (0,1)"
)
if c.fetchone() is not None:
self._assert(
Expand All @@ -2555,7 +2567,7 @@ def _check_schema(self, c):
+ "not 0 or 1",
)

for col_name in ("min", "min_is_inclusive", "max", "max_is_inclusive"):
for col_name in ("min", min_is_inclusive, "max", max_is_inclusive):
c.execute(
"SELECT 1 FROM gpkg_data_column_constraints WHERE "
+ "constraint_type IN ('enum', 'glob') AND "
Expand Down Expand Up @@ -2742,8 +2754,11 @@ def check(self):
("Wrong application_id: %s. " + "Expected one of GP10, GP11, GPKG")
% str(application_id),
)

if application_id == gpkg:
if application_id == gp10:
self.version = (1, 0)
elif application_id == gp11:
self.version = (1, 1)
elif application_id == gpkg:
f.seek(60, 0)
user_version = f.read(4)
expected_version = 10200
Expand All @@ -2754,6 +2769,11 @@ def check(self):
"Wrong user_version: %d. Expected >= %d"
% (user_version, expected_version),
)
self.version = (
expected_version // 10000,
(expected_version % 10000) // 100,
expected_version % 100,
)

conn = sqlite3.connect(":memory:")
c = conn.cursor()
Expand Down

0 comments on commit b7f4bb1

Please sign in to comment.