Skip to content
This repository has been archived by the owner on May 27, 2022. It is now read-only.

Commit

Permalink
Fix WFS by reverting a commit, not really stable
Browse files Browse the repository at this point in the history
  • Loading branch information
@m-kuhn
m-kuhn committed Apr 20, 2020
1 parent 79912c1 commit aaebde2
Show file tree
Hide file tree
Showing 2 changed files with 276 additions and 0 deletions.
275 changes: 275 additions & 0 deletions recipes/qgis/patches/0001-Revert-QgsBackgroundCachedSharedData-createCache-sim.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,275 @@
From 0cce902f7dcefc15edf7c8f17c6d194a285a7b3a Mon Sep 17 00:00:00 2001
From: Matthias Kuhn <[email protected]>
Date: Mon, 20 Apr 2020 17:25:54 +0200
Subject: [PATCH] Revert "QgsBackgroundCachedSharedData::createCache():
simplify code"

This reverts commit 55ec9c8303069b41ce65d077b2c06cefc82df6a2.
---
.../wfs/qgsbackgroundcachedshareddata.cpp | 219 ++++++++++++------
1 file changed, 144 insertions(+), 75 deletions(-)

diff --git a/src/providers/wfs/qgsbackgroundcachedshareddata.cpp b/src/providers/wfs/qgsbackgroundcachedshareddata.cpp
index eb0d016130..be8292fe2b 100644
--- a/src/providers/wfs/qgsbackgroundcachedshareddata.cpp
+++ b/src/providers/wfs/qgsbackgroundcachedshareddata.cpp
@@ -31,7 +31,6 @@

#include <cpl_vsi.h>
#include <cpl_conv.h>
-#include <gdal.h>
#include <ogr_api.h>

#include <sqlite3.h>
@@ -145,6 +144,16 @@ bool QgsBackgroundCachedSharedData::getUserVisibleIdFromSpatialiteId( QgsFeature
return false;
}

+// We have an issue with GDAL 1.10 and older that is using spatialite_init() which is
+// incompatible with how the QGIS SpatiaLite provider works.
+// The symptom of the issue is the error message
+// 'Unable to Initialize SpatiaLite Metadata: no such function: InitSpatialMetadata'
+// So in that case we must use only QGIS functions to avoid the conflict
+// The difference is that in the QGIS way we have to create the template database
+// on disk, which is a slightly bit slower. But due to later caching, this is
+// not so a big deal.
+#define USE_OGR_FOR_DB_CREATION
+
static QString quotedIdentifier( QString id )
{
id.replace( '\"', QLatin1String( "\"\"" ) );
@@ -198,49 +207,109 @@ bool QgsBackgroundCachedSharedData::createCache()
if ( mDistinctSelect )
cacheFields.append( QgsField( QgsBackgroundCachedFeatureIteratorConstants::FIELD_MD5, QVariant::String, QStringLiteral( "string" ) ) );

- // Creating a SpatiaLite database can be quite slow on some file systems
- // so we create a GDAL in-memory file, and then copy it on
- // the file system.
- GDALDriverH hDrv = GDALGetDriverByName( "SQLite" );
- if ( !hDrv )
- {
- QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache." ), mComponentTranslated );
- return false;
- }
- QString vsimemFilename;
- vsimemFilename.sprintf( "/vsimem/qgis_cache_template_%p/features.sqlite", this );
- mCacheTablename = CPLGetBasename( vsimemFilename.toStdString().c_str() );
- VSIUnlink( vsimemFilename.toStdString().c_str() );
- const char *apszOptions[] = { "INIT_WITH_EPSG=NO", "SPATIALITE=YES", nullptr };
- GDALDatasetH hDS = GDALCreate( hDrv, vsimemFilename.toUtf8().constData(), 0, 0, 0, GDT_Unknown, const_cast<char **>( apszOptions ) );
- if ( !hDS )
+ bool ogrWaySuccessful = false;
+ QString fidName( QStringLiteral( "__ogc_fid" ) );
+ QString geometryFieldname( QStringLiteral( "__spatialite_geometry" ) );
+#ifdef USE_OGR_FOR_DB_CREATION
+ // Only GDAL >= 2.0 can use an alternate geometry or FID field name
+ // but QgsVectorFileWriter will refuse anyway to create a ogc_fid, so we will
+ // do it manually
+ bool useReservedNames = cacheFields.lookupField( QStringLiteral( "ogc_fid" ) ) >= 0;
+ if ( !useReservedNames )
{
- QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache." ), mComponentTranslated );
- return false;
- }
- GDALClose( hDS );
+ // Creating a SpatiaLite database can be quite slow on some file systems
+ // so we create a GDAL in-memory file, and then copy it on
+ // the file system.
+ QString vsimemFilename;
+ QStringList datasourceOptions;
+ QStringList layerOptions;
+ datasourceOptions.push_back( QStringLiteral( "INIT_WITH_EPSG=NO" ) );
+ layerOptions.push_back( QStringLiteral( "LAUNDER=NO" ) ); // to get exact matches for field names, especially regarding case
+ layerOptions.push_back( QStringLiteral( "FID=__ogc_fid" ) );
+ layerOptions.push_back( QStringLiteral( "GEOMETRY_NAME=__spatialite_geometry" ) );
+ vsimemFilename.sprintf( "/vsimem/qgis_cache_template_%p/features.sqlite", this );
+ mCacheTablename = CPLGetBasename( vsimemFilename.toStdString().c_str() );
+ VSIUnlink( vsimemFilename.toStdString().c_str() );
+ std::unique_ptr< QgsVectorFileWriter > writer = qgis::make_unique< QgsVectorFileWriter >( vsimemFilename, QString(),
+ cacheFields, QgsWkbTypes::Polygon, QgsCoordinateReferenceSystem(), QStringLiteral( "SpatiaLite" ), datasourceOptions, layerOptions );
+ if ( writer->hasError() == QgsVectorFileWriter::NoError )
+ {
+ writer.reset();
+
+ // Copy the temporary database back to disk
+ vsi_l_offset nLength = 0;
+ GByte *pabyData = VSIGetMemFileBuffer( vsimemFilename.toStdString().c_str(), &nLength, TRUE );
+ Q_ASSERT( !QFile::exists( mCacheDbname ) );
+ VSILFILE *fp = VSIFOpenL( mCacheDbname.toStdString().c_str(), "wb " );
+ if ( fp )
+ {
+ VSIFWriteL( pabyData, 1, nLength, fp );
+ VSIFCloseL( fp );
+ CPLFree( pabyData );
+ }
+ else
+ {
+ CPLFree( pabyData );
+ QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
+ return false;
+ }

- // Copy the temporary database back to disk
- vsi_l_offset nLength = 0;
- GByte *pabyData = VSIGetMemFileBuffer( vsimemFilename.toStdString().c_str(), &nLength, TRUE );
- Q_ASSERT( !QFile::exists( mCacheDbname ) );
- VSILFILE *fp = VSIFOpenL( mCacheDbname.toStdString().c_str(), "wb " );
- if ( fp )
- {
- VSIFWriteL( pabyData, 1, nLength, fp );
- VSIFCloseL( fp );
- CPLFree( pabyData );
+ ogrWaySuccessful = true;
+ }
+ else
+ {
+ // Be tolerant on failures. Some (Windows) GDAL >= 1.11 builds may
+ // not define SPATIALITE_412_OR_LATER, and thus the call to
+ // spatialite_init() may cause failures, which will require using the
+ // slower method
+ writer.reset();
+ VSIUnlink( vsimemFilename.toStdString().c_str() );
+ }
}
- else
+#endif
+ if ( !ogrWaySuccessful )
{
- CPLFree( pabyData );
- QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
- return false;
- }
+ static QMutex sMutexDBnameCreation;
+ static QByteArray sCachedDBTemplate;
+ QMutexLocker mutexDBnameCreationHolder( &sMutexDBnameCreation );
+ if ( sCachedDBTemplate.size() == 0 )
+ {
+ // Create a template SpatiaLite DB
+ QTemporaryFile tempFile;
+ tempFile.open();
+ tempFile.setAutoRemove( false );
+ tempFile.close();
+
+ QString errCause;
+ bool created = QgsProviderRegistry::instance()->createDb( QStringLiteral( "spatialite" ), tempFile.fileName(), errCause );
+ if ( !created )
+ {
+ QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
+ return false;
+ }

+ // Ingest it in a buffer
+ QFile file( tempFile.fileName() );
+ if ( file.open( QIODevice::ReadOnly ) )
+ sCachedDBTemplate = file.readAll();
+ file.close();
+ QFile::remove( tempFile.fileName() );
+ }

- QString fidName( QStringLiteral( "__ogc_fid" ) );
- QString geometryFieldname( QStringLiteral( "__spatialite_geometry" ) );
+ // Copy the in-memory template SpatiaLite DB into the target DB
+ Q_ASSERT( !QFile::exists( mCacheDbname ) );
+ QFile dbFile( mCacheDbname );
+ if ( !dbFile.open( QIODevice::WriteOnly | QIODevice::Truncate ) )
+ {
+ QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
+ return false;
+ }
+ if ( dbFile.write( sCachedDBTemplate ) < 0 )
+ {
+ QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
+ return false;
+ }
+ }

spatialite_database_unique_ptr database;
bool ret = true;
@@ -255,47 +324,47 @@ bool QgsBackgroundCachedSharedData::createCache()

( void )sqlite3_exec( database.get(), "BEGIN", nullptr, nullptr, nullptr );

- mCacheTablename = QStringLiteral( "features" );
- sql = QStringLiteral( "CREATE TABLE %1 (%2 INTEGER PRIMARY KEY" ).arg( mCacheTablename, fidName );
-
- for ( const QgsField &field : qgis::as_const( cacheFields ) )
- {
- QString type( QStringLiteral( "VARCHAR" ) );
- if ( field.type() == QVariant::Int )
- type = QStringLiteral( "INTEGER" );
- else if ( field.type() == QVariant::LongLong )
- type = QStringLiteral( "BIGINT" );
- else if ( field.type() == QVariant::Double )
- type = QStringLiteral( "REAL" );
- else if ( field.type() == QVariant::StringList )
- type = QStringLiteral( "JSONSTRINGLIST" );
-
- sql += QStringLiteral( ", %1 %2" ).arg( quotedIdentifier( field.name() ), type );
- }
- sql += QLatin1String( ")" );
- rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
- if ( rc != SQLITE_OK )
+ if ( !ogrWaySuccessful )
{
- QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
- ret = false;
- }
+ mCacheTablename = QStringLiteral( "features" );
+ sql = QStringLiteral( "CREATE TABLE %1 (%2 INTEGER PRIMARY KEY" ).arg( mCacheTablename, fidName );

- sql = QStringLiteral( "SELECT AddGeometryColumn('%1','%2',0,'POLYGON',2)" ).arg( mCacheTablename, geometryFieldname );
- rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
- if ( rc != SQLITE_OK )
- {
- QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
- ret = false;
- }
+ for ( const QgsField &field : qgis::as_const( cacheFields ) )
+ {
+ QString type( QStringLiteral( "VARCHAR" ) );
+ if ( field.type() == QVariant::Int )
+ type = QStringLiteral( "INTEGER" );
+ else if ( field.type() == QVariant::LongLong )
+ type = QStringLiteral( "BIGINT" );
+ else if ( field.type() == QVariant::Double )
+ type = QStringLiteral( "REAL" );
+
+ sql += QStringLiteral( ", %1 %2" ).arg( quotedIdentifier( field.name() ), type );
+ }
+ sql += QLatin1String( ")" );
+ rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
+ if ( rc != SQLITE_OK )
+ {
+ QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
+ ret = false;
+ }

- sql = QStringLiteral( "SELECT CreateSpatialIndex('%1','%2')" ).arg( mCacheTablename, geometryFieldname );
- rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
- if ( rc != SQLITE_OK )
- {
- QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
- ret = false;
- }
+ sql = QStringLiteral( "SELECT AddGeometryColumn('%1','%2',0,'POLYGON',2)" ).arg( mCacheTablename, geometryFieldname );
+ rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
+ if ( rc != SQLITE_OK )
+ {
+ QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
+ ret = false;
+ }

+ sql = QStringLiteral( "SELECT CreateSpatialIndex('%1','%2')" ).arg( mCacheTablename, geometryFieldname );
+ rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
+ if ( rc != SQLITE_OK )
+ {
+ QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
+ ret = false;
+ }
+ }

// We need an index on the uniqueId, since we will check for duplicates, particularly
// useful in the case we do overlapping BBOX requests
--
2.25.2

1 change: 1 addition & 0 deletions recipes/qgis/recipe.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ function prebuild_qgis() {
fi
# reintroduce with proj6 (but do not delete legacy.h file in the patch)
#patch -p1 < $RECIPE_qgis/patches/0001-Use-qrc-for-crs-mapping.patch
patch -p1 < $RECIPE_qgis/patches/0001-Revert-QgsBackgroundCachedSharedData-createCache-sim.patch

touch .patched
}
Expand Down

0 comments on commit aaebde2

Please sign in to comment.