Skip to content

Commit

Permalink
QgsBackgroundCachedSharedData::createCache(): simplify code
Browse files Browse the repository at this point in the history
  • Loading branch information
@rouault
rouault committed Jan 22, 2020
1 parent 07b64aa commit 55ec9c8
Showing 1 changed file with 73 additions and 144 deletions.
217 changes: 73 additions & 144 deletions src/providers/wfs/qgsbackgroundcachedshareddata.cpp
View file
Expand Up @@ -30,6 +30,7 @@

#include <cpl_vsi.h>
#include <cpl_conv.h>
#include <gdal.h>
#include <ogr_api.h>

#include <sqlite3.h>
Expand Down Expand Up @@ -143,16 +144,6 @@ bool QgsBackgroundCachedSharedData::getUserVisibleIdFromSpatialiteId( QgsFeature
return false;
}

// We have an issue with GDAL 1.10 and older that is using spatialite_init() which is
// incompatible with how the QGIS SpatiaLite provider works.
// The symptom of the issue is the error message
// 'Unable to Initialize SpatiaLite Metadata: no such function: InitSpatialMetadata'
// So in that case we must use only QGIS functions to avoid the conflict
// The difference is that in the QGIS way we have to create the template database
// on disk, which is a slightly bit slower. But due to later caching, this is
// not so a big deal.
#define USE_OGR_FOR_DB_CREATION

static QString quotedIdentifier( QString id )
{
id.replace( '\"', QLatin1String( "\"\"" ) );
Expand Down Expand Up @@ -202,109 +193,49 @@ bool QgsBackgroundCachedSharedData::createCache()
if ( mDistinctSelect )
cacheFields.append( QgsField( QgsBackgroundCachedFeatureIteratorConstants::FIELD_MD5, QVariant::String, QStringLiteral( "string" ) ) );

bool ogrWaySuccessful = false;
QString fidName( QStringLiteral( "__ogc_fid" ) );
QString geometryFieldname( QStringLiteral( "__spatialite_geometry" ) );
#ifdef USE_OGR_FOR_DB_CREATION
// Only GDAL >= 2.0 can use an alternate geometry or FID field name
// but QgsVectorFileWriter will refuse anyway to create a ogc_fid, so we will
// do it manually
bool useReservedNames = cacheFields.lookupField( QStringLiteral( "ogc_fid" ) ) >= 0;
if ( !useReservedNames )
// Creating a SpatiaLite database can be quite slow on some file systems
// so we create a GDAL in-memory file, and then copy it on
// the file system.
GDALDriverH hDrv = GDALGetDriverByName( "SQLite" );
if ( !hDrv )
{
// Creating a SpatiaLite database can be quite slow on some file systems
// so we create a GDAL in-memory file, and then copy it on
// the file system.
QString vsimemFilename;
QStringList datasourceOptions;
QStringList layerOptions;
datasourceOptions.push_back( QStringLiteral( "INIT_WITH_EPSG=NO" ) );
layerOptions.push_back( QStringLiteral( "LAUNDER=NO" ) ); // to get exact matches for field names, especially regarding case
layerOptions.push_back( QStringLiteral( "FID=__ogc_fid" ) );
layerOptions.push_back( QStringLiteral( "GEOMETRY_NAME=__spatialite_geometry" ) );
vsimemFilename.sprintf( "/vsimem/qgis_cache_template_%p/features.sqlite", this );
mCacheTablename = CPLGetBasename( vsimemFilename.toStdString().c_str() );
VSIUnlink( vsimemFilename.toStdString().c_str() );
std::unique_ptr< QgsVectorFileWriter > writer = qgis::make_unique< QgsVectorFileWriter >( vsimemFilename, QString(),
cacheFields, QgsWkbTypes::Polygon, QgsCoordinateReferenceSystem(), QStringLiteral( "SpatiaLite" ), datasourceOptions, layerOptions );
if ( writer->hasError() == QgsVectorFileWriter::NoError )
{
writer.reset();

// Copy the temporary database back to disk
vsi_l_offset nLength = 0;
GByte *pabyData = VSIGetMemFileBuffer( vsimemFilename.toStdString().c_str(), &nLength, TRUE );
Q_ASSERT( !QFile::exists( mCacheDbname ) );
VSILFILE *fp = VSIFOpenL( mCacheDbname.toStdString().c_str(), "wb " );
if ( fp )
{
VSIFWriteL( pabyData, 1, nLength, fp );
VSIFCloseL( fp );
CPLFree( pabyData );
}
else
{
CPLFree( pabyData );
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
return false;
}
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache." ), mComponentTranslated );
return false;
}
QString vsimemFilename;
vsimemFilename.sprintf( "/vsimem/qgis_cache_template_%p/features.sqlite", this );
mCacheTablename = CPLGetBasename( vsimemFilename.toStdString().c_str() );
VSIUnlink( vsimemFilename.toStdString().c_str() );
const char *apszOptions[] = { "INIT_WITH_EPSG=NO", "SPATIALITE=YES", nullptr };
GDALDatasetH hDS = GDALCreate( hDrv, vsimemFilename.toUtf8().constData(), 0, 0, 0, GDT_Unknown, const_cast<char **>( apszOptions ) );
if ( !hDS )
{
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache." ), mComponentTranslated );

This comment has been minimized.

Sign in to view

Copy link
@m-kuhn

m-kuhn Apr 20, 2020 •
edited

Member

@rouault on QField we are running into this message here since this commit. (and data from wfs is only rendered once, when fetched first, any subsequent data access returns nothing, likely trying to fetch data from an inexistent cache).

Do you have any pointer for what could be going wrong or how to get more error information?
For reference, we are still running GDAL 2.4, could that be the source of the issue? Or a required build option / dependency to get vsimem support?

This comment has been minimized.

Sign in to view

Copy link
@rouault

rouault Apr 20, 2020

Author Contributor

Maybe your GDAL build has SQLite support, but without linking to Spatialite ?

This comment has been minimized.

Sign in to view

Copy link
@m-kuhn

m-kuhn Apr 20, 2020

Member

Thanks for the quick reply... Indeed!

configure:26968: checking for spatialite
configure:26989: result: disabled
return false;
}
GDALClose( hDS );

ogrWaySuccessful = true;
}
else
{
// Be tolerant on failures. Some (Windows) GDAL >= 1.11 builds may
// not define SPATIALITE_412_OR_LATER, and thus the call to
// spatialite_init() may cause failures, which will require using the
// slower method
writer.reset();
VSIUnlink( vsimemFilename.toStdString().c_str() );
}
// Copy the temporary database back to disk
vsi_l_offset nLength = 0;
GByte *pabyData = VSIGetMemFileBuffer( vsimemFilename.toStdString().c_str(), &nLength, TRUE );
Q_ASSERT( !QFile::exists( mCacheDbname ) );
VSILFILE *fp = VSIFOpenL( mCacheDbname.toStdString().c_str(), "wb " );
if ( fp )
{
VSIFWriteL( pabyData, 1, nLength, fp );
VSIFCloseL( fp );
CPLFree( pabyData );
}
#endif
if ( !ogrWaySuccessful )
else
{
static QMutex sMutexDBnameCreation;
static QByteArray sCachedDBTemplate;
QMutexLocker mutexDBnameCreationHolder( &sMutexDBnameCreation );
if ( sCachedDBTemplate.size() == 0 )
{
// Create a template SpatiaLite DB
QTemporaryFile tempFile;
tempFile.open();
tempFile.setAutoRemove( false );
tempFile.close();

QString errCause;
bool created = QgsProviderRegistry::instance()->createDb( QStringLiteral( "spatialite" ), tempFile.fileName(), errCause );
if ( !created )
{
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
return false;
}
CPLFree( pabyData );
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
return false;
}

// Ingest it in a buffer
QFile file( tempFile.fileName() );
if ( file.open( QIODevice::ReadOnly ) )
sCachedDBTemplate = file.readAll();
file.close();
QFile::remove( tempFile.fileName() );
}

// Copy the in-memory template SpatiaLite DB into the target DB
Q_ASSERT( !QFile::exists( mCacheDbname ) );
QFile dbFile( mCacheDbname );
if ( !dbFile.open( QIODevice::WriteOnly | QIODevice::Truncate ) )
{
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
return false;
}
if ( dbFile.write( sCachedDBTemplate ) < 0 )
{
QgsMessageLog::logMessage( QObject::tr( "Cannot create temporary SpatiaLite cache" ), mComponentTranslated );
return false;
}
}
QString fidName( QStringLiteral( "__ogc_fid" ) );
QString geometryFieldname( QStringLiteral( "__spatialite_geometry" ) );

spatialite_database_unique_ptr database;
bool ret = true;
Expand All @@ -319,48 +250,46 @@ bool QgsBackgroundCachedSharedData::createCache()

( void )sqlite3_exec( database.get(), "BEGIN", nullptr, nullptr, nullptr );

if ( !ogrWaySuccessful )
{
mCacheTablename = QStringLiteral( "features" );
sql = QStringLiteral( "CREATE TABLE %1 (%2 INTEGER PRIMARY KEY" ).arg( mCacheTablename, fidName );
mCacheTablename = QStringLiteral( "features" );
sql = QStringLiteral( "CREATE TABLE %1 (%2 INTEGER PRIMARY KEY" ).arg( mCacheTablename, fidName );

for ( const QgsField &field : qgis::as_const( cacheFields ) )
{
QString type( QStringLiteral( "VARCHAR" ) );
if ( field.type() == QVariant::Int )
type = QStringLiteral( "INTEGER" );
else if ( field.type() == QVariant::LongLong )
type = QStringLiteral( "BIGINT" );
else if ( field.type() == QVariant::Double )
type = QStringLiteral( "REAL" );

sql += QStringLiteral( ", %1 %2" ).arg( quotedIdentifier( field.name() ), type );
}
sql += QLatin1String( ")" );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}
for ( const QgsField &field : qgis::as_const( cacheFields ) )
{
QString type( QStringLiteral( "VARCHAR" ) );
if ( field.type() == QVariant::Int )
type = QStringLiteral( "INTEGER" );
else if ( field.type() == QVariant::LongLong )
type = QStringLiteral( "BIGINT" );
else if ( field.type() == QVariant::Double )
type = QStringLiteral( "REAL" );

sql += QStringLiteral( ", %1 %2" ).arg( quotedIdentifier( field.name() ), type );
}
sql += QLatin1String( ")" );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}

sql = QStringLiteral( "SELECT AddGeometryColumn('%1','%2',0,'POLYGON',2)" ).arg( mCacheTablename, geometryFieldname );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}
sql = QStringLiteral( "SELECT AddGeometryColumn('%1','%2',0,'POLYGON',2)" ).arg( mCacheTablename, geometryFieldname );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}

sql = QStringLiteral( "SELECT CreateSpatialIndex('%1','%2')" ).arg( mCacheTablename, geometryFieldname );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}
sql = QStringLiteral( "SELECT CreateSpatialIndex('%1','%2')" ).arg( mCacheTablename, geometryFieldname );
rc = sqlite3_exec( database.get(), sql.toUtf8(), nullptr, nullptr, nullptr );
if ( rc != SQLITE_OK )
{
QgsDebugMsg( QStringLiteral( "%1 failed" ).arg( sql ) );
ret = false;
}


// We need an index on the uniqueId, since we will check for duplicates, particularly
// useful in the case we do overlapping BBOX requests
sql = QStringLiteral( "CREATE INDEX idx_%2 ON %1(%2)" ).arg( mCacheTablename, QgsBackgroundCachedFeatureIteratorConstants::FIELD_UNIQUE_ID );
Expand Down

0 comments on commit 55ec9c8

Please sign in to comment.