[OGR provider] Allow opening (GeoPackage) datasets with many layers

Currently each time you instanciate a QgsOgrProvider layer, a GDAL dataset is created. In the case of GeoPackage, this means a SQLite connection and a file handle. As GDAL enables Spatialite function on GeoPackage connections, we are bound to Spatialite limits, and Spatialite has a hard limit on a maximum of 64 simultaneous connections. Thus we cannot open more than 64 layers of the same GeoPackage. This commits enables sharing of the same GDALDataset object among several QgsOgrProvider object. Care is made to reuse a GDALDataset object only if the QgsOgrProvider do not point to the same layer. Mutexes are also taken to allow safe instanciation and use of QgsOgrProvider objects from multiple threads (but a same QgsOgrProvider should not be used by more than one thread at a time)
qgis · Oct 20, 2017 · 8f3d44d · 8f3d44d
1 parent e59f1d5
commit 8f3d44d
Show file tree

Hide file tree

Showing 6 changed files with 1,678 additions and 527 deletions.
diff --git a/src/providers/ogr/qgsogrconnpool.h b/src/providers/ogr/qgsogrconnpool.h
@@ -37,7 +37,7 @@ inline void qgsConnectionPool_ConnectionCreate( const QString &connInfo, QgsOgrC
 {
   c = new QgsOgrConn;
   QString filePath = connInfo.left( connInfo.indexOf( QLatin1String( "|" ) ) );
-  c->ds = GDALOpenEx( filePath.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr );
+  c->ds = QgsOgrProviderUtils::GDALOpenWrapper( filePath.toUtf8().constData(), false, nullptr, nullptr );
   c->path = connInfo;
   c->valid = true;
 }

diff --git a/src/providers/ogr/qgsogrdataitems.cpp b/src/providers/ogr/qgsogrdataitems.cpp
@@ -54,7 +54,7 @@ QgsOgrLayerItem::QgsOgrLayerItem( QgsDataItem *parent,
 
   OGRRegisterAll();
   GDALDriverH hDriver;
-  GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), true, false, &hDriver );
+  GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), true, nullptr,  &hDriver );
 
   if ( hDataSource )
   {
@@ -408,7 +408,7 @@ QVector<QgsDataItem *> QgsOgrDataCollectionItem::createChildren()
   QVector<QgsDataItem *> children;
 
   GDALDriverH hDriver;
-  GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), false, false, &hDriver );
+  GDALDatasetH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( mPath.toUtf8().constData(), false, nullptr, &hDriver );
   if ( !hDataSource )
     return children;
   int numLayers = GDALDatasetGetLayerCount( hDataSource );
@@ -639,7 +639,7 @@ QGISEXTERN QgsDataItem *dataItem( QString path, QgsDataItem *parentItem )
   // do not print errors, but write to debug
   CPLPushErrorHandler( CPLQuietErrorHandler );
   CPLErrorReset();
-  OGRDataSourceH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( path.toUtf8().constData(), false, false, &hDriver );
+  OGRDataSourceH hDataSource = QgsOgrProviderUtils::GDALOpenWrapper( path.toUtf8().constData(), false, nullptr, &hDriver );
   CPLPopErrorHandler();
 
   if ( ! hDataSource )

diff --git a/src/providers/ogr/qgsogrfeatureiterator.cpp b/src/providers/ogr/qgsogrfeatureiterator.cpp
@@ -47,7 +47,8 @@ QgsOgrFeatureIterator::QgsOgrFeatureIterator( QgsOgrFeatureSource *source, bool
   , mFilterFids( mRequest.filterFids() )
   , mFilterFidsIt( mFilterFids.constBegin() )
 {
-  mConn = QgsOgrConnPool::instance()->acquireConnection( mSource->mDataSource );
+  //QgsDebugMsg( "Feature iterator of " + mSource->mLayerName + ": acquiring connection");
+  mConn = QgsOgrConnPool::instance()->acquireConnection( QgsOgrProviderUtils::connectionPoolId( mSource->mDataSource ) );
   if ( !mConn->ds )
   {
     return;
@@ -316,7 +317,10 @@ bool QgsOgrFeatureIterator::close()
   }
 
   if ( mConn )
+  {
+    //QgsDebugMsg( "Feature iterator of " + mSource->mLayerName + ": releasing connection");
     QgsOgrConnPool::instance()->releaseConnection( mConn );
+  }
 
   mConn = nullptr;
   ogrLayer = nullptr;
@@ -437,12 +441,12 @@ QgsOgrFeatureSource::QgsOgrFeatureSource( const QgsOgrProvider *p )
 {
   for ( int i = ( p->mFirstFieldIsFid ) ? 1 : 0; i < mFields.size(); i++ )
     mFieldsWithoutFid.append( mFields.at( i ) );
-  QgsOgrConnPool::instance()->ref( mDataSource );
+  QgsOgrConnPool::instance()->ref( QgsOgrProviderUtils::connectionPoolId( mDataSource ) );
 }
 
 QgsOgrFeatureSource::~QgsOgrFeatureSource()
 {
-  QgsOgrConnPool::instance()->unref( mDataSource );
+  QgsOgrConnPool::instance()->unref( QgsOgrProviderUtils::connectionPoolId( mDataSource ) );
 }
 
 QgsFeatureIterator QgsOgrFeatureSource::getFeatures( const QgsFeatureRequest &request )