Skip to content

Commit 7fa11c8

Browse files
rouaultrouault
authoredNov 15, 2017
Merge pull request #5632 from rouault/gdal_shared_instance
[GDAL provider] Use same underlying GDAL dataset for clone() (fixes #16006)
·
ltr-3_40final-3_0_0
2 parents 31299b0 + 670764d commit 7fa11c8

File tree

2 files changed

+453
-41
lines changed

2 files changed

+453
-41
lines changed
 

‎src/providers/gdal/qgsgdalprovider.cpp

Lines changed: 387 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,24 @@
6464
static QString PROVIDER_KEY = QStringLiteral( "gdal" );
6565
static QString PROVIDER_DESCRIPTION = QStringLiteral( "GDAL provider" );
6666

67+
// To avoid potential races when destroying related instances ("main" and clones)
68+
static QMutex gGdaProviderMutex( QMutex::Recursive );
69+
70+
QMap< QgsGdalProvider *, QVector<QgsGdalProvider::DatasetPair> > QgsGdalProvider::mgDatasetCache;
71+
72+
int QgsGdalProvider::mgDatasetCacheSize = 0;
73+
74+
// Number of cached datasets from which we will try to do eviction when a
75+
// provider has 2 or more cached datasets
76+
const int MIN_THRESHOLD_FOR_CACHE_CLEANUP = 10;
77+
78+
// Maximum number of cached datasets
79+
// We try to keep at least 1 cached dataset per parent provider between
80+
// MIN_THRESHOLD_FOR_CACHE_CLEANUP and MAX_CACHE_SIZE. But we don't want to
81+
// maintain more than MAX_CACHE_SIZE datasets opened to avoid running short of
82+
// file descriptors.
83+
const int MAX_CACHE_SIZE = 50;
84+
6785
struct QgsGdalProgress
6886
{
6987
int type;
@@ -106,6 +124,8 @@ int CPL_STDCALL progressCallback( double dfComplete,
106124

107125
QgsGdalProvider::QgsGdalProvider( const QString &uri, const QgsError &error )
108126
: QgsRasterDataProvider( uri )
127+
, mpRefCounter( new QAtomicInt( 1 ) )
128+
, mpLightRefCounter( new QAtomicInt( 1 ) )
109129
, mUpdate( false )
110130
{
111131
mGeoTransform[0] = 0;
@@ -119,6 +139,10 @@ QgsGdalProvider::QgsGdalProvider( const QString &uri, const QgsError &error )
119139

120140
QgsGdalProvider::QgsGdalProvider( const QString &uri, bool update, GDALDatasetH dataset )
121141
: QgsRasterDataProvider( uri )
142+
, mpRefCounter( new QAtomicInt( 1 ) )
143+
, mpMutex( new QMutex( QMutex::Recursive ) )
144+
, mpParent( new QgsGdalProvider * ( this ) )
145+
, mpLightRefCounter( new QAtomicInt( 1 ) )
122146
, mUpdate( update )
123147
{
124148
mGeoTransform[0] = 0;
@@ -159,40 +183,80 @@ QgsGdalProvider::QgsGdalProvider( const QString &uri, bool update, GDALDatasetH
159183
if ( dataset )
160184
{
161185
mGdalBaseDataset = dataset;
186+
initBaseDataset();
162187
}
163188
else
164189
{
165-
// Try to open using VSIFileHandler (see qgsogrprovider.cpp)
166-
QString vsiPrefix = QgsZipItem::vsiPrefix( uri );
167-
if ( !vsiPrefix.isEmpty() )
168-
{
169-
if ( !uri.startsWith( vsiPrefix ) )
170-
setDataSourceUri( vsiPrefix + uri );
171-
QgsDebugMsg( QString( "Trying %1 syntax, uri= %2" ).arg( vsiPrefix, dataSourceUri() ) );
172-
}
190+
initIfNeeded();
191+
}
192+
}
173193

174-
QString gdalUri = dataSourceUri();
194+
QgsGdalProvider::QgsGdalProvider( const QgsGdalProvider &other )
195+
: QgsRasterDataProvider( other.dataSourceUri() )
196+
, mUpdate( false )
197+
{
198+
// The JP2OPENJPEG driver might consume too much memory on large datasets
199+
// so make sure to really use a single one.
200+
bool forceUseSameDataset =
201+
( other.mGdalBaseDataset &&
202+
GDALGetDatasetDriver( other.mGdalBaseDataset ) == GDALGetDriverByName( "JP2OPENJPEG" ) ) ||
203+
CSLTestBoolean( CPLGetConfigOption( "QGIS_GDAL_FORCE_USE_SAME_DATASET", "FALSE" ) );
175204

176-
CPLErrorReset();
177-
mGdalBaseDataset = gdalOpen( gdalUri.toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
205+
if ( forceUseSameDataset )
206+
{
207+
++ ( *other.mpRefCounter );
208+
mpRefCounter = other.mpRefCounter;
209+
mpMutex = other.mpMutex;
210+
mpLightRefCounter = new QAtomicInt( 1 );
211+
mHasInit = other.mHasInit;
212+
mValid = other.mValid;
213+
mGdalBaseDataset = other.mGdalBaseDataset;
214+
mGdalDataset = other.mGdalDataset;
215+
}
216+
else
217+
{
218+
219+
++ ( *other.mpLightRefCounter );
220+
221+
mpRefCounter = new QAtomicInt( 1 );
222+
mpLightRefCounter = other.mpLightRefCounter;
223+
mpMutex = new QMutex( QMutex::Recursive );
224+
mpParent = other.mpParent;
178225

179-
if ( !mGdalBaseDataset )
226+
if ( getCachedGdalHandles( const_cast<QgsGdalProvider *>( &other ), mGdalBaseDataset, mGdalDataset ) )
180227
{
181-
QString msg = QStringLiteral( "Cannot open GDAL dataset %1:\n%2" ).arg( dataSourceUri(), QString::fromUtf8( CPLGetLastErrorMsg() ) );
182-
appendError( ERRMSG( msg ) );
183-
return;
228+
QgsDebugMsgLevel( "recycling already opened dataset", 5 );
229+
mHasInit = true;
230+
mValid = other.mValid;
231+
}
232+
else
233+
{
234+
QgsDebugMsgLevel( "will need to open new dataset", 5 );
235+
mHasInit = false;
236+
mValid = false;
184237
}
185238

186-
QgsDebugMsg( "GdalDataset opened" );
187239
}
188-
initBaseDataset();
240+
241+
mHasPyramids = other.mHasPyramids;
242+
mGdalDataType = other.mGdalDataType;
243+
mExtent = other.mExtent;
244+
mWidth = other.mWidth;
245+
mHeight = other.mHeight;
246+
mXBlockSize = other.mXBlockSize;
247+
mYBlockSize = other.mYBlockSize;
248+
memcpy( mGeoTransform, other.mGeoTransform, sizeof( mGeoTransform ) );
249+
mCrs = other.mCrs;
250+
mPyramidList = other.mPyramidList;
251+
mSubLayers = other.mSubLayers;
252+
mMaskBandExposedAsAlpha = other.mMaskBandExposedAsAlpha;
253+
mBandCount = other.mBandCount;
254+
copyBaseSettings( other );
189255
}
190256

191257
QgsGdalProvider *QgsGdalProvider::clone() const
192258
{
193-
QgsGdalProvider *provider = new QgsGdalProvider( dataSourceUri() );
194-
provider->copyBaseSettings( *this );
195-
return provider;
259+
return new QgsGdalProvider( *this );
196260
}
197261

198262
bool QgsGdalProvider::crsFromWkt( const char *wkt )
@@ -233,15 +297,163 @@ bool QgsGdalProvider::crsFromWkt( const char *wkt )
233297
return mCrs.isValid();
234298
}
235299

236-
QgsGdalProvider::~QgsGdalProvider()
300+
bool QgsGdalProvider::getCachedGdalHandles( QgsGdalProvider *provider,
301+
GDALDatasetH &gdalBaseDataset,
302+
GDALDatasetH &gdalDataset )
237303
{
238-
if ( mGdalBaseDataset )
304+
QMutexLocker locker( &gGdaProviderMutex );
305+
306+
auto iter = mgDatasetCache.find( provider );
307+
if ( iter == mgDatasetCache.end() )
239308
{
240-
GDALDereferenceDataset( mGdalBaseDataset );
309+
return false;
241310
}
242-
if ( mGdalDataset )
311+
312+
if ( !iter.value().isEmpty() )
243313
{
244-
GDALClose( mGdalDataset );
314+
DatasetPair pair = iter.value().takeFirst();
315+
mgDatasetCacheSize --;
316+
gdalBaseDataset = pair.mGdalBaseDataset;
317+
gdalDataset = pair.mGdalDataset;
318+
return true;
319+
}
320+
return false;
321+
}
322+
323+
bool QgsGdalProvider::cacheGdalHandlesForLaterReuse( QgsGdalProvider *provider,
324+
GDALDatasetH gdalBaseDataset,
325+
GDALDatasetH gdalDataset )
326+
{
327+
QMutexLocker locker( &gGdaProviderMutex );
328+
329+
// If the cache size goes above the soft limit, try to do evict a cached
330+
// dataset for the provider that has the most cached entries
331+
if ( mgDatasetCacheSize >= MIN_THRESHOLD_FOR_CACHE_CLEANUP )
332+
{
333+
auto iter = mgDatasetCache.find( provider );
334+
if ( iter == mgDatasetCache.end() || iter.value().isEmpty() )
335+
{
336+
QgsGdalProvider *candidateProvider = nullptr;
337+
int nLargestCountOfCachedDatasets = 0;
338+
for ( iter = mgDatasetCache.begin(); iter != mgDatasetCache.end(); ++iter )
339+
{
340+
if ( iter.value().size() > nLargestCountOfCachedDatasets )
341+
{
342+
candidateProvider = iter.key();
343+
nLargestCountOfCachedDatasets = iter.value().size();
344+
}
345+
}
346+
347+
Q_ASSERT( candidateProvider );
348+
Q_ASSERT( !mgDatasetCache[ candidateProvider ].isEmpty() );
349+
350+
// If the candidate is ourselves, then do nothing
351+
if ( candidateProvider == provider )
352+
return false;
353+
354+
// If the candidate provider has at least 2 cached datasets, then
355+
// we can evict one.
356+
// In the case where providers have at most one cached dataset, then
357+
// evict one arbitrarily
358+
if ( nLargestCountOfCachedDatasets >= 2 ||
359+
mgDatasetCacheSize >= MAX_CACHE_SIZE )
360+
{
361+
mgDatasetCacheSize --;
362+
DatasetPair pair = mgDatasetCache[ candidateProvider ].takeLast();
363+
if ( pair.mGdalBaseDataset )
364+
{
365+
GDALDereferenceDataset( pair.mGdalBaseDataset );
366+
}
367+
if ( pair.mGdalDataset )
368+
{
369+
GDALClose( pair.mGdalDataset );
370+
}
371+
}
372+
}
373+
else
374+
{
375+
return false;
376+
}
377+
}
378+
379+
// Add handles to the cache
380+
auto iter = mgDatasetCache.find( provider );
381+
if ( iter == mgDatasetCache.end() )
382+
{
383+
mgDatasetCache[provider] = QVector<DatasetPair>();
384+
iter = mgDatasetCache.find( provider );
385+
}
386+
387+
mgDatasetCacheSize ++;
388+
DatasetPair pair;
389+
pair.mGdalBaseDataset = gdalBaseDataset;
390+
pair.mGdalDataset = gdalDataset;
391+
iter.value().push_back( pair );
392+
393+
return true;
394+
}
395+
396+
void QgsGdalProvider::closeCachedGdalHandlesFor( QgsGdalProvider *provider )
397+
{
398+
QMutexLocker locker( &gGdaProviderMutex );
399+
auto iter = mgDatasetCache.find( provider );
400+
if ( iter != mgDatasetCache.end() )
401+
{
402+
while ( !iter.value().isEmpty() )
403+
{
404+
mgDatasetCacheSize --;
405+
DatasetPair pair = iter.value().takeLast();
406+
if ( pair.mGdalBaseDataset )
407+
{
408+
GDALDereferenceDataset( pair.mGdalBaseDataset );
409+
}
410+
if ( pair.mGdalDataset )
411+
{
412+
GDALClose( pair.mGdalDataset );
413+
}
414+
}
415+
mgDatasetCache.erase( iter );
416+
}
417+
}
418+
419+
420+
QgsGdalProvider::~QgsGdalProvider()
421+
{
422+
QMutexLocker locker( &gGdaProviderMutex );
423+
424+
int lightRefCounter = -- ( *mpLightRefCounter );
425+
int refCounter = -- ( *mpRefCounter );
426+
if ( refCounter == 0 )
427+
{
428+
if ( mpParent && *mpParent && *mpParent != this && mGdalBaseDataset &&
429+
cacheGdalHandlesForLaterReuse( *mpParent, mGdalBaseDataset, mGdalDataset ) )
430+
{
431+
// do nothing
432+
}
433+
else
434+
{
435+
if ( mGdalBaseDataset )
436+
{
437+
GDALDereferenceDataset( mGdalBaseDataset );
438+
}
439+
if ( mGdalDataset )
440+
{
441+
GDALClose( mGdalDataset );
442+
}
443+
444+
if ( mpParent && *mpParent == this )
445+
{
446+
*mpParent = nullptr;
447+
closeCachedGdalHandlesFor( this );
448+
}
449+
}
450+
delete mpMutex;
451+
delete mpRefCounter;
452+
if ( lightRefCounter == 0 )
453+
{
454+
delete mpLightRefCounter;
455+
delete mpParent;
456+
}
245457
}
246458
}
247459

@@ -264,6 +476,9 @@ void QgsGdalProvider::closeDataset()
264476

265477
QString QgsGdalProvider::metadata()
266478
{
479+
QMutexLocker locker( mpMutex );
480+
if ( !initIfNeeded() )
481+
return QString();
267482
QString myMetadata;
268483
myMetadata += QString( GDALGetDescription( GDALGetDatasetDriver( mGdalDataset ) ) );
269484
myMetadata += QLatin1String( "<br>" );
@@ -399,6 +614,8 @@ QString QgsGdalProvider::metadata()
399614
QgsRasterBlock *QgsGdalProvider::block( int bandNo, const QgsRectangle &extent, int width, int height, QgsRasterBlockFeedback *feedback )
400615
{
401616
QgsRasterBlock *block = new QgsRasterBlock( dataType( bandNo ), width, height );
617+
if ( !initIfNeeded() )
618+
return block;
402619
if ( sourceHasNoDataValue( bandNo ) && useSourceNoDataValue( bandNo ) )
403620
{
404621
block->setNoDataValue( sourceNoDataValue( bandNo ) );
@@ -423,6 +640,10 @@ QgsRasterBlock *QgsGdalProvider::block( int bandNo, const QgsRectangle &extent,
423640

424641
void QgsGdalProvider::readBlock( int bandNo, int xBlock, int yBlock, void *block )
425642
{
643+
QMutexLocker locker( mpMutex );
644+
if ( !initIfNeeded() )
645+
return;
646+
426647
// TODO!!!: Check data alignment!!! May it happen that nearest value which
427648
// is not nearest is assigned to an output cell???
428649

@@ -440,6 +661,10 @@ void QgsGdalProvider::readBlock( int bandNo, int xBlock, int yBlock, void *block
440661

441662
void QgsGdalProvider::readBlock( int bandNo, QgsRectangle const &extent, int pixelWidth, int pixelHeight, void *block, QgsRasterBlockFeedback *feedback )
442663
{
664+
QMutexLocker locker( mpMutex );
665+
if ( !initIfNeeded() )
666+
return;
667+
443668
QgsDebugMsgLevel( "thePixelWidth = " + QString::number( pixelWidth ), 5 );
444669
QgsDebugMsgLevel( "thePixelHeight = " + QString::number( pixelHeight ), 5 );
445670
QgsDebugMsgLevel( "theExtent: " + extent.toString(), 5 );
@@ -872,6 +1097,9 @@ void QgsGdalProvider::computeMinMax( int bandNo ) const
8721097
*/
8731098
QList<QgsColorRampShader::ColorRampItem> QgsGdalProvider::colorTable( int bandNumber )const
8741099
{
1100+
QMutexLocker locker( mpMutex );
1101+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1102+
return QList<QgsColorRampShader::ColorRampItem>();
8751103
return QgsGdalProviderBase::colorTable( mGdalDataset, bandNumber );
8761104
}
8771105

@@ -903,6 +1131,10 @@ int QgsGdalProvider::ySize() const { return mHeight; }
9031131

9041132
QString QgsGdalProvider::generateBandName( int bandNumber ) const
9051133
{
1134+
QMutexLocker locker( mpMutex );
1135+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1136+
return QString();
1137+
9061138
if ( strcmp( GDALGetDriverShortName( GDALGetDatasetDriver( mGdalDataset ) ), "netCDF" ) == 0 )
9071139
{
9081140
char **GDALmetadata = GDALGetMetadata( mGdalDataset, nullptr );
@@ -971,6 +1203,10 @@ QString QgsGdalProvider::generateBandName( int bandNumber ) const
9711203

9721204
QgsRasterIdentifyResult QgsGdalProvider::identify( const QgsPointXY &point, QgsRaster::IdentifyFormat format, const QgsRectangle &boundingBox, int width, int height, int /*dpi*/ )
9731205
{
1206+
QMutexLocker locker( mpMutex );
1207+
if ( !initIfNeeded() )
1208+
return QgsRasterIdentifyResult( ERR( tr( "Cannot read data" ) ) );
1209+
9741210
QgsDebugMsg( QString( "thePoint = %1 %2" ).arg( point.x(), 0, 'g', 10 ).arg( point.y(), 0, 'g', 10 ) );
9751211

9761212
QMap<int, QVariant> results;
@@ -1058,6 +1294,10 @@ QgsRasterIdentifyResult QgsGdalProvider::identify( const QgsPointXY &point, QgsR
10581294

10591295
int QgsGdalProvider::capabilities() const
10601296
{
1297+
QMutexLocker locker( mpMutex );
1298+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1299+
return 0;
1300+
10611301
int capability = QgsRasterDataProvider::Identify
10621302
| QgsRasterDataProvider::IdentifyValue
10631303
| QgsRasterDataProvider::Size
@@ -1076,6 +1316,10 @@ int QgsGdalProvider::capabilities() const
10761316

10771317
Qgis::DataType QgsGdalProvider::sourceDataType( int bandNo ) const
10781318
{
1319+
QMutexLocker locker( mpMutex );
1320+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1321+
return dataTypeFromGdal( GDT_Byte );
1322+
10791323
if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
10801324
return dataTypeFromGdal( GDT_Byte );
10811325

@@ -1118,7 +1362,7 @@ Qgis::DataType QgsGdalProvider::sourceDataType( int bandNo ) const
11181362

11191363
Qgis::DataType QgsGdalProvider::dataType( int bandNo ) const
11201364
{
1121-
if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
1365+
if ( mMaskBandExposedAsAlpha && bandNo == mBandCount )
11221366
return dataTypeFromGdal( GDT_Byte );
11231367

11241368
if ( bandNo <= 0 || bandNo > mGdalDataType.count() ) return Qgis::UnknownDataType;
@@ -1128,6 +1372,10 @@ Qgis::DataType QgsGdalProvider::dataType( int bandNo ) const
11281372

11291373
double QgsGdalProvider::bandScale( int bandNo ) const
11301374
{
1375+
QMutexLocker locker( mpMutex );
1376+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1377+
return 1.0;
1378+
11311379
GDALRasterBandH myGdalBand = getBand( bandNo );
11321380
int bGotScale;
11331381
double myScale = GDALGetRasterScale( myGdalBand, &bGotScale );
@@ -1139,6 +1387,10 @@ double QgsGdalProvider::bandScale( int bandNo ) const
11391387

11401388
double QgsGdalProvider::bandOffset( int bandNo ) const
11411389
{
1390+
QMutexLocker locker( mpMutex );
1391+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1392+
return 0.0;
1393+
11421394
GDALRasterBandH myGdalBand = getBand( bandNo );
11431395
int bGotOffset;
11441396
double myOffset = GDALGetRasterOffset( myGdalBand, &bGotOffset );
@@ -1150,14 +1402,15 @@ double QgsGdalProvider::bandOffset( int bandNo ) const
11501402

11511403
int QgsGdalProvider::bandCount() const
11521404
{
1153-
if ( mGdalDataset )
1154-
return GDALGetRasterCount( mGdalDataset ) + ( mMaskBandExposedAsAlpha ? 1 : 0 );
1155-
else
1156-
return 1;
1405+
return mBandCount;
11571406
}
11581407

11591408
int QgsGdalProvider::colorInterpretation( int bandNo ) const
11601409
{
1410+
QMutexLocker locker( mpMutex );
1411+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
1412+
return colorInterpretationFromGdal( GCI_Undefined );
1413+
11611414
if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
11621415
return colorInterpretationFromGdal( GCI_AlphaBand );
11631416
GDALRasterBandH myGdalBand = GDALGetRasterBand( mGdalDataset, bandNo );
@@ -1231,6 +1484,10 @@ bool QgsGdalProvider::hasHistogram( int bandNo,
12311484
int sampleSize,
12321485
bool includeOutOfRange )
12331486
{
1487+
QMutexLocker locker( mpMutex );
1488+
if ( !initIfNeeded() )
1489+
return false;
1490+
12341491
QgsDebugMsg( QString( "theBandNo = %1 binCount = %2 minimum = %3 maximum = %4 sampleSize = %5" ).arg( bandNo ).arg( binCount ).arg( minimum ).arg( maximum ).arg( sampleSize ) );
12351492

12361493
// First check if cached in mHistograms
@@ -1314,6 +1571,10 @@ QgsRasterHistogram QgsGdalProvider::histogram( int bandNo,
13141571
int sampleSize,
13151572
bool includeOutOfRange, QgsRasterBlockFeedback *feedback )
13161573
{
1574+
QMutexLocker locker( mpMutex );
1575+
if ( !initIfNeeded() )
1576+
return QgsRasterHistogram();
1577+
13171578
QgsDebugMsg( QString( "theBandNo = %1 binCount = %2 minimum = %3 maximum = %4 sampleSize = %5" ).arg( bandNo ).arg( binCount ).arg( minimum ).arg( maximum ).arg( sampleSize ) );
13181579

13191580
QgsRasterHistogram myHistogram;
@@ -1467,6 +1728,8 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
14671728
const QString &resamplingMethod, QgsRaster::RasterPyramidsFormat format,
14681729
const QStringList &configOptions, QgsRasterBlockFeedback *feedback )
14691730
{
1731+
QMutexLocker locker( mpMutex );
1732+
14701733
//TODO: Consider making rasterPyramidList modifyable by this method to indicate if the pyramid exists after build attempt
14711734
//without requiring the user to rebuild the pyramid list to get the updated information
14721735

@@ -1766,6 +2029,8 @@ QList<QgsRasterPyramid> QgsGdalProvider::buildPyramidList()
17662029

17672030
QList<QgsRasterPyramid> QgsGdalProvider::buildPyramidList( QList<int> overviewList )
17682031
{
2032+
QMutexLocker locker( mpMutex );
2033+
17692034
int myWidth = mWidth;
17702035
int myHeight = mHeight;
17712036
GDALRasterBandH myGDALBand = GDALGetRasterBand( mGdalDataset, 1 ); //just use the first band
@@ -2126,6 +2391,10 @@ bool QgsGdalProvider::hasStatistics( int bandNo,
21262391
const QgsRectangle &boundingBox,
21272392
int sampleSize )
21282393
{
2394+
QMutexLocker locker( mpMutex );
2395+
if ( !initIfNeeded() )
2396+
return false;
2397+
21292398
QgsDebugMsg( QString( "theBandNo = %1 sampleSize = %2" ).arg( bandNo ).arg( sampleSize ) );
21302399

21312400
// First check if cached in mStatistics
@@ -2208,6 +2477,10 @@ bool QgsGdalProvider::hasStatistics( int bandNo,
22082477

22092478
QgsRasterBandStats QgsGdalProvider::bandStatistics( int bandNo, int stats, const QgsRectangle &boundingBox, int sampleSize, QgsRasterBlockFeedback *feedback )
22102479
{
2480+
QMutexLocker locker( mpMutex );
2481+
if ( !initIfNeeded() )
2482+
return QgsRasterBandStats();
2483+
22112484
QgsDebugMsg( QString( "theBandNo = %1 sampleSize = %2" ).arg( bandNo ).arg( sampleSize ) );
22122485

22132486
// TODO: null values set on raster layer!!!
@@ -2367,8 +2640,47 @@ QgsRasterBandStats QgsGdalProvider::bandStatistics( int bandNo, int stats, const
23672640

23682641
} // QgsGdalProvider::bandStatistics
23692642

2643+
bool QgsGdalProvider::initIfNeeded()
2644+
{
2645+
if ( mHasInit )
2646+
return mValid;
2647+
2648+
mHasInit = true;
2649+
2650+
QString gdalUri = dataSourceUri();
2651+
2652+
// Try to open using VSIFileHandler (see qgsogrprovider.cpp)
2653+
QString vsiPrefix = QgsZipItem::vsiPrefix( gdalUri );
2654+
if ( !vsiPrefix.isEmpty() )
2655+
{
2656+
if ( !gdalUri.startsWith( vsiPrefix ) )
2657+
setDataSourceUri( vsiPrefix + gdalUri );
2658+
QgsDebugMsg( QString( "Trying %1 syntax, uri= %2" ).arg( vsiPrefix, dataSourceUri() ) );
2659+
}
2660+
2661+
gdalUri = dataSourceUri();
2662+
2663+
CPLErrorReset();
2664+
mGdalBaseDataset = gdalOpen( gdalUri.toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
2665+
2666+
if ( !mGdalBaseDataset )
2667+
{
2668+
QString msg = QStringLiteral( "Cannot open GDAL dataset %1:\n%2" ).arg( dataSourceUri(), QString::fromUtf8( CPLGetLastErrorMsg() ) );
2669+
appendError( ERRMSG( msg ) );
2670+
return false;
2671+
}
2672+
2673+
QgsDebugMsg( "GdalDataset opened" );
2674+
2675+
initBaseDataset();
2676+
return mValid;
2677+
}
2678+
2679+
23702680
void QgsGdalProvider::initBaseDataset()
23712681
{
2682+
mHasInit = true;
2683+
mValid = true;
23722684
#if 0
23732685
for ( int i = 0; i < GDALGetRasterCount( mGdalBaseDataset ); i++ )
23742686
{
@@ -2441,13 +2753,15 @@ void QgsGdalProvider::initBaseDataset()
24412753

24422754
GDALClose( mGdalDataset );
24432755
mGdalDataset = nullptr;
2756+
mValid = false;
24442757
return;
24452758
}
24462759
// if there are subdatasets, leave the dataset open for subsequent queries
24472760
else
24482761
{
24492762
QgsDebugMsg( QObject::tr( "Cannot get GDAL raster band: %1" ).arg( msg ) +
24502763
QString( " but dataset has %1 subdatasets" ).arg( mSubLayers.size() ) );
2764+
mValid = false;
24512765
return;
24522766
}
24532767
}
@@ -2503,13 +2817,22 @@ void QgsGdalProvider::initBaseDataset()
25032817
mWidth = GDALGetRasterXSize( mGdalDataset );
25042818
mHeight = GDALGetRasterYSize( mGdalDataset );
25052819

2820+
// Check if the dataset has a mask band, that applies to the whole dataset
2821+
// If so then expose it as an alpha band.
2822+
int nMaskFlags = GDALGetMaskFlags( myGDALBand );
2823+
const int bandCount = GDALGetRasterCount( mGdalDataset );
2824+
if ( ( nMaskFlags == 0 && bandCount == 1 ) || nMaskFlags == GMF_PER_DATASET )
2825+
{
2826+
mMaskBandExposedAsAlpha = true;
2827+
}
2828+
2829+
mBandCount = bandCount + ( mMaskBandExposedAsAlpha ? 1 : 0 );
25062830

25072831
GDALGetBlockSize( GDALGetRasterBand( mGdalDataset, 1 ), &mXBlockSize, &mYBlockSize );
25082832
//
25092833
// Determine the nodata value and data type
25102834
//
25112835
//mValidNoDataValue = true;
2512-
const int bandCount = GDALGetRasterCount( mGdalBaseDataset );
25132836
for ( int i = 1; i <= bandCount; i++ )
25142837
{
25152838
GDALRasterBandH myGdalBand = GDALGetRasterBand( mGdalDataset, i );
@@ -2616,19 +2939,13 @@ void QgsGdalProvider::initBaseDataset()
26162939
//QgsDebugMsg( QString( "mInternalNoDataValue[%1] = %2" ).arg( i - 1 ).arg( mInternalNoDataValue[i-1] ) );
26172940
}
26182941

2619-
// Check if the dataset has a mask band, that applies to the whole dataset
2620-
// If so then expose it as an alpha band.
2621-
int nMaskFlags = GDALGetMaskFlags( myGDALBand );
2622-
if ( ( nMaskFlags == 0 && bandCount == 1 ) || nMaskFlags == GMF_PER_DATASET )
2942+
if ( mMaskBandExposedAsAlpha )
26232943
{
2624-
mMaskBandExposedAsAlpha = true;
26252944
mSrcNoDataValue.append( std::numeric_limits<double>::quiet_NaN() );
26262945
mSrcHasNoDataValue.append( false );
26272946
mUseSrcNoDataValue.append( false );
26282947
mGdalDataType.append( GDT_Byte );
26292948
}
2630-
2631-
mValid = true;
26322949
}
26332950

26342951
char **papszFromStringList( const QStringList &list )
@@ -2686,6 +3003,10 @@ QGISEXTERN QgsGdalProvider *create(
26863003

26873004
bool QgsGdalProvider::write( void *data, int band, int width, int height, int xOffset, int yOffset )
26883005
{
3006+
QMutexLocker locker( mpMutex );
3007+
if ( !initIfNeeded() )
3008+
return false;
3009+
26893010
if ( !mGdalDataset )
26903011
{
26913012
return false;
@@ -2700,6 +3021,10 @@ bool QgsGdalProvider::write( void *data, int band, int width, int height, int xO
27003021

27013022
bool QgsGdalProvider::setNoDataValue( int bandNo, double noDataValue )
27023023
{
3024+
QMutexLocker locker( mpMutex );
3025+
if ( !initIfNeeded() )
3026+
return false;
3027+
27033028
if ( !mGdalDataset )
27043029
{
27053030
return false;
@@ -2721,6 +3046,16 @@ bool QgsGdalProvider::setNoDataValue( int bandNo, double noDataValue )
27213046

27223047
bool QgsGdalProvider::remove()
27233048
{
3049+
QMutexLocker locker( mpMutex );
3050+
if ( !initIfNeeded() )
3051+
return false;
3052+
3053+
while ( *mpRefCounter != 1 )
3054+
{
3055+
QgsDebugMsg( QString( "Waiting for ref counter for %1 to drop to 1" ).arg( dataSourceUri() ) );
3056+
QThread::msleep( 100 );
3057+
}
3058+
27243059
if ( mGdalDataset )
27253060
{
27263061
GDALDriverH driver = GDALGetDatasetDriver( mGdalDataset );
@@ -2903,6 +3238,10 @@ bool QgsGdalProvider::isEditable() const
29033238

29043239
bool QgsGdalProvider::setEditable( bool enabled )
29053240
{
3241+
QMutexLocker locker( mpMutex );
3242+
if ( !initIfNeeded() )
3243+
return false;
3244+
29063245
if ( enabled == mUpdate )
29073246
return false;
29083247

@@ -2912,6 +3251,12 @@ bool QgsGdalProvider::setEditable( bool enabled )
29123251
if ( mGdalDataset != mGdalBaseDataset )
29133252
return false; // ignore the case of warped VRT for now (more complicated setup)
29143253

3254+
while ( *mpRefCounter != 1 )
3255+
{
3256+
QgsDebugMsg( QString( "Waiting for ref counter for %1 to drop to 1" ).arg( dataSourceUri() ) );
3257+
QThread::msleep( 100 );
3258+
}
3259+
29153260
closeDataset();
29163261

29173262
mUpdate = enabled;
@@ -2933,6 +3278,10 @@ bool QgsGdalProvider::setEditable( bool enabled )
29333278

29343279
GDALRasterBandH QgsGdalProvider::getBand( int bandNo ) const
29353280
{
3281+
QMutexLocker locker( mpMutex );
3282+
if ( !const_cast<QgsGdalProvider *>( this )->initIfNeeded() )
3283+
return nullptr;
3284+
29363285
if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
29373286
return GDALGetMaskBand( GDALGetRasterBand( mGdalDataset, 1 ) );
29383287
else

‎src/providers/gdal/qgsgdalprovider.h

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
#include <QMap>
3434
#include <QVector>
3535

36+
class QMutex;
37+
3638
class QgsRasterPyramid;
3739

3840
/**
@@ -76,6 +78,12 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
7678

7779
~QgsGdalProvider();
7880

81+
/**
82+
* Clone the provider.
83+
*
84+
* The underlying GDAL dataset is shared among the main provider and its
85+
* clones.
86+
*/
7987
QgsGdalProvider *clone() const override;
8088

8189
QString name() const override;
@@ -142,9 +150,6 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
142150
QgsRasterBlockFeedback *feedback = nullptr ) override;
143151
QList<QgsRasterPyramid> buildPyramidList( QList<int> overviewList = QList<int>() ) override;
144152

145-
//! \brief Close data set and release related data
146-
void closeDataset();
147-
148153
static QMap<QString, QString> supportedMimes();
149154

150155
bool isEditable() const override;
@@ -158,6 +163,37 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
158163
QString validatePyramidsConfigOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
159164
const QStringList &configOptions, const QString &fileFormat ) override;
160165
private:
166+
167+
QgsGdalProvider( const QgsGdalProvider &other );
168+
169+
//! Whether mGdalDataset and mGdalBaseDataset have been attempted to be set
170+
bool mHasInit = false;
171+
172+
//! Open mGdalDataset/mGdalBaseDataset if needed.
173+
bool initIfNeeded();
174+
175+
// There are 2 cloning mechanisms.
176+
// * Either the cloned provider use the same GDAL handles as the main provider
177+
// instance, in which case *mpRefCounter is used to count how many providers
178+
// use the main provider. And *mpMutex is used to protect access to the
179+
// GDAL resource.
180+
// * Or the cloned provider use its own GDAL handles, but with a cache mechanism
181+
// to avoid constant opening/closing of datasets. In that case *mpParent is
182+
// used to point to the main provider, and *mpLightRefCounter to count how
183+
// many providers point to that main provider.
184+
185+
// reference counter to know how many main and shared provider instances are linked
186+
QAtomicInt *mpRefCounter = nullptr;
187+
188+
// mutex to protect access to mGdalDataset among main and shared provider instances
189+
QMutex *mpMutex = nullptr;
190+
191+
// pointer to a QgsGdalProvider* that is the parent. Note when *mpParent == this, we are the parent.
192+
QgsGdalProvider **mpParent = nullptr;
193+
194+
// reference counter to know how many main and related provider instances are linked
195+
mutable QAtomicInt *mpLightRefCounter = nullptr;
196+
161197
// update mode
162198
bool mUpdate;
163199

@@ -187,6 +223,7 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
187223
int mHeight = 0;
188224
int mXBlockSize = 0;
189225
int mYBlockSize = 0;
226+
int mBandCount = 1;
190227

191228
//mutable QList<bool> mMinMaxComputed;
192229

@@ -217,6 +254,32 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
217254

218255
//! Wrapper for GDALGetRasterBand() that takes into account mMaskBandExposedAsAlpha.
219256
GDALRasterBandH getBand( int bandNo ) const;
257+
258+
//! \brief Close data set and release related data
259+
void closeDataset();
260+
261+
//! Pair of GDAL base dataset and "real" dataset handles.
262+
struct DatasetPair
263+
{
264+
GDALDatasetH mGdalBaseDataset;
265+
GDALDatasetH mGdalDataset;
266+
};
267+
268+
// Dataset cache
269+
static QMap< QgsGdalProvider *, QVector<DatasetPair> > mgDatasetCache;
270+
271+
// Number of cached datasets in mgDatasetCache ( == sum(iter.value().size() )
272+
static int mgDatasetCacheSize;
273+
274+
//! Add handles to the cache if possible for the specified parent provider, in which case true is returned. If false returned, then the handles should be processed appropriately by the caller
275+
static bool cacheGdalHandlesForLaterReuse( QgsGdalProvider *provider, GDALDatasetH gdalBaseDataset, GDALDatasetH gdalDataset );
276+
277+
//! Get cached handles for the specified provider, in which case true is returned and 2 handles are set.
278+
static bool getCachedGdalHandles( QgsGdalProvider *provider, GDALDatasetH &gdalBaseDataset, GDALDatasetH &gdalDataset );
279+
280+
//! Close all cached dataset for the specified provider.
281+
static void closeCachedGdalHandlesFor( QgsGdalProvider *provider );
282+
220283
};
221284

222285
#endif

0 commit comments

Comments
 (0)
Please sign in to comment.