Skip to content

Commit

Permalink
[GDAL provider] Use same underlying GDAL dataset for clone() (fixes #…
Browse files Browse the repository at this point in the history 
…16006)

This will improve efficiency of GDAL block cache during raster rendering,
since the cached blocks are only valid during GDALDataset instance lifetime.
  • Loading branch information
@rouault
rouault committed Nov 14, 2017
1 parent c09c96b commit 3b2b98b
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 7 deletions.
107 changes: 103 additions & 4 deletions src/providers/gdal/qgsgdalprovider.cpp
View file
Expand Up @@ -106,6 +106,7 @@ int CPL_STDCALL progressCallback( double dfComplete,

QgsGdalProvider::QgsGdalProvider( const QString &uri, const QgsError &error )
: QgsRasterDataProvider( uri )
, mpRefCounter( new int( 1 ) )
, mUpdate( false )
{
mGeoTransform[0] = 0;
Expand All @@ -119,6 +120,8 @@ QgsGdalProvider::QgsGdalProvider( const QString &uri, const QgsError &error )

QgsGdalProvider::QgsGdalProvider( const QString &uri, bool update, GDALDatasetH dataset )
: QgsRasterDataProvider( uri )
, mpRefCounter( new int( 1 ) )
, mpMutex( new QMutex( QMutex::Recursive ) )
, mUpdate( update )
{
mGeoTransform[0] = 0;
Expand Down Expand Up @@ -188,11 +191,42 @@ QgsGdalProvider::QgsGdalProvider( const QString &uri, bool update, GDALDatasetH
initBaseDataset();
}

QgsGdalProvider::QgsGdalProvider( const QgsGdalProvider &other )
: QgsRasterDataProvider( other.dataSourceUri() )
, mpRefCounter( other.mpRefCounter )
, mpMutex( other.mpMutex )
, mUpdate( false )
{
QMutexLocker locker( mpMutex );
( *mpRefCounter ) ++;

mValid = other.mValid;
mHasPyramids = other.mHasPyramids;
mGdalDataType = other.mGdalDataType;
mExtent = other.mExtent;
mWidth = other.mWidth;
mHeight = other.mHeight;
mXBlockSize = other.mXBlockSize;
mYBlockSize = other.mYBlockSize;
mGdalBaseDataset = other.mGdalBaseDataset;
mGdalDataset = other.mGdalDataset;
memcpy( mGeoTransform, other.mGeoTransform, sizeof( mGeoTransform ) );
mCrs = other.mCrs;
mPyramidList = other.mPyramidList;
mSubLayers = other.mSubLayers;
mMaskBandExposedAsAlpha = other.mMaskBandExposedAsAlpha;
copyBaseSettings( other );
}

QgsGdalProvider *QgsGdalProvider::clone() const
{
#if 0
QgsGdalProvider *provider = new QgsGdalProvider( dataSourceUri() );
provider->copyBaseSettings( *this );
return provider;
#else
return new QgsGdalProvider( *this );
#endif
}

bool QgsGdalProvider::crsFromWkt( const char *wkt )
Expand Down Expand Up @@ -235,13 +269,23 @@ bool QgsGdalProvider::crsFromWkt( const char *wkt )

QgsGdalProvider::~QgsGdalProvider()
{
if ( mGdalBaseDataset )
int refCounter;
{
GDALDereferenceDataset( mGdalBaseDataset );
QMutexLocker locker( mpMutex );
refCounter = -- ( *mpRefCounter );
}
if ( mGdalDataset )
if ( refCounter == 0 )
{
GDALClose( mGdalDataset );
if ( mGdalBaseDataset )
{
GDALDereferenceDataset( mGdalBaseDataset );
}
if ( mGdalDataset )
{
GDALClose( mGdalDataset );
}
delete mpMutex;
delete mpRefCounter;
}
}

Expand All @@ -264,6 +308,7 @@ void QgsGdalProvider::closeDataset()

QString QgsGdalProvider::metadata()
{
QMutexLocker locker( mpMutex );
QString myMetadata;
myMetadata += QString( GDALGetDescription( GDALGetDatasetDriver( mGdalDataset ) ) );
myMetadata += QLatin1String( "<br>" );
Expand Down Expand Up @@ -423,6 +468,8 @@ QgsRasterBlock *QgsGdalProvider::block( int bandNo, const QgsRectangle &extent,

void QgsGdalProvider::readBlock( int bandNo, int xBlock, int yBlock, void *block )
{
QMutexLocker locker( mpMutex );

// TODO!!!: Check data alignment!!! May it happen that nearest value which
// is not nearest is assigned to an output cell???

Expand All @@ -440,6 +487,8 @@ void QgsGdalProvider::readBlock( int bandNo, int xBlock, int yBlock, void *block

void QgsGdalProvider::readBlock( int bandNo, QgsRectangle const &extent, int pixelWidth, int pixelHeight, void *block, QgsRasterBlockFeedback *feedback )
{
QMutexLocker locker( mpMutex );

QgsDebugMsgLevel( "thePixelWidth = " + QString::number( pixelWidth ), 5 );
QgsDebugMsgLevel( "thePixelHeight = " + QString::number( pixelHeight ), 5 );
QgsDebugMsgLevel( "theExtent: " + extent.toString(), 5 );
Expand Down Expand Up @@ -903,6 +952,8 @@ int QgsGdalProvider::ySize() const { return mHeight; }

QString QgsGdalProvider::generateBandName( int bandNumber ) const
{
QMutexLocker locker( mpMutex );

if ( strcmp( GDALGetDriverShortName( GDALGetDatasetDriver( mGdalDataset ) ), "netCDF" ) == 0 )
{
char **GDALmetadata = GDALGetMetadata( mGdalDataset, nullptr );
Expand Down Expand Up @@ -1058,6 +1109,8 @@ QgsRasterIdentifyResult QgsGdalProvider::identify( const QgsPointXY &point, QgsR

int QgsGdalProvider::capabilities() const
{
QMutexLocker locker( mpMutex );

int capability = QgsRasterDataProvider::Identify
| QgsRasterDataProvider::IdentifyValue
| QgsRasterDataProvider::Size
Expand All @@ -1076,6 +1129,8 @@ int QgsGdalProvider::capabilities() const

Qgis::DataType QgsGdalProvider::sourceDataType( int bandNo ) const
{
QMutexLocker locker( mpMutex );

if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
return dataTypeFromGdal( GDT_Byte );

Expand Down Expand Up @@ -1118,6 +1173,8 @@ Qgis::DataType QgsGdalProvider::sourceDataType( int bandNo ) const

Qgis::DataType QgsGdalProvider::dataType( int bandNo ) const
{
QMutexLocker locker( mpMutex );

if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
return dataTypeFromGdal( GDT_Byte );

Expand All @@ -1128,6 +1185,8 @@ Qgis::DataType QgsGdalProvider::dataType( int bandNo ) const

double QgsGdalProvider::bandScale( int bandNo ) const
{
QMutexLocker locker( mpMutex );

GDALRasterBandH myGdalBand = getBand( bandNo );
int bGotScale;
double myScale = GDALGetRasterScale( myGdalBand, &bGotScale );
Expand All @@ -1139,6 +1198,8 @@ double QgsGdalProvider::bandScale( int bandNo ) const

double QgsGdalProvider::bandOffset( int bandNo ) const
{
QMutexLocker locker( mpMutex );

GDALRasterBandH myGdalBand = getBand( bandNo );
int bGotOffset;
double myOffset = GDALGetRasterOffset( myGdalBand, &bGotOffset );
Expand All @@ -1150,6 +1211,8 @@ double QgsGdalProvider::bandOffset( int bandNo ) const

int QgsGdalProvider::bandCount() const
{
QMutexLocker locker( mpMutex );

if ( mGdalDataset )
return GDALGetRasterCount( mGdalDataset ) + ( mMaskBandExposedAsAlpha ? 1 : 0 );
else
Expand All @@ -1158,6 +1221,8 @@ int QgsGdalProvider::bandCount() const

int QgsGdalProvider::colorInterpretation( int bandNo ) const
{
QMutexLocker locker( mpMutex );

if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
return colorInterpretationFromGdal( GCI_AlphaBand );
GDALRasterBandH myGdalBand = GDALGetRasterBand( mGdalDataset, bandNo );
Expand Down Expand Up @@ -1231,6 +1296,8 @@ bool QgsGdalProvider::hasHistogram( int bandNo,
int sampleSize,
bool includeOutOfRange )
{
QMutexLocker locker( mpMutex );

QgsDebugMsg( QString( "theBandNo = %1 binCount = %2 minimum = %3 maximum = %4 sampleSize = %5" ).arg( bandNo ).arg( binCount ).arg( minimum ).arg( maximum ).arg( sampleSize ) );

// First check if cached in mHistograms
Expand Down Expand Up @@ -1314,6 +1381,8 @@ QgsRasterHistogram QgsGdalProvider::histogram( int bandNo,
int sampleSize,
bool includeOutOfRange, QgsRasterBlockFeedback *feedback )
{
QMutexLocker locker( mpMutex );

QgsDebugMsg( QString( "theBandNo = %1 binCount = %2 minimum = %3 maximum = %4 sampleSize = %5" ).arg( bandNo ).arg( binCount ).arg( minimum ).arg( maximum ).arg( sampleSize ) );

QgsRasterHistogram myHistogram;
Expand Down Expand Up @@ -1467,6 +1536,8 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
const QString &resamplingMethod, QgsRaster::RasterPyramidsFormat format,
const QStringList &configOptions, QgsRasterBlockFeedback *feedback )
{
QMutexLocker locker( mpMutex );

//TODO: Consider making rasterPyramidList modifyable by this method to indicate if the pyramid exists after build attempt
//without requiring the user to rebuild the pyramid list to get the updated information

Expand Down Expand Up @@ -1766,6 +1837,8 @@ QList<QgsRasterPyramid> QgsGdalProvider::buildPyramidList()

QList<QgsRasterPyramid> QgsGdalProvider::buildPyramidList( QList<int> overviewList )
{
QMutexLocker locker( mpMutex );

int myWidth = mWidth;
int myHeight = mHeight;
GDALRasterBandH myGDALBand = GDALGetRasterBand( mGdalDataset, 1 ); //just use the first band
Expand Down Expand Up @@ -2126,6 +2199,8 @@ bool QgsGdalProvider::hasStatistics( int bandNo,
const QgsRectangle &boundingBox,
int sampleSize )
{
QMutexLocker locker( mpMutex );

QgsDebugMsg( QString( "theBandNo = %1 sampleSize = %2" ).arg( bandNo ).arg( sampleSize ) );

// First check if cached in mStatistics
Expand Down Expand Up @@ -2208,6 +2283,8 @@ bool QgsGdalProvider::hasStatistics( int bandNo,

QgsRasterBandStats QgsGdalProvider::bandStatistics( int bandNo, int stats, const QgsRectangle &boundingBox, int sampleSize, QgsRasterBlockFeedback *feedback )
{
QMutexLocker locker( mpMutex );

QgsDebugMsg( QString( "theBandNo = %1 sampleSize = %2" ).arg( bandNo ).arg( sampleSize ) );

// TODO: null values set on raster layer!!!
Expand Down Expand Up @@ -2686,6 +2763,8 @@ QGISEXTERN QgsGdalProvider *create(

bool QgsGdalProvider::write( void *data, int band, int width, int height, int xOffset, int yOffset )
{
QMutexLocker locker( mpMutex );

if ( !mGdalDataset )
{
return false;
Expand All @@ -2700,6 +2779,8 @@ bool QgsGdalProvider::write( void *data, int band, int width, int height, int xO

bool QgsGdalProvider::setNoDataValue( int bandNo, double noDataValue )
{
QMutexLocker locker( mpMutex );

if ( !mGdalDataset )
{
return false;
Expand All @@ -2721,6 +2802,14 @@ bool QgsGdalProvider::setNoDataValue( int bandNo, double noDataValue )

bool QgsGdalProvider::remove()
{
QMutexLocker locker( mpMutex );

while ( *mpRefCounter != 1 )
{
QgsDebugMsg( QString( "Waiting for ref counter for %1 to drop to 1" ).arg( dataSourceUri() ) );
QThread::msleep( 100 );
}

if ( mGdalDataset )
{
GDALDriverH driver = GDALGetDatasetDriver( mGdalDataset );
Expand Down Expand Up @@ -2903,6 +2992,8 @@ bool QgsGdalProvider::isEditable() const

bool QgsGdalProvider::setEditable( bool enabled )
{
QMutexLocker locker( mpMutex );

if ( enabled == mUpdate )
return false;

Expand All @@ -2912,6 +3003,12 @@ bool QgsGdalProvider::setEditable( bool enabled )
if ( mGdalDataset != mGdalBaseDataset )
return false; // ignore the case of warped VRT for now (more complicated setup)

while ( *mpRefCounter != 1 )
{
QgsDebugMsg( QString( "Waiting for ref counter for %1 to drop to 1" ).arg( dataSourceUri() ) );
QThread::msleep( 100 );
}

closeDataset();

mUpdate = enabled;
Expand All @@ -2933,6 +3030,8 @@ bool QgsGdalProvider::setEditable( bool enabled )

GDALRasterBandH QgsGdalProvider::getBand( int bandNo ) const
{
QMutexLocker locker( mpMutex );

if ( mMaskBandExposedAsAlpha && bandNo == GDALGetRasterCount( mGdalDataset ) + 1 )
return GDALGetMaskBand( GDALGetRasterBand( mGdalDataset, 1 ) );
else
Expand Down
23 changes: 20 additions & 3 deletions src/providers/gdal/qgsgdalprovider.h
View file
Expand Up @@ -33,6 +33,8 @@
#include <QMap>
#include <QVector>

class QMutex;

class QgsRasterPyramid;

/**
Expand Down Expand Up @@ -76,6 +78,12 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase

~QgsGdalProvider();

/**
* Clone the provider.
*
* The underlying GDAL dataset is shared among the main provider and its
* clones.
*/
QgsGdalProvider *clone() const override;

QString name() const override;
Expand Down Expand Up @@ -142,9 +150,6 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
QgsRasterBlockFeedback *feedback = nullptr ) override;
QList<QgsRasterPyramid> buildPyramidList( QList<int> overviewList = QList<int>() ) override;

//! \brief Close data set and release related data
void closeDataset();

static QMap<QString, QString> supportedMimes();

bool isEditable() const override;
Expand All @@ -158,6 +163,15 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
QString validatePyramidsConfigOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
const QStringList &configOptions, const QString &fileFormat ) override;
private:

QgsGdalProvider( const QgsGdalProvider &other );

// reference counter to know how many main and cloned provider instances are linked
int *mpRefCounter = nullptr;

// mutex to protect access to mGdalDataset among main and cloned provider instances
QMutex *mpMutex = nullptr;

// update mode
bool mUpdate;

Expand Down Expand Up @@ -217,6 +231,9 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase

//! Wrapper for GDALGetRasterBand() that takes into account mMaskBandExposedAsAlpha.
GDALRasterBandH getBand( int bandNo ) const;

//! \brief Close data set and release related data
void closeDataset();
};

#endif
Expand Down

0 comments on commit 3b2b98b

Please sign in to comment.