Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[WFS provider] Do not issue full layer download when requesting featu…
…res by fids...

and when they are already in the local cache.

Fixes #42049
  • Loading branch information
rouault authored and nyalldawson committed Jun 14, 2021
1 parent da9b0fa commit ca20a98
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 26 deletions.
99 changes: 75 additions & 24 deletions src/providers/wfs/qgsbackgroundcachedfeatureiterator.cpp
Expand Up @@ -271,6 +271,7 @@ QgsBackgroundCachedFeatureIterator::QgsBackgroundCachedFeatureIterator(
const QgsFeatureRequest &request )
: QgsAbstractFeatureIteratorFromSource<QgsBackgroundCachedFeatureSource>( source, ownSource, request )
, mShared( shared )
, mCachedFeaturesIter( mCachedFeatures.begin() )
{
if ( !shared->clientSideFilterExpression().isEmpty() )
{
Expand Down Expand Up @@ -305,19 +306,41 @@ QgsBackgroundCachedFeatureIterator::QgsBackgroundCachedFeatureIterator(
if ( !threshold.isEmpty() )
mWriteTransferThreshold = threshold.toInt();

// Particular case: if a single feature is requested by Fid and we already
// have it in cache, no need to interrupt any download
// Particular case: if a single feature or a reasonable set of features
// are requested by Fid and we already have them in cache, no need to
// download anything.
auto cacheDataProvider = mShared->cacheDataProvider();
if ( cacheDataProvider &&
mRequest.filterType() == QgsFeatureRequest::FilterFid )
( mRequest.filterType() == QgsFeatureRequest::FilterFid ||
( mRequest.filterType() == QgsFeatureRequest::FilterFids && mRequest.filterFids().size() < 100000 ) ) )
{
QgsFeatureRequest requestCache = buildRequestCache( -1 );
QgsFeature f;
if ( cacheDataProvider->getFeatures( requestCache ).nextFeature( f ) )
QgsFeatureRequest requestCache;
QgsFeatureIds qgisIds;
if ( mRequest.filterType() == QgsFeatureRequest::FilterFid )
qgisIds.insert( mRequest.filterFid() );
else
qgisIds = mRequest.filterFids();
QgsFeatureIds dbIds = mShared->dbIdsFromQgisIds( qgisIds );
// Do all requested fids have been known at some point by the provider ?
if ( dbIds.size() == qgisIds.size() )
{
mCacheIterator = cacheDataProvider->getFeatures( requestCache );
mDownloadFinished = true;
return;
requestCache.setFilterFids( dbIds );
fillRequestCache( requestCache );
QgsFeatureIterator cacheIterator = cacheDataProvider->getFeatures( requestCache );
QgsFeature f;
while ( cacheIterator.nextFeature( f ) )
{
mCachedFeatures << f;
}
// Are are the requested fids actually in the cache ?
if ( mCachedFeatures.size() == dbIds.size() )
{
// Yes, no need to download anything.
mCachedFeaturesIter = mCachedFeatures.begin();
mDownloadFinished = true;
return;
}
mCachedFeatures.clear();
}
}

Expand All @@ -332,10 +355,12 @@ QgsBackgroundCachedFeatureIterator::QgsBackgroundCachedFeatureIterator(

QgsDebugMsgLevel( QStringLiteral( "QgsBackgroundCachedFeatureIterator::constructor(): genCounter=%1 " ).arg( genCounter ), 4 );

mCacheIterator = cacheDataProvider->getFeatures( buildRequestCache( genCounter ) );
QgsFeatureRequest requestCache = initRequestCache( genCounter ) ;
fillRequestCache( requestCache );
mCacheIterator = cacheDataProvider->getFeatures( requestCache );
}

QgsFeatureRequest QgsBackgroundCachedFeatureIterator::buildRequestCache( int genCounter )
QgsFeatureRequest QgsBackgroundCachedFeatureIterator::initRequestCache( int genCounter )
{
QgsFeatureRequest requestCache;

Expand Down Expand Up @@ -392,6 +417,11 @@ QgsFeatureRequest QgsBackgroundCachedFeatureIterator::buildRequestCache( int gen
}
}

return requestCache;
}

void QgsBackgroundCachedFeatureIterator::fillRequestCache( QgsFeatureRequest requestCache )
{
requestCache.setFilterRect( mFilterRect );

if ( !( mRequest.flags() & QgsFeatureRequest::NoGeometry ) ||
Expand All @@ -402,6 +432,8 @@ QgsFeatureRequest QgsBackgroundCachedFeatureIterator::buildRequestCache( int gen

if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
const QgsFields fields = mShared->fields();
auto cacheDataProvider = mShared->cacheDataProvider();
QgsFields dataProviderFields = cacheDataProvider->fields();
QgsAttributeList cacheSubSet;
const auto subsetOfAttributes = mRequest.subsetOfAttributes();
Expand Down Expand Up @@ -460,8 +492,6 @@ QgsFeatureRequest QgsBackgroundCachedFeatureIterator::buildRequestCache( int gen
}
requestCache.setSubsetOfAttributes( cacheSubSet );
}

return requestCache;
}

QgsBackgroundCachedFeatureIterator::~QgsBackgroundCachedFeatureIterator()
Expand Down Expand Up @@ -572,8 +602,21 @@ bool QgsBackgroundCachedFeatureIterator::fetchFeature( QgsFeature &f )

// First step is to iterate over the on-disk cache
QgsFeature cachedFeature;
while ( mCacheIterator.nextFeature( cachedFeature ) )
while ( true )
{
if ( !mCachedFeatures.empty() )
{
if ( mCachedFeaturesIter == mCachedFeatures.end() )
return false;
cachedFeature = *mCachedFeaturesIter;
++mCachedFeaturesIter;
}
else
{
if ( !mCacheIterator.nextFeature( cachedFeature ) )
break;
}

if ( mInterruptionChecker && mInterruptionChecker->isCanceled() )
return false;

Expand Down Expand Up @@ -803,17 +846,25 @@ bool QgsBackgroundCachedFeatureIterator::rewind()
if ( mClosed )
return false;

cleanupReaderStreamAndFile();

QgsFeatureRequest requestCache;
int genCounter = mShared->getUpdatedCounter();
if ( genCounter >= 0 )
requestCache.setFilterExpression( QString( QgsBackgroundCachedFeatureIteratorConstants::FIELD_GEN_COUNTER + " <= %1" ).arg( genCounter ) );
if ( !mCachedFeatures.empty() )
{
mCachedFeaturesIter = mCachedFeatures.begin();
}
else
mDownloadFinished = true;
auto cacheDataProvider = mShared->cacheDataProvider();
if ( cacheDataProvider )
mCacheIterator = cacheDataProvider->getFeatures( requestCache );
{
cleanupReaderStreamAndFile();

QgsFeatureRequest requestCache;
int genCounter = mShared->getUpdatedCounter();
if ( genCounter >= 0 )
requestCache.setFilterExpression( QString( QgsBackgroundCachedFeatureIteratorConstants::FIELD_GEN_COUNTER + " <= %1" ).arg( genCounter ) );
else
mDownloadFinished = true;
auto cacheDataProvider = mShared->cacheDataProvider();
if ( cacheDataProvider )
mCacheIterator = cacheDataProvider->getFeatures( requestCache );
}

return true;
}

Expand Down
11 changes: 9 additions & 2 deletions src/providers/wfs/qgsbackgroundcachedfeatureiterator.h
Expand Up @@ -352,6 +352,10 @@ class QgsBackgroundCachedFeatureIterator final: public QObject,
QgsFeedback *mInterruptionChecker = nullptr;
bool mTimeoutOrInterruptionOccurred = false;

//! Cached features for request by fid
QVector<QgsFeature> mCachedFeatures;
QVector<QgsFeature>::iterator mCachedFeaturesIter;

//! this mutex synchronizes the mWriterXXXX variables between featureReceivedSynchronous() and fetchFeature()
QMutex mMutex;
QWaitCondition mWaitCond;
Expand All @@ -378,8 +382,11 @@ class QgsBackgroundCachedFeatureIterator final: public QObject,

///////////////// METHODS

//! Translate mRequest to a request compatible of the Spatialite cache
QgsFeatureRequest buildRequestCache( int gencounter );
//! Translate mRequest to a request compatible of the Spatialite cache (first part)
QgsFeatureRequest initRequestCache( int gencounter );

//! Finishes to fill the request to the cache (second part)
void fillRequestCache( QgsFeatureRequest );

bool fetchFeature( QgsFeature &f ) override;

Expand Down

0 comments on commit ca20a98

Please sign in to comment.