[FEATURE] Feature limit support for feature requests

Limits the maximum number of features returned by the iterator. Some providers (postgres, spatialite, MS SQL) pass the limit on to the provider to result in faster queries.
qgis · Dec 4, 2015 · edb16d0 · edb16d0
1 parent 60ad688
commit edb16d0
Show file tree

Hide file tree

Showing 12 changed files with 158 additions and 26 deletions.
diff --git a/python/core/qgsfeaturerequest.sip b/python/core/qgsfeaturerequest.sip
@@ -92,6 +92,19 @@ class QgsFeatureRequest
      */
     QgsFeatureRequest& disableFilter();
 
+    /** Set the maximum number of features to request.
+     * @param limit maximum number of features, or -1 to request all features.
+     * @see limit()
+     * @note added in QGIS 2.14
+     */
+    QgsFeatureRequest& setLimit( long limit );
+
+    /** Returns the maximum number of features to request, or -1 if no limit set.
+     * @see setLimit
+     * @note added in QGIS 2.14
+     */
+    long limit() const;
+
     //! Set flags that affect how features will be fetched
     QgsFeatureRequest& setFlags( const Flags& flags );
     const Flags& flags() const;

diff --git a/src/core/qgsexpression.cpp b/src/core/qgsexpression.cpp
@@ -2184,6 +2184,7 @@ static QVariant fcnGetFeature( const QVariantList& values, const QgsExpressionCo
   QgsFeatureRequest req;
   req.setFilterExpression( QString( "%1=%2" ).arg( QgsExpression::quotedColumnRef( attribute ),
                            QgsExpression::quotedString( attVal.toString() ) ) );
+  req.setLimit( 1 );
   if ( !parent->needsGeometry() )
   {
     req.setFlags( QgsFeatureRequest::NoGeometry );

diff --git a/src/core/qgsfeatureiterator.cpp b/src/core/qgsfeatureiterator.cpp
@@ -22,6 +22,7 @@ QgsAbstractFeatureIterator::QgsAbstractFeatureIterator( const QgsFeatureRequest&
     : mRequest( request )
     , mClosed( false )
     , refs( 0 )
+    , mFetchedCount( 0 )
     , mGeometrySimplifier( NULL )
     , mLocalSimplification( false )
 {
@@ -36,6 +37,10 @@ QgsAbstractFeatureIterator::~QgsAbstractFeatureIterator()
 bool QgsAbstractFeatureIterator::nextFeature( QgsFeature& f )
 {
   bool dataOk = false;
+  if ( mRequest.limit() >= 0 && mFetchedCount >= mRequest.limit() )
+  {
+    return false;
+  }
 
   switch ( mRequest.filterType() )
   {
@@ -59,6 +64,9 @@ bool QgsAbstractFeatureIterator::nextFeature( QgsFeature& f )
     if ( geometry )
       simplify( f );
   }
+  if ( dataOk )
+    mFetchedCount++;
+
   return dataOk;
 }
 

diff --git a/src/core/qgsfeatureiterator.h b/src/core/qgsfeatureiterator.h
@@ -87,6 +87,9 @@ class CORE_EXPORT QgsAbstractFeatureIterator
     void deref(); //!< remove reference, delete if refs == 0
     friend class QgsFeatureIterator;
 
+    //! Number of features already fetched by iterator
+    long mFetchedCount;
+
     //! Setup the simplification of geometries to fetch using the specified simplify method
     virtual bool prepareSimplification( const QgsSimplifyMethod& simplifyMethod );
 
@@ -198,11 +201,17 @@ inline bool QgsFeatureIterator::nextFeature( QgsFeature& f )
 
 inline bool QgsFeatureIterator::rewind()
 {
+  if ( mIter )
+    mIter->mFetchedCount = 0;
+
   return mIter ? mIter->rewind() : false;
 }
 
 inline bool QgsFeatureIterator::close()
 {
+  if ( mIter )
+    mIter->mFetchedCount = 0;
+
   return mIter ? mIter->close() : false;
 }
 

diff --git a/src/core/qgsfeaturerequest.cpp b/src/core/qgsfeaturerequest.cpp
@@ -27,6 +27,7 @@ QgsFeatureRequest::QgsFeatureRequest()
     , mFilterFid( -1 )
     , mFilterExpression( 0 )
     , mFlags( 0 )
+    , mLimit( -1 )
 {
 }
 
@@ -35,6 +36,7 @@ QgsFeatureRequest::QgsFeatureRequest( QgsFeatureId fid )
     , mFilterFid( fid )
     , mFilterExpression( 0 )
     , mFlags( 0 )
+    , mLimit( -1 )
 {
 }
 
@@ -44,6 +46,7 @@ QgsFeatureRequest::QgsFeatureRequest( const QgsRectangle& rect )
     , mFilterFid( -1 )
     , mFilterExpression( 0 )
     , mFlags( 0 )
+    , mLimit( -1 )
 {
 }
 
@@ -53,6 +56,7 @@ QgsFeatureRequest::QgsFeatureRequest( const QgsExpression& expr, const QgsExpres
     , mFilterExpression( new QgsExpression( expr.expression() ) )
     , mExpressionContext( context )
     , mFlags( 0 )
+    , mLimit( -1 )
 {
 }
 
@@ -79,6 +83,7 @@ QgsFeatureRequest& QgsFeatureRequest::operator=( const QgsFeatureRequest & rh )
   mExpressionContext = rh.mExpressionContext;
   mAttrs = rh.mAttrs;
   mSimplifyMethod = rh.mSimplifyMethod;
+  mLimit = rh.mLimit;
   return *this;
 }
 
@@ -102,7 +107,7 @@ QgsFeatureRequest& QgsFeatureRequest::setFilterFid( QgsFeatureId fid )
   return *this;
 }
 
-QgsFeatureRequest&QgsFeatureRequest::setFilterFids( const QgsFeatureIds& fids )
+QgsFeatureRequest& QgsFeatureRequest::setFilterFids( const QgsFeatureIds& fids )
 {
   mFilter = FilterFids;
   mFilterFids = fids;
@@ -117,7 +122,7 @@ QgsFeatureRequest& QgsFeatureRequest::setFilterExpression( const QString& expres
   return *this;
 }
 
-QgsFeatureRequest&QgsFeatureRequest::combineFilterExpression( const QString& expression )
+QgsFeatureRequest& QgsFeatureRequest::combineFilterExpression( const QString& expression )
 {
   if ( mFilterExpression )
   {
@@ -136,6 +141,12 @@ QgsFeatureRequest &QgsFeatureRequest::setExpressionContext( const QgsExpressionC
   return *this;
 }
 
+QgsFeatureRequest& QgsFeatureRequest::setLimit( long limit )
+{
+  mLimit = limit;
+  return *this;
+}
+
 QgsFeatureRequest& QgsFeatureRequest::setFlags( const QgsFeatureRequest::Flags& flags )
 {
   mFlags = flags;

diff --git a/src/core/qgsfeaturerequest.h b/src/core/qgsfeaturerequest.h
@@ -175,6 +175,19 @@ class CORE_EXPORT QgsFeatureRequest
      */
     QgsFeatureRequest& disableFilter() { mFilter = FilterNone; return *this; }
 
+    /** Set the maximum number of features to request.
+     * @param limit maximum number of features, or -1 to request all features.
+     * @see limit()
+     * @note added in QGIS 2.14
+     */
+    QgsFeatureRequest& setLimit( long limit );
+
+    /** Returns the maximum number of features to request, or -1 if no limit set.
+     * @see setLimit
+     * @note added in QGIS 2.14
+     */
+    long limit() const { return mLimit; }
+
     //! Set flags that affect how features will be fetched
     QgsFeatureRequest& setFlags( const QgsFeatureRequest::Flags& flags );
     const Flags& flags() const { return mFlags; }
@@ -223,6 +236,7 @@ class CORE_EXPORT QgsFeatureRequest
     Flags mFlags;
     QgsAttributeList mAttrs;
     QgsSimplifyMethod mSimplifyMethod;
+    long mLimit;
 };
 
 Q_DECLARE_OPERATORS_FOR_FLAGS( QgsFeatureRequest::Flags )

diff --git a/src/providers/mssql/qgsmssqlfeatureiterator.cpp b/src/providers/mssql/qgsmssqlfeatureiterator.cpp
@@ -63,6 +63,9 @@ void QgsMssqlFeatureIterator::BuildStatement( const QgsFeatureRequest& request )
   // build sql statement
   mStatement = QString( "SELECT " );
 
+  if ( request.limit() >= 0 && request.filterType() != QgsFeatureRequest::FilterExpression )
+    mStatement += QString( "TOP %1 " ).arg( mRequest.limit() );
+
   mStatement += QString( "[%1]" ).arg( mSource->mFidColName );
   mFidCol = mSource->mFields.indexFromName( mSource->mFidColName );
   mAttributesToFetch.append( mFidCol );

diff --git a/src/providers/postgres/qgspostgresfeatureiterator.cpp b/src/providers/postgres/qgspostgresfeatureiterator.cpp
@@ -59,6 +59,8 @@ QgsPostgresFeatureIterator::QgsPostgresFeatureIterator( QgsPostgresFeatureSource
   mCursorName = mConn->uniqueCursorName();
   QString whereClause;
 
+  bool limitAtProvider = ( mRequest.limit() >= 0 );
+
   if ( !request.filterRect().isNull() && !mSource->mGeometryColumn.isNull() )
   {
     whereClause = whereClauseRect();
@@ -76,15 +78,25 @@ QgsPostgresFeatureIterator::QgsPostgresFeatureIterator( QgsPostgresFeatureSource
 
     whereClause = QgsPostgresUtils::andWhereClauses( whereClause, fidsWhereClause );
   }
-  else if ( request.filterType() == QgsFeatureRequest::FilterExpression
-            && QSettings().value( "/qgis/compileExpressions", true ).toBool() )
+  else if ( request.filterType() == QgsFeatureRequest::FilterExpression )
   {
-    QgsPostgresExpressionCompiler compiler = QgsPostgresExpressionCompiler( source );
+    if ( QSettings().value( "/qgis/compileExpressions", true ).toBool() )
+    {
+      QgsPostgresExpressionCompiler compiler = QgsPostgresExpressionCompiler( source );
 
-    if ( compiler.compile( request.filterExpression() ) == QgsSqlExpressionCompiler::Complete )
+      if ( compiler.compile( request.filterExpression() ) == QgsSqlExpressionCompiler::Complete )
+      {
+        whereClause = QgsPostgresUtils::andWhereClauses( whereClause, compiler.result() );
+        mExpressionCompiled = true;
+      }
+      else
+      {
+        limitAtProvider = false;
+      }
+    }
+    else
     {
-      whereClause = QgsPostgresUtils::andWhereClauses( whereClause, compiler.result() );
-      mExpressionCompiled = true;
+      limitAtProvider = false;
     }
   }
 
@@ -96,7 +108,7 @@ QgsPostgresFeatureIterator::QgsPostgresFeatureIterator( QgsPostgresFeatureSource
     whereClause += '(' + mSource->mSqlWhereClause + ')';
   }
 
-  if ( !declareCursor( whereClause ) )
+  if ( !declareCursor( whereClause, limitAtProvider ? mRequest.limit() : -1 ) )
   {
     mClosed = true;
     iteratorClosed();
@@ -324,7 +336,7 @@ QString QgsPostgresFeatureIterator::whereClauseRect()
 
 
 
-bool QgsPostgresFeatureIterator::declareCursor( const QString& whereClause )
+bool QgsPostgresFeatureIterator::declareCursor( const QString& whereClause, long limit )
 {
   mFetchGeometry = !( mRequest.flags() & QgsFeatureRequest::NoGeometry ) && !mSource->mGeometryColumn.isNull();
 #if 0
@@ -483,6 +495,9 @@ bool QgsPostgresFeatureIterator::declareCursor( const QString& whereClause )
   if ( !whereClause.isEmpty() )
     query += QString( " WHERE %1" ).arg( whereClause );
 
+  if ( limit >= 0 )
+    query += QString( " LIMIT %1" ).arg( limit );
+
   if ( !mConn->openCursor( mCursorName, query ) )
   {
 

diff --git a/src/providers/postgres/qgspostgresfeatureiterator.h b/src/providers/postgres/qgspostgresfeatureiterator.h
@@ -97,7 +97,7 @@ class QgsPostgresFeatureIterator : public QgsAbstractFeatureIteratorFromSource<Q
     QString whereClauseRect();
     bool getFeature( QgsPostgresResult &queryResult, int row, QgsFeature &feature );
     void getFeatureAttribute( int idx, QgsPostgresResult& queryResult, int row, int& col, QgsFeature& feature );
-    bool declareCursor( const QString& whereClause );
+    bool declareCursor( const QString& whereClause, long limit = -1 );
 
     QString mCursorName;
 

diff --git a/src/providers/spatialite/qgsspatialitefeatureiterator.cpp b/src/providers/spatialite/qgsspatialitefeatureiterator.cpp
@@ -37,6 +37,11 @@ QgsSpatiaLiteFeatureIterator::QgsSpatiaLiteFeatureIterator( QgsSpatiaLiteFeature
 
   QStringList whereClauses;
   QString whereClause;
+
+  //beware - limitAtProvider needs to be set to false if the request cannot be completely handled
+  //by the provider (eg utilising QGIS expression filters)
+  bool limitAtProvider = ( mRequest.limit() >= 0 );
+
   if ( !request.filterRect().isNull() && !mSource->mGeometryColumn.isNull() )
   {
     // some kind of MBR spatial filtering is required
@@ -63,23 +68,34 @@ QgsSpatiaLiteFeatureIterator::QgsSpatiaLiteFeatureIterator( QgsSpatiaLiteFeature
       whereClauses.append( whereClause );
     }
   }
-  else if ( request.filterType() == QgsFeatureRequest::FilterExpression
-            && QSettings().value( "/qgis/compileExpressions", true ).toBool() )
+  else if ( request.filterType() == QgsFeatureRequest::FilterExpression )
   {
-    QgsSpatiaLiteExpressionCompiler compiler = QgsSpatiaLiteExpressionCompiler( source );
+    if ( QSettings().value( "/qgis/compileExpressions", true ).toBool() )
+    {
+      QgsSpatiaLiteExpressionCompiler compiler = QgsSpatiaLiteExpressionCompiler( source );
 
-    QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
+      QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
 
-    if ( result == QgsSqlExpressionCompiler::Complete || result == QgsSqlExpressionCompiler::Partial )
-    {
-      whereClause = compiler.result();
-      if ( !whereClause.isEmpty() )
+      if ( result == QgsSqlExpressionCompiler::Complete || result == QgsSqlExpressionCompiler::Partial )
+      {
+        whereClause = compiler.result();
+        if ( !whereClause.isEmpty() )
+        {
+          whereClauses.append( whereClause );
+          //if only partial success when compiling expression, we need to double-check results using QGIS' expressions
+          mExpressionCompiled = ( result == QgsSqlExpressionCompiler::Complete );
+        }
+      }
+      if ( result != QgsSqlExpressionCompiler::Complete )
       {
-        whereClauses.append( whereClause );
-        //if only partial success when compiling expression, we need to double-check results using QGIS' expressions
-        mExpressionCompiled = ( result == QgsSqlExpressionCompiler::Complete );
+        //can't apply limit at provider side as we need to check all results using QGIS expressions
+        limitAtProvider = false;
       }
     }
+    else
+    {
+      limitAtProvider = false;
+    }
   }
 
   if ( !mSource->mSubsetString.isEmpty() )
@@ -94,7 +110,7 @@ QgsSpatiaLiteFeatureIterator::QgsSpatiaLiteFeatureIterator( QgsSpatiaLiteFeature
   whereClause = whereClauses.join( " AND " );
 
   // preparing the SQL statement
-  if ( !prepareStatement( whereClause ) )
+  if ( !prepareStatement( whereClause, limitAtProvider ? mRequest.limit() : -1 ) )
   {
     // some error occurred
     sqliteStatement = NULL;
@@ -183,7 +199,7 @@ bool QgsSpatiaLiteFeatureIterator::close()
 ////
 
 
-bool QgsSpatiaLiteFeatureIterator::prepareStatement( const QString& whereClause )
+bool QgsSpatiaLiteFeatureIterator::prepareStatement( const QString& whereClause, long limit )
 {
   if ( !mHandle )
     return false;
@@ -222,6 +238,9 @@ bool QgsSpatiaLiteFeatureIterator::prepareStatement( const QString& whereClause
     if ( !whereClause.isEmpty() )
       sql += QString( " WHERE %1" ).arg( whereClause );
 
+    if ( limit >= 0 )
+      sql += QString( " LIMIT %1" ).arg( limit );
+
     if ( sqlite3_prepare_v2( mHandle->handle(), sql.toUtf8().constData(), -1, &sqliteStatement, NULL ) != SQLITE_OK )
     {
       // some error occurred

diff --git a/src/providers/spatialite/qgsspatialitefeatureiterator.h b/src/providers/spatialite/qgsspatialitefeatureiterator.h
@@ -77,7 +77,7 @@ class QgsSpatiaLiteFeatureIterator : public QgsAbstractFeatureIteratorFromSource
     QString whereClauseFid();
     QString whereClauseFids();
     QString mbr( const QgsRectangle& rect );
-    bool prepareStatement( const QString& whereClause );
+    bool prepareStatement( const QString& whereClause, long limit = -1 );
     QString quotedPrimaryKey();
     bool getFeature( sqlite3_stmt *stmt, QgsFeature &feature );
     QString fieldName( const QgsField& fld );