Merge pull request #40726 from elpaso/connections-api-results-iterator

Connections API: execSql iterator
qgis · Dec 23, 2020 · e520af3 · e520af3
2 parents a5898bc + 303f689
commit e520af3
Show file tree

Hide file tree

Showing 14 changed files with 592 additions and 183 deletions.
diff --git a/python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in b/python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in
@@ -49,7 +49,7 @@ is not supported or cannot be performed without errors.
     {
         SIP_PYOBJECT __repr__();
 %MethodCode
-        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rows().size() );
+        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rowCount() );
         sipRes = PyUnicode_FromString( str.toUtf8().constData() );
 %End
 
@@ -63,16 +63,54 @@ Returns the column names
 Returns the results rows
 %End
 
-        void appendColumn( const QString &columnName );
+        qlonglong rowCount() const;
 %Docstring
-Appends ``columnName`` to the list of column names.
+Returns the row count
+
+.. note::
+
+   the value may not be exact or it may be -1 if not known
+%End
+
+        bool hasNextRow() const;
+%Docstring
+Returns ``True`` if there are more rows to fetch
 %End
 
-        void appendRow( const QList<QVariant> &row );
+        QList<QVariant> nextRow();
 %Docstring
-Appends ``row`` to the results.
+Returns the next result row or an empty row if there are no rows left
 %End
 
+        QueryResult *__iter__();
+%MethodCode
+        sipRes = sipCpp;
+%End
+
+        SIP_PYOBJECT __next__();
+%MethodCode
+        QList<QVariant> result;
+        Py_BEGIN_ALLOW_THREADS
+        result = sipCpp->nextRow( );
+        Py_END_ALLOW_THREADS
+        if ( ! result.isEmpty() )
+        {
+          const sipTypeDef *qvariantlist_type = sipFindType( "QList<QVariant>" );
+          sipRes = sipConvertFromNewType( new QList<QVariant>( result ), qvariantlist_type, Py_None );
+        }
+        else
+        {
+          PyErr_SetString( PyExc_StopIteration, "" );
+        }
+%End
+
+
+
+
+
+
+
+
     };
 
 

diff --git a/src/core/providers/ogr/qgsgeopackageproviderconnection.cpp b/src/core/providers/ogr/qgsgeopackageproviderconnection.cpp
@@ -70,7 +70,7 @@ QString QgsGeoPackageProviderConnection::tableUri( const QString &schema, const
   const auto tableInfo { table( schema, name ) };
   if ( tableInfo.flags().testFlag( QgsAbstractDatabaseProviderConnection::TableFlag::Raster ) )
   {
-    return QStringLiteral( "GPKG:%1:%2" ).arg( uri() ).arg( name );
+    return QStringLiteral( "GPKG:%1:%2" ).arg( uri(), name );
   }
   else
   {
@@ -123,7 +123,7 @@ void QgsGeoPackageProviderConnection::dropVectorTable( const QString &schema, co
   const QString layerUri { QStringLiteral( "%1|layername=%2" ).arg( uri(), name ) };
   if ( ! QgsOgrProviderUtils::deleteLayer( layerUri, errCause ) )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error deleting vector/aspatial table %1: %2" ).arg( name ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error deleting vector/aspatial table %1: %2" ).arg( name, errCause ) );
   }
 }
 
@@ -306,7 +306,7 @@ QList<QgsGeoPackageProviderConnection::TableProperty> QgsGeoPackageProviderConne
 
   if ( ! errCause.isEmpty() )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error listing tables from %1: %2" ).arg( uri() ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error listing tables from %1: %2" ).arg( uri(), errCause ) );
   }
   // Filters
   if ( flags )
@@ -356,11 +356,10 @@ void QgsGeoPackageProviderConnection::setDefaultCapabilities()
 
 QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnection::executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback ) const
 {
-  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
-    return results;
+    return QgsAbstractDatabaseProviderConnection::QueryResult();
   }
 
   QString errCause;
@@ -370,72 +369,110 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnecti
 
     if ( feedback && feedback->isCanceled() )
     {
-      return results;
+      return QgsAbstractDatabaseProviderConnection::QueryResult();
     }
 
     OGRLayerH ogrLayer( GDALDatasetExecuteSQL( hDS.get(), sql.toUtf8().constData(), nullptr, nullptr ) );
+
+    // Read fields
     if ( ogrLayer )
     {
+
+      auto iterator = std::make_shared<QgsGeoPackageProviderResultIterator>( std::move( hDS ), ogrLayer );
+      QgsAbstractDatabaseProviderConnection::QueryResult results( iterator );
+      // Note: Returns the number of features in the layer. For dynamic databases the count may not be exact.
+      //       If bForce is FALSE, and it would be expensive to establish the feature count a value of -1 may
+      //       be returned indicating that the count isn’t know.
+      results.setRowCount( OGR_L_GetFeatureCount( ogrLayer, 0 /* bForce=false: do not scan the whole layer */ ) );
+
       gdal::ogr_feature_unique_ptr fet;
-      QgsFields fields;
-      while ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
+      if ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
       {
 
-        if ( feedback && feedback->isCanceled() )
-        {
-          break;
-        }
+        QgsFields fields { QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) ) };
+        iterator->setFields( fields );
 
-        QVariantList row;
-
-        // Try to get the right type for the returned values
-        if ( fields.isEmpty() )
+        for ( const auto &f : qgis::as_const( fields ) )
         {
-          fields = QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) );
-          for ( const auto &f : qgis::as_const( fields ) )
-          {
-            results.appendColumn( f.name() );
-          }
+          results.appendColumn( f.name() );
         }
+      }
 
-        if ( ! fields.isEmpty() )
-        {
-          QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), fields, QTextCodec::codecForName( "UTF-8" ) ) };
-          const QgsAttributes &constAttrs { f.attributes() };
-          for ( int i = 0; i < constAttrs.length(); i++ )
-          {
-            row.push_back( constAttrs.at( i ) );
-          }
-        }
-        else // Fallback to strings
-        {
-          for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
-          {
-            row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
-          }
-        }
+      // Check for errors
+      errCause = CPLGetLastErrorMsg( );
 
-        results.appendRow( row );
+      if ( ! errCause.isEmpty() )
+      {
+        throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
       }
-      GDALDatasetReleaseResultSet( hDS.get(), ogrLayer );
+
+      OGR_L_ResetReading( ogrLayer );
+      iterator->nextRow();
+      return results;
     }
     errCause = CPLGetLastErrorMsg( );
   }
   else
   {
     errCause = QObject::tr( "There was an error opening GPKG %1!" ).arg( uri() );
   }
-  if ( ! errCause.isEmpty() )
+
+  if ( !errCause.isEmpty() )
+  {
+    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
+  }
+
+  return QgsAbstractDatabaseProviderConnection::QueryResult();
+}
+
+QVariantList QgsGeoPackageProviderResultIterator::nextRow()
+{
+  const QVariantList currentRow { mNextRow };
+  mNextRow = nextRowPrivate();
+  return currentRow;
+}
+
+QVariantList QgsGeoPackageProviderResultIterator::nextRowPrivate()
+{
+  QVariantList row;
+  if ( mHDS && mOgrLayer )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql ).arg( errCause ) );
+    gdal::ogr_feature_unique_ptr fet;
+    if ( fet.reset( OGR_L_GetNextFeature( mOgrLayer ) ), fet )
+    {
+      if ( ! mFields.isEmpty() )
+      {
+        QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), mFields, QTextCodec::codecForName( "UTF-8" ) ) };
+        const QgsAttributes &constAttrs { f.attributes() };
+        for ( int i = 0; i < constAttrs.length(); i++ )
+        {
+          row.push_back( constAttrs.at( i ) );
+        }
+      }
+      else // Fallback to strings
+      {
+        for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
+        {
+          row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
+        }
+      }
+    }
   }
-  return results;
+  return row;
 }
 
+bool QgsGeoPackageProviderResultIterator::hasNextRow() const
+{
+  return ! mNextRow.isEmpty();
+}
+
+void QgsGeoPackageProviderResultIterator::setFields( const QgsFields &fields )
+{
+  mFields = fields;
+}
 
 QList<QgsVectorDataProvider::NativeType> QgsGeoPackageProviderConnection::nativeTypes() const
 {
-  QList<QgsVectorDataProvider::NativeType> types;
   QgsVectorLayer::LayerOptions options { false, true };
   options.skipCrsValidation = true;
   const QgsVectorLayer vl { uri(), QStringLiteral( "temp_layer" ), QStringLiteral( "ogr" ), options };
@@ -444,7 +481,12 @@ QList<QgsVectorDataProvider::NativeType> QgsGeoPackageProviderConnection::native
     const QString errorCause { vl.dataProvider() &&vl.dataProvider()->hasErrors() ?
                                vl.dataProvider()->errors().join( '\n' ) :
                                QObject::tr( "unknown error" ) };
-    throw QgsProviderConnectionException( QObject::tr( "Error retrieving native types for %1: %2" ).arg( uri() ).arg( errorCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error retrieving native types for %1: %2" ).arg( uri(), errorCause ) );
   }
   return vl.dataProvider()->nativeTypes();
 }
+
+QgsGeoPackageProviderResultIterator::~QgsGeoPackageProviderResultIterator()
+{
+  GDALDatasetReleaseResultSet( mHDS.get(), mOgrLayer );
+}
diff --git a/src/core/providers/ogr/qgsgeopackageproviderconnection.h b/src/core/providers/ogr/qgsgeopackageproviderconnection.h
@@ -13,14 +13,43 @@
  *   (at your option) any later version.                                   *
  *                                                                         *
  ***************************************************************************/
+
 #ifndef QGSGEOPACKAGEPROVIDERCONNECTION_H
 #define QGSGEOPACKAGEPROVIDERCONNECTION_H
 
 #include "qgsabstractdatabaseproviderconnection.h"
+#include "qgsogrutils.h"
 
 ///@cond PRIVATE
 #define SIP_NO_FILE
 
+
+
+struct QgsGeoPackageProviderResultIterator: public QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator
+{
+    QgsGeoPackageProviderResultIterator( gdal::ogr_datasource_unique_ptr hDS, OGRLayerH ogrLayer )
+      : mHDS( std::move( hDS ) )
+      , mOgrLayer( ogrLayer )
+    {}
+
+    ~QgsGeoPackageProviderResultIterator();
+
+    QVariantList nextRow() override;
+    bool hasNextRow() const override;
+
+    void setFields( const QgsFields &fields );
+
+  private:
+
+    gdal::ogr_datasource_unique_ptr mHDS;
+    OGRLayerH mOgrLayer;
+    QgsFields mFields;
+    QVariantList mNextRow;
+
+    QVariantList nextRowPrivate();
+
+};
+
 class QgsGeoPackageProviderConnection : public QgsAbstractDatabaseProviderConnection
 {
   public:

diff --git a/src/core/qgsabstractdatabaseproviderconnection.cpp b/src/core/qgsabstractdatabaseproviderconnection.cpp
@@ -442,15 +442,65 @@ QStringList QgsAbstractDatabaseProviderConnection::QueryResult::columns() const
 
 QList<QList<QVariant> > QgsAbstractDatabaseProviderConnection::QueryResult::rows() const
 {
+  // mRowCount might be -1 (unknown)
+  while ( mResultIterator && ( mRowCount < 0 || mRows.count() < mRowCount ) )
+  {
+    const QVariantList row { mResultIterator->nextRow() };
+    if ( row.isEmpty() )
+    {
+      break;
+    }
+    mRows.push_back( row );
+  }
   return mRows;
 }
 
+QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::nextRow()
+{
+  if ( ! mResultIterator && ! mResultIterator->hasNextRow() )
+  {
+    return QList<QVariant>();
+  }
+
+  const QList<QVariant> row { mResultIterator->nextRow() };
+
+  if ( ! row.isEmpty() )
+  {
+    mRows.push_back( row );
+  }
+
+  return row;
+}
+
+
+qlonglong QgsAbstractDatabaseProviderConnection::QueryResult::rowCount() const
+{
+  return mRowCount;
+}
+
+bool QgsAbstractDatabaseProviderConnection::QueryResult::hasNextRow() const
+{
+  if ( ! mResultIterator )
+  {
+    return false;
+  }
+  return mResultIterator->hasNextRow();
+}
+
+///@cond PRIVATE
+
 void QgsAbstractDatabaseProviderConnection::QueryResult::appendColumn( const QString &columnName )
 {
   mColumns.push_back( columnName );
 }
 
-void QgsAbstractDatabaseProviderConnection::QueryResult::appendRow( const QList<QVariant> &row )
+void QgsAbstractDatabaseProviderConnection::QueryResult::setRowCount( const qlonglong &rowCount )
 {
-  mRows.push_back( row );
+  mRowCount = rowCount;
 }
+
+QgsAbstractDatabaseProviderConnection::QueryResult::QueryResult( std::shared_ptr<QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator> iterator )
+  : mResultIterator( iterator )
+{}
+
+///@endcond private