Threaded query model

qgis · Jan 13, 2021 · f5ffc56 · f5ffc56
1 parent d3bed16
commit f5ffc56
Show file tree

Hide file tree

Showing 6 changed files with 208 additions and 67 deletions.
diff --git a/python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in b/python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in
@@ -58,9 +58,10 @@ is not supported or cannot be performed without errors.
 Returns the column names
 %End
 
-        QList<QList<QVariant> > rows() const;
+        QList<QList<QVariant> > rows(QgsFeedback* feedback = 0) const;
 %Docstring
-Returns the results rows by calling the iterator internally.
+Returns the results rows by calling the iterator internally,
+an optional ``feedback`` can be used to interrupt the fetching loop.
 
 .. note::
 
@@ -70,21 +71,26 @@ Returns the results rows by calling the iterator internally.
 
         qlonglong rowCount() const;
 %Docstring
-Returns the row count
+Returns the estimated row count, the value may not be exact or it may be -1 if not known.
 
-.. note::
-
-   the value may not be exact or it may be -1 if not known
+.. seealso:: :py:func:`fetchedRowCount`
 %End
 
         bool hasNextRow() const;
 %Docstring
 Returns ``True`` if there are more rows to fetch
 %End
 
-        QList<QVariant> nextRow();
+        QList<QVariant> nextRow() const;
 %Docstring
 Returns the next result row or an empty row if there are no rows left
+%End
+
+        qlonglong fetchedRowCount( ) const;
+%Docstring
+Returns the number of fetched rows
+
+.. seealso:: :py:func:`rowCount`
 %End
 
         QList<QVariant> at( qlonglong index ) const;
@@ -96,6 +102,7 @@ Returns the row data at 0-based index ``index``, if index is not valid, an empty
    this method calls the iterator internally until ``index`` row is retrieved
 %End
 
+        // Python iterator
         QueryResult *__iter__();
 %MethodCode
         sipRes = sipCpp;

diff --git a/src/core/qgsabstractdatabaseproviderconnection.cpp b/src/core/qgsabstractdatabaseproviderconnection.cpp
@@ -17,6 +17,8 @@
 #include "qgsvectorlayer.h"
 #include "qgsexception.h"
 #include "qgslogger.h"
+#include "qgsfeedback.h"
+
 #include <QVariant>
 #include <QObject>
 
@@ -435,15 +437,19 @@ void QgsAbstractDatabaseProviderConnection::TableProperty::setSchema( const QStr
 }
 
 
+///@cond PRIVATE
+
 QStringList QgsAbstractDatabaseProviderConnection::QueryResult::columns() const
 {
   return mColumns;
 }
 
-QList<QList<QVariant> > QgsAbstractDatabaseProviderConnection::QueryResult::rows() const
+QList<QList<QVariant> > QgsAbstractDatabaseProviderConnection::QueryResult::rows( QgsFeedback* feedback) const
 {
   // mRowCount might be -1 (unknown)
-  while ( mResultIterator && ( mRowCount < 0 || mRows.count() < mRowCount ) )
+  while ( mResultIterator &&
+          ( mRowCount < 0 || mRows.count() < mRowCount ) &&
+          hasNextRow() && ( ! feedback || ! feedback->isCanceled() ) )
   {
     const QVariantList row( mResultIterator->nextRow() );
     if ( row.isEmpty() )
@@ -455,9 +461,9 @@ QList<QList<QVariant> > QgsAbstractDatabaseProviderConnection::QueryResult::rows
   return mRows;
 }
 
-QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::nextRow()
+QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::nextRow() const
 {
-  if ( ! mResultIterator && ! mResultIterator->hasNextRow() )
+  if ( ! mResultIterator || ! hasNextRow() )
   {
     return QList<QVariant>();
   }
@@ -472,14 +478,22 @@ QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::nextRow()
   return row;
 }
 
+qlonglong QgsAbstractDatabaseProviderConnection::QueryResult::fetchedRowCount() const
+{
+  return mRows.count();
+}
+
 QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::at( qlonglong index ) const
 {
   if ( index < 0 )
+  {
     return QList<QVariant>();
+  }
+
   // Fetch rows until the index
   if ( index >= mRows.count( ) )
   {
-    while ( mResultIterator && ( mRowCount < 0 || mRows.count() < mRowCount ) && index >= mRows.count() )
+    while ( mResultIterator && ( mRowCount < 0 || mRows.count() < mRowCount ) && index >= mRows.count() && hasNextRow() )
     {
       const QVariantList row { mResultIterator->nextRow() };
       if ( row.isEmpty() )
@@ -510,10 +524,14 @@ bool QgsAbstractDatabaseProviderConnection::QueryResult::hasNextRow() const
   {
     return false;
   }
-  return mResultIterator->hasNextRow();
+  const bool results { mResultIterator->hasNextRow() };
+  if ( ! results )
+  {
+    mResultIterator.reset();
+  }
+  return results;
 }
 
-///@cond PRIVATE
 
 void QgsAbstractDatabaseProviderConnection::QueryResult::appendColumn( const QString &columnName )
 {
@@ -529,4 +547,5 @@ QgsAbstractDatabaseProviderConnection::QueryResult::QueryResult( std::shared_ptr
   : mResultIterator( iterator )
 {}
 
+
 ///@endcond private
diff --git a/src/core/qgsabstractdatabaseproviderconnection.h b/src/core/qgsabstractdatabaseproviderconnection.h
@@ -98,15 +98,17 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
         QStringList columns() const;
 
         /**
-         * Returns the results rows by calling the iterator internally.
+         * Returns the results rows by calling the iterator internally,
+         * an optional \a feedback can be used to interrupt the fetching loop.
+         *
          * \note results are cached internally and subsequent calls to this method
          * will return the cached results.
          */
-        QList<QList<QVariant> > rows() const;
+        QList<QList<QVariant> > rows(QgsFeedback* feedback = nullptr) const;
 
         /**
-          * Returns the row count
-          * \note the value may not be exact or it may be -1 if not known
+          * Returns the estimated row count, the value may not be exact or it may be -1 if not known.
+          * \see fetchedRowCount()
          */
         qlonglong rowCount() const;
 
@@ -118,7 +120,13 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
         /**
          * Returns the next result row or an empty row if there are no rows left
          */
-        QList<QVariant> nextRow();
+        QList<QVariant> nextRow() const;
+
+        /**
+         * Returns the number of fetched rows
+         * \see rowCount()
+         */
+        qlonglong fetchedRowCount( ) const;
 
         /**
          * Returns the row data at 0-based index \a index, if index is not valid, an empty list is returned.
@@ -127,6 +135,7 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
         QList<QVariant> at( qlonglong index ) const;
 
 #ifdef SIP_RUN
+        // Python iterator
         QueryResult *__iter__();
         % MethodCode
         sipRes = sipCpp;
@@ -189,9 +198,9 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
 
 ///@endcond private
 
-      private:
+  private:
 
-        std::shared_ptr<QueryResultIterator> mResultIterator;
+        mutable std::shared_ptr<QueryResultIterator> mResultIterator;
         QStringList mColumns;
         mutable QList<QList<QVariant>> mRows;
         qlonglong mRowCount = 0;

diff --git a/src/core/qgsqueryresultmodel.cpp b/src/core/qgsqueryresultmodel.cpp
@@ -15,12 +15,37 @@
  ***************************************************************************/
 #include "qgsqueryresultmodel.h"
 
-const int QgsQueryResultModel::ROWS_TO_FETCH = 200;
+const int ResultWorker::ROWS_TO_FETCH = 200;
 
-QgsQueryResultModel::QgsQueryResultModel( const QgsAbstractDatabaseProviderConnection::QueryResult &queryResult, QObject *parent )
+QgsQueryResultModel::QgsQueryResultModel(const QgsAbstractDatabaseProviderConnection::QueryResult& queryResult, QObject *parent )
   : QAbstractListModel( parent )
   , mQueryResult( queryResult )
+  , mColumns( queryResult.columns() )
+  , mRowCount( mQueryResult.fetchedRowCount() )
 {
+  if ( mQueryResult.hasNextRow() )
+  {
+    ResultWorker *worker = new ResultWorker( &mQueryResult );
+    worker->moveToThread(&mWorkerThread);
+    //connect(&mWorkerThread, &QThread::finished, worker, &ResultWorker::stopFetching );
+    connect(&mWorkerThread, &QThread::finished, worker, &QObject::deleteLater);
+    connect(&mWorkerThread, &QThread::started, worker, &ResultWorker::fetchRows);
+    connect(worker, &ResultWorker::rowsReady, this, &QgsQueryResultModel::newRowsReady );
+    mWorkerThread.start();
+  }
+}
+
+void QgsQueryResultModel::newRowsReady( int newRowCount )
+{
+  beginInsertRows( QModelIndex(), mRowCount, mRowCount + newRowCount - 1 );
+  mRowCount += newRowCount;
+  endInsertRows();
+}
+
+QgsQueryResultModel::~QgsQueryResultModel()
+{
+  mWorkerThread.quit();
+  mWorkerThread.wait();
 }
 
 int QgsQueryResultModel::rowCount( const QModelIndex &parent ) const
@@ -34,15 +59,13 @@ int QgsQueryResultModel::columnCount( const QModelIndex &parent ) const
 {
   if ( parent.isValid() )
     return 0;
-  return mQueryResult.columns().count();
+  return mColumns.count();
 }
 
 QVariant QgsQueryResultModel::data( const QModelIndex &index, int role ) const
 {
-  if ( !index.isValid() || index.row() < 0 || index.column() > mQueryResult.columns().count() - 1 )
-    return QVariant();
-
-  if ( index.row() >= mRowCount || index.row() < 0 )
+  if ( !index.isValid() || index.row() < 0 || index.column() > mColumns.count() - 1 ||
+       index.row() >= mRowCount )
     return QVariant();
 
   if ( role == Qt::DisplayRole )
@@ -56,30 +79,28 @@ QVariant QgsQueryResultModel::data( const QModelIndex &index, int role ) const
   return QVariant();
 }
 
-void QgsQueryResultModel::fetchMore( const QModelIndex &parent )
+void ResultWorker::fetchRows()
 {
-  if ( parent.isValid() )
-    return;
-
-  QVariantList newRows;
-  for ( int i = 0; i < ROWS_TO_FETCH && mQueryResult.hasNextRow(); ++i )
+  int rowCount = 0;
+  while ( mQueryResult->hasNextRow() && mStopFetching == 0 )
   {
-    newRows.push_back( mQueryResult.nextRow() );
+     mQueryResult->nextRow();
+     ++rowCount;
+     if ( rowCount == ROWS_TO_FETCH && mStopFetching == 0 )
+     {
+       emit rowsReady( rowCount );
+       rowCount = 0;
+     }
   }
 
-  if ( !newRows.isEmpty() )
+  if ( rowCount  > 0 && mStopFetching == 0 )
   {
-    const qlonglong newRowsCount { newRows.count() };
-    beginInsertRows( QModelIndex(), newRowsCount, newRowsCount + newRows.count() - 1 );
-    mRowCount += newRowsCount;
-    endInsertRows();
+    emit rowsReady( rowCount );
   }
 }
 
-bool QgsQueryResultModel::canFetchMore( const QModelIndex &parent ) const
+void ResultWorker::stopFetching()
 {
-  if ( parent.isValid() )
-    return false;
-
-  return mQueryResult.hasNextRow();
+  qDebug() << "Stop fetching" << mQueryResult->fetchedRowCount();
+  mStopFetching = 1;
 }
diff --git a/src/core/qgsqueryresultmodel.h b/src/core/qgsqueryresultmodel.h
@@ -17,12 +17,52 @@
 #define QgsQueryResultModel_H
 
 #include <QAbstractListModel>
+#include <QThread>
 
 #include "qgis_core.h"
 #include "qgis_sip.h"
 
 #include "qgsabstractdatabaseproviderconnection.h"
 
+
+
+///@cond private
+
+#ifndef SIP_RUN
+
+class ResultWorker: public QObject
+{
+    Q_OBJECT
+
+
+  public:
+
+    ResultWorker( const QgsAbstractDatabaseProviderConnection::QueryResult* queryResult )
+      : mQueryResult( queryResult )
+    {}
+
+    void fetchRows();
+
+    void stopFetching();
+
+  signals:
+
+    void rowsReady(int newRowsCount);
+
+  private:
+
+      const QgsAbstractDatabaseProviderConnection::QueryResult* mQueryResult = nullptr;
+      QAtomicInt mStopFetching = 0;
+      // Batch of rows to fetch before emitting rowsReady
+      static const int ROWS_TO_FETCH;
+
+};
+
+#endif
+
+///@encond private
+
+
 /**
  * The QgsQueryResultModel class is a model for QgsAbstractDatabaseProviderConnection::QueryResult
  *
@@ -37,23 +77,28 @@ class CORE_EXPORT QgsQueryResultModel : public QAbstractListModel
     /**
      * Constructs a QgsQueryResultModel from a \a queryResult with optional \a parent
      */
-    QgsQueryResultModel( const QgsAbstractDatabaseProviderConnection::QueryResult &queryResult, QObject *parent = nullptr );
+    QgsQueryResultModel(const QgsAbstractDatabaseProviderConnection::QueryResult& queryResult, QObject *parent = nullptr );
+
+    ~QgsQueryResultModel();
 
     // QAbstractItemModel interface
   public:
 
     int rowCount( const QModelIndex &parent ) const override;
     int columnCount( const QModelIndex &parent ) const override;
     QVariant data( const QModelIndex &index, int role ) const override;
-    void fetchMore( const QModelIndex &parent ) override;
-    bool canFetchMore( const QModelIndex &parent ) const override;
+
+  public slots:
+
+    void newRowsReady(int newRowsCount );
 
   private:
 
-    QgsAbstractDatabaseProviderConnection::QueryResult mQueryResult;
+    const QgsAbstractDatabaseProviderConnection::QueryResult& mQueryResult;
+    QStringList mColumns;
     qlonglong mRowCount = 0;
-    // Batch of rows to fetch
-    static const int ROWS_TO_FETCH;
+    QThread mWorkerThread;
+
 };
 
 #endif // qgsqueryresultmodel.h