Make iteration of features from vector layers with joins actually thr…

…ead safe
qgis · Dec 14, 2022 · 6355518 · 6355518
1 parent eedf0d7
commit 6355518
Show file tree

Hide file tree

Showing 6 changed files with 84 additions and 14 deletions.
diff --git a/python/core/auto_generated/vector/qgsvectorlayerfeatureiterator.sip.in b/python/core/auto_generated/vector/qgsvectorlayerfeatureiterator.sip.in
@@ -71,6 +71,9 @@ Returns the layer id of the source layer.
 
 
 
+
+
+
   private:
     QgsVectorLayerFeatureSource( const QgsVectorLayerFeatureSource &other );
 };

diff --git a/python/core/auto_generated/vector/qgsvectorlayerjoininfo.sip.in b/python/core/auto_generated/vector/qgsvectorlayerjoininfo.sip.in
@@ -217,7 +217,22 @@ has been set.
 Returns the list of field names to use for joining considering
 blocklisted fields and subset.
 
+.. warning::
+
+   This method is NOT thread safe, and MUST be called from the thread where the vector layers
+   participating in the join reside. See variant which accepts a :py:class:`QgsFields` argument for a thread safe alternative.
+
 .. versionadded:: 3.0
+%End
+
+    static QStringList joinFieldNamesSubset( const QgsVectorLayerJoinInfo &info, const QgsFields &joinLayerFields, bool blocklisted = true );
+%Docstring
+Returns the list of field names to use for joining considering
+blocklisted fields and subset.
+
+This method is thread safe.
+
+.. versionadded:: 3.30
 %End
 
     bool operator==( const QgsVectorLayerJoinInfo &other ) const;

diff --git a/src/core/vector/qgsvectorlayerfeatureiterator.cpp b/src/core/vector/qgsvectorlayerfeatureiterator.cpp
@@ -46,6 +46,16 @@ QgsVectorLayerFeatureSource::QgsVectorLayerFeatureSource( const QgsVectorLayer *
     layer->mJoinBuffer->createJoinCaches();
 
   mJoinBuffer.reset( layer->mJoinBuffer->clone() );
+  for ( const QgsVectorLayerJoinInfo &joinInfo : mJoinBuffer->vectorJoins() )
+  {
+    if ( QgsVectorLayer *joinLayer = joinInfo.joinLayer() )
+    {
+      JoinLayerSource source;
+      source.joinSource = std::make_shared< QgsVectorLayerFeatureSource >( joinLayer );
+      source.joinLayerFields = joinLayer->fields();
+      mJoinSources.insert( joinLayer->id(), source );
+    }
+  }
 
   mExpressionFieldBuffer.reset( new QgsExpressionFieldBuffer( *layer->mExpressionFieldBuffer ) );
   mCrs = layer->crs();
@@ -753,19 +763,19 @@ void QgsVectorLayerFeatureIterator::prepareJoin( int fieldIdx )
   const QgsVectorLayerJoinInfo *joinInfo = mSource->mJoinBuffer->joinForFieldIndex( fieldIdx, mSource->mFields, sourceLayerIndex );
   Q_ASSERT( joinInfo );
 
-  QgsVectorLayer *joinLayer = joinInfo->joinLayer();
-  if ( !joinLayer )
+  auto joinSourceIt = mSource->mJoinSources.constFind( joinInfo->joinLayerId() );
+  if ( joinSourceIt == mSource->mJoinSources.constEnd() )
     return;  // invalid join (unresolved reference to layer)
 
   if ( !mFetchJoinInfo.contains( joinInfo ) )
   {
     FetchJoinInfo info;
     info.joinInfo = joinInfo;
-    info.joinSource = std::make_shared< QgsVectorLayerFeatureSource >( joinLayer );
+    info.joinSource = joinSourceIt->joinSource;
     info.indexOffset = mSource->mJoinBuffer->joinedFieldsOffset( joinInfo, mSource->mFields );
     info.targetField = mSource->mFields.indexFromName( joinInfo->targetFieldName() );
-    info.joinField = joinLayer->fields().indexFromName( joinInfo->joinFieldName() );
-    info.joinLayerFields = joinLayer->fields();
+    info.joinField = joinSourceIt->joinLayerFields.indexFromName( joinInfo->joinFieldName() );
+    info.joinLayerFields = joinSourceIt->joinLayerFields;
 
     // for joined fields, we always need to request the targetField from the provider too
     if ( !mPreparedFields.contains( info.targetField ) && !mFieldsToPrepare.contains( info.targetField ) )
@@ -1155,7 +1165,7 @@ void QgsVectorLayerFeatureIterator::FetchJoinInfo::addJoinedAttributesDirect( Qg
   // so we do not have to cache everything
   if ( joinInfo->hasSubset() )
   {
-    const QStringList subsetNames = QgsVectorLayerJoinInfo::joinFieldNamesSubset( *joinInfo );
+    const QStringList subsetNames = QgsVectorLayerJoinInfo::joinFieldNamesSubset( *joinInfo, joinLayerFields );
     const QVector<int> subsetIndices = QgsVectorLayerJoinBuffer::joinSubsetIndices( joinLayerFields, subsetNames );
     joinedAttributeIndices = qgis::setToList( qgis::listToSet( attributes ).intersect( qgis::listToSet( subsetIndices.toList() ) ) );
   }

diff --git a/src/core/vector/qgsvectorlayerfeatureiterator.h b/src/core/vector/qgsvectorlayerfeatureiterator.h
@@ -94,7 +94,35 @@ class CORE_EXPORT QgsVectorLayerFeatureSource : public QgsAbstractFeatureSource
   protected:
 
     std::unique_ptr< QgsAbstractFeatureSource > mProviderFeatureSource;
+
     std::unique_ptr< QgsVectorLayerJoinBuffer > mJoinBuffer;
+
+#ifndef SIP_RUN
+
+    /**
+     * Contains join layer source information prepared in a thread-safe way, ready for vector
+     * layer feature iterators with joins to utilize.
+     *
+     * \since QGIS 3.30
+     */
+    struct JoinLayerSource
+    {
+
+      /**
+       * Feature source for join
+       */
+      std::shared_ptr< QgsVectorLayerFeatureSource > joinSource;
+
+      /**
+       * Fields from joined layer.
+       */
+      QgsFields joinLayerFields;
+    };
+
+    //! Contains prepared join sources by layer ID
+    QMap< QString, JoinLayerSource > mJoinSources;
+#endif
+
     std::unique_ptr< QgsExpressionFieldBuffer > mExpressionFieldBuffer;
 
     QgsFields mFields;

diff --git a/src/core/vector/qgsvectorlayerjoininfo.cpp b/src/core/vector/qgsvectorlayerjoininfo.cpp
@@ -84,6 +84,11 @@ QgsFeature QgsVectorLayerJoinInfo::extractJoinedFeature( const QgsFeature &featu
 }
 
 QStringList QgsVectorLayerJoinInfo::joinFieldNamesSubset( const QgsVectorLayerJoinInfo &info, bool blocklisted )
+{
+  return joinFieldNamesSubset( info, info.joinLayer() ? info.joinLayer()->fields() : QgsFields(), blocklisted );
+}
+
+QStringList QgsVectorLayerJoinInfo::joinFieldNamesSubset( const QgsVectorLayerJoinInfo &info, const QgsFields &joinLayerFields, bool blocklisted )
 {
   QStringList fieldNames;
 
@@ -100,15 +105,11 @@ QStringList QgsVectorLayerJoinInfo::joinFieldNamesSubset( const QgsVectorLayerJo
     }
     else
     {
-      if ( auto *lJoinLayer = info.joinLayer() )
+      for ( const QgsField &f : joinLayerFields )
       {
-        const QgsFields fields { lJoinLayer->fields() };
-        for ( const QgsField &f : fields )
-        {
-          if ( !info.joinFieldNamesBlockList().contains( f.name() )
-               && f.name() != info.joinFieldName() )
-            fieldNames.append( f.name() );
-        }
+        if ( !info.joinFieldNamesBlockList().contains( f.name() )
+             && f.name() != info.joinFieldName() )
+          fieldNames.append( f.name() );
       }
     }
   }

diff --git a/src/core/vector/qgsvectorlayerjoininfo.h b/src/core/vector/qgsvectorlayerjoininfo.h
@@ -187,10 +187,23 @@ class CORE_EXPORT QgsVectorLayerJoinInfo
      * Returns the list of field names to use for joining considering
      * blocklisted fields and subset.
      *
+     * \warning This method is NOT thread safe, and MUST be called from the thread where the vector layers
+     * participating in the join reside. See variant which accepts a QgsFields argument for a thread safe alternative.
+     *
      * \since QGIS 3.0
      */
     static QStringList joinFieldNamesSubset( const QgsVectorLayerJoinInfo &info, bool blocklisted = true );
 
+    /**
+     * Returns the list of field names to use for joining considering
+     * blocklisted fields and subset.
+     *
+     * This method is thread safe.
+     *
+     * \since QGIS 3.30
+     */
+    static QStringList joinFieldNamesSubset( const QgsVectorLayerJoinInfo &info, const QgsFields &joinLayerFields, bool blocklisted = true );
+
     bool operator==( const QgsVectorLayerJoinInfo &other ) const
     {
       return mTargetFieldName == other.mTargetFieldName &&