Add provider suite test for thread safety of provider getFeatures()

Hammers providers by iterating through features from multiple threads
qgis · Jan 16, 2018 · 741afdc · 741afdc
1 parent c2e0775
commit 741afdc
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 0 deletions.
diff --git a/python/core/core_auto.sip b/python/core/core_auto.sip
@@ -423,4 +423,5 @@
 %Include layertree/qgslayertreeregistrybridge.sip
 %Include qgsuserprofilemanager.sip
 %Include symbology/qgsarrowsymbollayer.sip
+%Include qgstestutils.sip
 %Include qgsuserprofile.sip
diff --git a/python/core/qgstestutils.sip b/python/core/qgstestutils.sip
@@ -0,0 +1,36 @@
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgstestutils.h                                              *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/
+
+
+
+
+%ModuleHeaderCode
+#include "qgstestutils.h"
+%End
+
+
+namespace QgsTestUtils
+{
+
+  bool testProviderIteratorThreadSafety( QgsVectorDataProvider *provider, const QgsFeatureRequest &request = QgsFeatureRequest() );
+%Docstring
+Runs a thready safety test on iterators from a vector data ``provider``, by concurrently
+requesting features from the provider.
+
+This method returns true... or it segfaults.
+%End
+
+};
+
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgstestutils.h                                              *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
@@ -319,6 +319,7 @@ SET(QGIS_CORE_SRCS
   qgsxmlutils.cpp
   qgssettings.cpp
   qgsarchive.cpp
+  qgstestutils.cpp
   qgsziputils.cpp
 
   composer/qgsaddremoveitemcommand.cpp
@@ -995,6 +996,7 @@ SET(QGIS_CORE_HDRS
   qgsmapthemecollection.h
   qgsxmlutils.h
   qgsarchive.h
+  qgstestutils.h
   qgsziputils.h
   qgsvector.h
   qgslocalec.h

diff --git a/src/core/qgstestutils.cpp b/src/core/qgstestutils.cpp
@@ -0,0 +1,51 @@
+/***************************************************************************
+                                  qgstestutils.cpp
+                              --------------------
+    begin                : January 2018
+    copyright            : (C) 2018 by Nyall Dawson
+    email                : nyall.dawson@gmail.com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgstestutils.h"
+#include "qgsvectordataprovider.h"
+#include "qgsconnectionpool.h"
+#include <QtConcurrentMap>
+///@cond PRIVATE
+///
+
+static void getFeaturesForProvider( QPair< QgsVectorDataProvider *, QgsFeatureRequest > pair )
+{
+  QgsFeatureIterator it = pair.first->getFeatures( pair.second );
+  QgsFeature f;
+  while ( it.nextFeature( f ) )
+  {
+
+  }
+}
+
+bool QgsTestUtils::testProviderIteratorThreadSafety( QgsVectorDataProvider *provider, const QgsFeatureRequest &request )
+{
+  constexpr int JOBS_TO_RUN = 100;
+  QList< QPair< QgsVectorDataProvider *, QgsFeatureRequest > > jobs;
+  jobs.reserve( JOBS_TO_RUN );
+  for ( int i = 0; i < JOBS_TO_RUN; ++i )
+  {
+    jobs.append( qMakePair( provider, request ) );
+  }
+
+  //freaking hammmer the provider with a ton of concurrent requests.
+  //thread unsafe providers... you better be ready!!!!
+  QtConcurrent::blockingMap( jobs, getFeaturesForProvider );
+
+  return true;
+}
+
+
+///@endcond
diff --git a/src/core/qgstestutils.h b/src/core/qgstestutils.h
@@ -0,0 +1,48 @@
+/***************************************************************************
+                                  qgstestutils.h
+                              ---------------------
+    begin                : January 2018
+    copyright            : (C) 2018 by Nyall Dawson
+    email                : nyall.dawson@gmail.com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSTESTUTILS_H
+#define QGSTESTUTILS_H
+
+#include "qgis_core.h"
+#include "qgis.h"
+#include "qgsfeaturerequest.h"
+
+class QgsVectorDataProvider;
+
+#ifdef SIP_RUN
+% ModuleHeaderCode
+#include "qgstestutils.h"
+% End
+#endif
+
+///@cond PRIVATE
+
+// Used only for utilities required for the QGIS Python unit tests - not stable or public API
+namespace QgsTestUtils
+{
+
+  /**
+   * Runs a thready safety test on iterators from a vector data \a provider, by concurrently
+   * requesting features from the provider.
+   *
+   * This method returns true... or it segfaults.
+   */
+  CORE_EXPORT bool testProviderIteratorThreadSafety( QgsVectorDataProvider *provider, const QgsFeatureRequest &request = QgsFeatureRequest() );
+
+};
+
+///@endcond
+#endif //QGSTESTUTILS_H
diff --git a/tests/src/python/providertestbase.py b/tests/src/python/providertestbase.py
@@ -26,6 +26,7 @@
     QgsVectorDataProvider,
     QgsVectorLayerFeatureSource,
     QgsFeatureSink,
+    QgsTestUtils,
     NULL
 )
 
@@ -192,6 +193,15 @@ def getSubsetStringNoMatching(self):
         """Individual providers may need to override this depending on their subset string formats"""
         return '"name"=\'AppleBearOrangePear\''
 
+    def testGetFeaturesThreadSafety(self):
+        # no request
+        self.assertTrue(QgsTestUtils.testProviderIteratorThreadSafety(self.source))
+
+        # filter rect request
+        extent = QgsRectangle(-73, 70, -63, 80)
+        request = QgsFeatureRequest().setFilterRect(extent)
+        self.assertTrue(QgsTestUtils.testProviderIteratorThreadSafety(self.source, request))
+
     def testOrderBy(self):
         try:
             self.disableCompiler()

diff --git a/tests/src/python/test_provider_virtual.py b/tests/src/python/test_provider_virtual.py
@@ -81,6 +81,11 @@ def setUp(self):
         print("****************************************************")
         pass
 
+    def testGetFeaturesThreadSafety(self):
+        # provider does not work with this test - sqlite mutex prevents
+        # execution
+        pass
+
     def tearDown(self):
         """Run after each test."""
         pass

diff --git a/tests/testdata/provider/bug_17795.gpkg b/tests/testdata/provider/bug_17795.gpkg