Merge pull request #4799 from nyalldawson/m2

Run processing algorithms using task manager
qgis · Jul 7, 2017 · 52f4c5e · 52f4c5e
2 parents 29c4fcf + 240dd19
commit 52f4c5e
Show file tree

Hide file tree

Showing 52 changed files with 1,323 additions and 515 deletions.
diff --git a/python/analysis/vector/qgsgeometrysnapper.sip b/python/analysis/vector/qgsgeometrysnapper.sip
@@ -33,11 +33,11 @@ class QgsGeometrySnapper : QObject
       EndPointToEndPoint,
     };
 
-    QgsGeometrySnapper( QgsVectorLayer *referenceLayer );
+    QgsGeometrySnapper( QgsFeatureSource *referenceSource );
 %Docstring
- Constructor for QgsGeometrySnapper. A reference layer which contains geometries to snap to must be
+ Constructor for QgsGeometrySnapper. A reference feature source which contains geometries to snap to must be
  set. It is assumed that all geometries snapped using this object will have the
- same CRS as the reference layer (ie, no reprojection is performed).
+ same CRS as the reference source (ie, no reprojection is performed).
 %End
 
     QgsGeometry snapGeometry( const QgsGeometry &geometry, double snapTolerance, SnapMode mode = PreferNodes ) const;

diff --git a/python/core/core.sip b/python/core/core.sip
@@ -104,5 +104,7 @@ done:
 %Include core_auto.sip
 
 %VirtualErrorHandler processing_exception_handler
-    throw QgsProcessingException( getTraceback() );
+    QString trace = getTraceback();
+    SIP_RELEASE_GIL( sipGILState );
+    throw QgsProcessingException( trace );
 %End
diff --git a/python/core/processing/qgsprocessingalgorithm.sip b/python/core/processing/qgsprocessingalgorithm.sip
@@ -43,6 +43,13 @@ class QgsProcessingAlgorithm
     virtual ~QgsProcessingAlgorithm();
 
 
+
+    virtual QgsProcessingAlgorithm *create() const = 0 /Factory/;
+%Docstring
+ Creates a copy of the algorithm, ready for execution.
+ :rtype: QgsProcessingAlgorithm
+%End
+
     virtual QString name() const = 0;
 %Docstring
  Returns the algorithm name, used for identifying the algorithm. This string
@@ -217,7 +224,9 @@ class QgsProcessingAlgorithm
     QVariantMap run( const QVariantMap &parameters,
                      QgsProcessingContext &context, QgsProcessingFeedback *feedback, bool *ok /Out/ = 0 ) const;
 %Docstring
- Executes the algorithm using the specified ``parameters``.
+ Executes the algorithm using the specified ``parameters``. This method internally
+ creates a copy of the algorithm before running it, so it is safe to call
+ on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider.
 
  The ``context`` argument specifies the context in which the algorithm is being run.
 
@@ -227,6 +236,55 @@ class QgsProcessingAlgorithm
 
  :return: A map of algorithm outputs. These may be output layer references, or calculated
  values such as statistical calculations.
+
+.. note::
+
+   this method can only be called from the main thread. Use prepare(), runPrepared() and postProcess()
+ if you need to run algorithms from a background thread, or use the QgsProcessingAlgRunnerTask class.
+ :rtype: QVariantMap
+%End
+
+    bool prepare( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );
+%Docstring
+ Prepares the algorithm for execution. This must be run in the main thread, and allows the algorithm
+ to pre-evaluate input parameters in a thread-safe manner. This must be called before
+ calling runPrepared() (which is safe to do in any thread).
+.. seealso:: runPrepared()
+.. seealso:: postProcess()
+.. note::
+
+   This method modifies the algorithm instance, so it is not safe to call
+ on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
+ of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
+ :rtype: bool
+%End
+
+    QVariantMap runPrepared( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );
+%Docstring
+ Runs the algorithm, which has been prepared by an earlier call to prepare().
+ This method is safe to call from any thread. Returns true if the algorithm was successfully executed.
+ After runPrepared() has finished, the postProcess() method should be called from the main thread
+ to allow the algorithm to perform any required cleanup tasks and return its final result.
+.. seealso:: prepare()
+.. seealso:: postProcess()
+.. note::
+
+   This method modifies the algorithm instance, so it is not safe to call
+ on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
+ of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
+ :rtype: QVariantMap
+%End
+
+    QVariantMap postProcess( QgsProcessingContext &context, QgsProcessingFeedback *feedback );
+%Docstring
+ Should be called in the main thread following the completion of runPrepared(). This method
+ allows the algorithm to perform any required cleanup tasks. The returned variant map
+ includes the results evaluated by the algorithm.
+.. note::
+
+   This method modifies the algorithm instance, so it is not safe to call
+ on algorithms directly retrieved from QgsProcessingRegistry and QgsProcessingProvider. Instead, a copy
+ of the algorithm should be created with clone() and prepare()/runPrepared() called on the copy.
  :rtype: QVariantMap
 %End
 
@@ -298,20 +356,99 @@ class QgsProcessingAlgorithm
  :rtype: bool
 %End
 
-    virtual QVariantMap processAlgorithm( const QVariantMap &parameters,
-                                          QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const = 0 /VirtualErrorHandler=processing_exception_handler/;
+    virtual bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) /VirtualErrorHandler=processing_exception_handler/;
+%Docstring
+ Prepares the algorithm to run using the specified ``parameters``. Algorithms should implement
+ their logic for evaluating parameter values here. The evaluated parameter results should
+ be stored in member variables ready for a call to processAlgorithm().
+
+ The ``context`` argument specifies the context in which the algorithm is being run.
+
+ prepareAlgorithm should be used to handle any thread-sensitive preparation which is required
+ by the algorithm. It will always be called from the same thread that ``context`` has thread
+ affinity with. While this will generally be the main thread, it is not guaranteed. For instance,
+ algorithms which are run as a step in a larger model or as a subcomponent of a script-based algorithm
+ will call prepareAlgorithm from the same thread as that model/script it being executed in.
+
+ Note that the processAlgorithm step uses a temporary context with affinity for the thread in
+ which the algorithm is executed, making it safe for processAlgorithm implementations to load
+ sources and sinks without issue. Implementing prepareAlgorithm is only required if special
+ thread safe handling is required by the algorithm.
+
+ Algorithm preparation progress should be reported using the supplied ``feedback`` object. Additionally,
+ well-behaved algorithms should periodically check ``feedback`` to determine whether the
+ algorithm should be canceled and exited early.
+
+ If the preparation was successful algorithms must return true. If a false value is returned
+ this indicates that the preparation could not be completed, and the algorithm execution
+ will be canceled.
+
+ :return: true if preparation was successful.
+.. seealso:: processAlgorithm()
+.. seealso:: postProcessAlgorithm()
+ :rtype: bool
+%End
+
+    virtual QVariantMap processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) = 0 /VirtualErrorHandler=processing_exception_handler/;
 %Docstring
  Runs the algorithm using the specified ``parameters``. Algorithms should implement
  their custom processing logic here.
 
- The ``context`` argument specifies the context in which the algorithm is being run.
+ The ``context`` argument gives a temporary context with thread affinity matching the thread
+ in which the algorithm is being run. This is a cut-back copy of the context passed to
+ the prepareAlgorithm() and postProcessAlgorithm() steps, but it is generally safe
+ for most algorithms to utilize this context for loading layers and creating sinks.
+ Any loaded layers or sinks created within this temporary context will be transferred
+ back to the main execution context upon successful completion of the processAlgorithm()
+ step.
 
  Algorithm progress should be reported using the supplied ``feedback`` object. Additionally,
  well-behaved algorithms should periodically check ``feedback`` to determine whether the
  algorithm should be canceled and exited early.
 
+ This method will not be called if the prepareAlgorithm() step failed (returned false).
+
+ c++ implementations of processAlgorithm can throw the QgsProcessingException exception
+ to indicate that a fatal error occurred within the execution. Python based subclasses
+ should raise GeoAlgorithmExecutionException for the same purpose.
+
  :return: A map of algorithm outputs. These may be output layer references, or calculated
+ values such as statistical calculations. Unless the algorithm subclass overrides
+ the postProcessAlgorithm() step this returned map will be used as the output for the
+ algorithm.
+
+.. seealso:: prepareAlgorithm()
+.. seealso:: postProcessAlgorithm()
+ :rtype: QVariantMap
+%End
+
+    virtual QVariantMap postProcessAlgorithm( QgsProcessingContext &context, QgsProcessingFeedback *feedback ) /VirtualErrorHandler=processing_exception_handler/;
+%Docstring
+ Allows the algorithm to perform any required cleanup tasks. The returned variant map
+ includes the results evaluated by the algorithm. These may be output layer references, or calculated
  values such as statistical calculations.
+
+ The ``context`` argument specifies the context in which the algorithm was run.
+
+ Postprocess progress should be reported using the supplied ``feedback`` object. Additionally,
+ well-behaved algorithms should periodically check ``feedback`` to determine whether the
+ post processing should be canceled and exited early.
+
+ postProcessAlgorithm should be used to handle any thread-sensitive cleanup which is required
+ by the algorithm. It will always be called from the same thread that ``context`` has thread
+ affinity with. While this will generally be the main thread, it is not guaranteed. For instance,
+ algorithms which are run as a step in a larger model or as a subcomponent of a script-based algorithm
+ will call postProcessAlgorithm from the same thread as that model/script it being executed in.
+
+ postProcessAlgorithm will not be called if the prepareAlgorithm() step failed (returned false),
+ or if an exception was raised by the processAlgorithm() step.
+
+ :return: A map of algorithm outputs. These may be output layer references, or calculated
+ values such as statistical calculations. Implementations which return a non-empty
+ map will override any results returned by processAlgorithm().
+
+.. seealso:: prepareAlgorithm()
+.. seealso:: processAlgorithm()
  :rtype: QVariantMap
 %End
 

diff --git a/python/core/processing/qgsprocessingalgrunnertask.sip b/python/core/processing/qgsprocessingalgrunnertask.sip
@@ -24,14 +24,24 @@ class QgsProcessingAlgRunnerTask : QgsTask
 
     QgsProcessingAlgRunnerTask( const QgsProcessingAlgorithm *algorithm,
                                 const QVariantMap &parameters,
-                                QgsProcessingContext &context );
+                                QgsProcessingContext &context,
+                                QgsProcessingFeedback *feedback = 0 );
 %Docstring
  Constructor for QgsProcessingAlgRunnerTask. Takes an ``algorithm``, algorithm ``parameters``
  and processing ``context``.
 %End
 
     virtual void cancel();
 
+  signals:
+
+    void executed( bool successful, const QVariantMap &results );
+%Docstring
+ Emitted when the algorithm has finished execution. If the algorithm completed
+ execution without errors then ``successful`` will be true. The ``results`` argument
+ contains the results reported by the algorithm.
+%End
+
   protected:
 
     virtual bool run();

diff --git a/python/core/processing/qgsprocessingcontext.sip b/python/core/processing/qgsprocessingcontext.sip
@@ -38,6 +38,12 @@ class QgsProcessingContext
 %End
 
 
+    void copyThreadSafeSettings( const QgsProcessingContext &other );
+%Docstring
+ Copies all settings which are safe for use across different threads from
+ ``other`` to this context.
+%End
+
     QgsProcessingContext::Flags flags() const;
 %Docstring
  Returns any flags set in the context.
@@ -126,7 +132,6 @@ Destination project
 .. seealso:: layersToLoadOnCompletion()
 %End
 
-
     QgsFeatureRequest::InvalidGeometryCheck invalidGeometryCheck() const;
 %Docstring
  Returns the behavior used for checking invalid geometries in input layers.
@@ -216,6 +221,39 @@ Destination project
 .. seealso:: setFeedback()
 %End
 
+    QThread *thread();
+%Docstring
+ Returns the thread in which the context lives.
+.. seealso:: pushToThread()
+ :rtype: QThread
+%End
+
+    void pushToThread( QThread *thread );
+%Docstring
+ Pushes the thread affinity for the context (including all layers contained in the temporaryLayerStore()) into
+ another ``thread``. This method is only safe to call when the current thread matches the existing thread
+ affinity for the context (see thread()).
+.. seealso:: thread()
+%End
+
+    void takeResultsFrom( QgsProcessingContext &context );
+%Docstring
+ Takes the results from another ``context`` and merges them with the results currently
+ stored in this context. This includes settings like any layers loaded in the temporaryLayerStore()
+ and layersToLoadOnCompletion().
+ This is only safe to call when both this context and the other ``context`` share the same
+ thread() affinity, and that thread is the current thread.
+%End
+
+    QgsMapLayer *takeResultLayer( const QString &id ) /TransferBack/;
+%Docstring
+ Takes the result map layer with matching ``id`` from the context and
+ transfers ownership of it back to the caller. This method can be used
+ to remove temporary layers which are not required for further processing
+ from a context.
+ :rtype: QgsMapLayer
+%End
+
   private:
     QgsProcessingContext( const QgsProcessingContext &other );
 };

diff --git a/python/core/processing/qgsprocessingmodelalgorithm.sip b/python/core/processing/qgsprocessingmodelalgorithm.sip
@@ -588,6 +588,8 @@ Copies are protected to avoid slicing
 
     virtual QString asPythonCommand( const QVariantMap &parameters, QgsProcessingContext &context ) const;
 
+    virtual QgsProcessingModelAlgorithm *create() const /Factory/;
+
 
     void setName( const QString &name );
 %Docstring
@@ -833,8 +835,8 @@ Copies are protected to avoid slicing
 
   protected:
 
-     virtual QVariantMap processAlgorithm( const QVariantMap &parameters,
-                                  QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const;
+    virtual QVariantMap processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback );
+
 
 };
 

diff --git a/python/core/processing/qgsprocessingparameters.sip b/python/core/processing/qgsprocessingparameters.sip
@@ -45,6 +45,12 @@ class QgsProcessingFeatureSourceDefinition
  True if only selected features in the source should be used by algorithms.
 %End
 
+    bool operator==( const QgsProcessingFeatureSourceDefinition &other );
+
+    bool operator!=( const QgsProcessingFeatureSourceDefinition &other );
+%Docstring
+ :rtype: bool
+%End
 
     operator QVariant() const;
 %Docstring

diff --git a/python/core/qgsmaplayerstore.sip b/python/core/qgsmaplayerstore.sip
@@ -208,6 +208,13 @@ class QgsMapLayerStore : QObject
 .. seealso:: removeMapLayers()
 %End
 
+    void transferLayersFromStore( QgsMapLayerStore *other );
+%Docstring
+ Transfers all the map layers contained within another map layer store and adds
+ them to this store.
+ Note that ``other`` and this store must have the same thread affinity.
+%End
+
   signals:
 
     void layersWillBeRemoved( const QStringList &layerIds );

diff --git a/python/plugins/processing/algs/qgis/Aspect.py b/python/plugins/processing/algs/qgis/Aspect.py
@@ -47,9 +47,9 @@
 
 class Aspect(QgisAlgorithm):
 
-    INPUT_LAYER = 'INPUT_LAYER'
+    INPUT = 'INPUT'
     Z_FACTOR = 'Z_FACTOR'
-    OUTPUT_LAYER = 'OUTPUT_LAYER'
+    OUTPUT = 'OUTPUT'
 
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'dem.png'))
@@ -60,13 +60,13 @@ def group(self):
     def __init__(self):
         super().__init__()
 
-        self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT_LAYER,
+        self.addParameter(QgsProcessingParameterRasterLayer(self.INPUT,
                                                             self.tr('Elevation layer')))
         self.addParameter(QgsProcessingParameterNumber(self.Z_FACTOR,
                                                        self.tr('Z factor'), QgsProcessingParameterNumber.Double,
                                                        1, False, 1, 999999.99))
-        self.addParameter(QgsProcessingParameterRasterOutput(self.OUTPUT_LAYER, self.tr('Aspect')))
-        self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT_LAYER, self.tr('Aspect')))
+        self.addParameter(QgsProcessingParameterRasterOutput(self.OUTPUT, self.tr('Aspect')))
+        self.addOutput(QgsProcessingOutputRasterLayer(self.OUTPUT, self.tr('Aspect')))
 
     def name(self):
         return 'aspect'
@@ -75,15 +75,15 @@ def displayName(self):
         return self.tr('Aspect')
 
     def processAlgorithm(self, parameters, context, feedback):
-        inputFile = exportRasterLayer(self.parameterAsRasterLayer(parameters, self.INPUT_LAYER, context))
+        inputFile = exportRasterLayer(self.parameterAsRasterLayer(parameters, self.INPUT, context))
         zFactor = self.parameterAsDouble(parameters, self.Z_FACTOR, context)
 
-        outputFile = self.parameterAsRasterOutputLayer(parameters, self.OUTPUT_LAYER, context)
+        outputFile = self.parameterAsRasterOutputLayer(parameters, self.OUTPUT, context)
 
         outputFormat = raster.formatShortNameFromFileName(outputFile)
 
         aspect = QgsAspectFilter(inputFile, outputFile, outputFormat)
         aspect.setZFactor(zFactor)
         aspect.processRaster(feedback)
 
-        return {self.OUTPUT_LAYER: outputFile}
+        return {self.OUTPUT: outputFile}