Merge pull request #4702 from nyalldawson/processing_pt31

More processing goodness, restore algs
qgis · Jun 12, 2017 · 40cae29 · 40cae29
2 parents 0ceeb29 + 2779260
commit 40cae29
Show file tree

Hide file tree

Showing 35 changed files with 1,020 additions and 349 deletions.
diff --git a/python/core/__init__.py b/python/core/__init__.py
@@ -33,6 +33,7 @@
 import functools
 from qgis._core import *
 
+
 # Boolean evaluation of QgsGeometry
 
 
@@ -44,7 +45,8 @@ def _geometryNonZero(self):
 QgsGeometry.__bool__ = _geometryNonZero
 
 
-def register_function(function, arg_count, group, usesgeometry=False, referenced_columns=[QgsFeatureRequest.ALL_ATTRIBUTES], **kwargs):
+def register_function(function, arg_count, group, usesgeometry=False,
+                      referenced_columns=[QgsFeatureRequest.ALL_ATTRIBUTES], **kwargs):
     """
     Register a Python function to be used as a expression function.
 
@@ -71,9 +73,11 @@ def myfunc(values, *args):
     :param usesgeometry:
     :return:
     """
+
     class QgsPyExpressionFunction(QgsExpressionFunction):
 
-        def __init__(self, func, name, args, group, helptext='', usesGeometry=True, referencedColumns=QgsFeatureRequest.ALL_ATTRIBUTES, expandargs=False):
+        def __init__(self, func, name, args, group, helptext='', usesGeometry=True,
+                     referencedColumns=QgsFeatureRequest.ALL_ATTRIBUTES, expandargs=False):
             QgsExpressionFunction.__init__(self, name, args, group, helptext)
             self.function = func
             self.expandargs = expandargs
@@ -126,13 +130,16 @@ def referencedColumns(self, node):
     if register and QgsExpression.isFunctionName(name):
         if not QgsExpression.unregisterFunction(name):
             msgtitle = QCoreApplication.translate("UserExpressions", "User expressions")
-            msg = QCoreApplication.translate("UserExpressions", "The user expression {0} already exists and could not be unregistered.").format(name)
+            msg = QCoreApplication.translate("UserExpressions",
+                                             "The user expression {0} already exists and could not be unregistered.").format(
+                name)
             QgsMessageLog.logMessage(msg + "\n", msgtitle, QgsMessageLog.WARNING)
             return None
 
     function.__name__ = name
     helptext = helptemplate.safe_substitute(name=name, doc=helptext)
-    f = QgsPyExpressionFunction(function, name, arg_count, group, helptext, usesgeometry, referenced_columns, expandargs)
+    f = QgsPyExpressionFunction(function, name, arg_count, group, helptext, usesgeometry, referenced_columns,
+                                expandargs)
 
     # This doesn't really make any sense here but does when used from a decorator context
     # so it can stay.
@@ -163,6 +170,7 @@ def add(values, *args):
 
     def wrapper(func):
         return register_function(func, args, group, **kwargs)
+
     return wrapper
 
 
@@ -174,6 +182,7 @@ def __init__(self, value):
     def __str__(self):
         return repr(self.value)
 
+
 # Define a `with edit(layer)` statement
 
 
@@ -264,3 +273,20 @@ def calculation_finished(exception, value=None):
 
 
 QgsTask.fromFunction = fromFunction
+
+
+# add some __repr__ methods to processing classes
+def processing_source_repr(self):
+    return "<QgsProcessingFeatureSourceDefinition {{'source':{}, 'selectedFeaturesOnly': {}}}>".format(
+        self.source.staticValue(), self.selectedFeaturesOnly)
+
+
+QgsProcessingFeatureSourceDefinition.__repr__ = processing_source_repr
+
+
+def processing_output_layer_repr(self):
+    return "<QgsProcessingOutputLayerDefinition {{'sink':{}, 'createOptions': {}}}>".format(self.sink.staticValue(),
+                                                                                            self.createOptions)
+
+
+QgsProcessingOutputLayerDefinition.__repr__ = processing_output_layer_repr
diff --git a/python/core/processing/qgsprocessingalgorithm.sip b/python/core/processing/qgsprocessingalgorithm.sip
@@ -29,6 +29,7 @@ class QgsProcessingAlgorithm
       FlagHideFromModeler,
       FlagSupportsBatch,
       FlagCanCancel,
+      FlagRequiresMatchingCrs,
       FlagDeprecated,
     };
     typedef QFlags<QgsProcessingAlgorithm::Flag> Flags;
@@ -244,6 +245,15 @@ class QgsProcessingAlgorithm
  :rtype: QgsExpressionContext
 %End
 
+    virtual bool validateInputCrs( const QVariantMap &parameters,
+                                   QgsProcessingContext &context ) const;
+%Docstring
+ Checks whether the coordinate reference systems for the specified set of ``parameters``
+ are valid for the algorithm. For instance, the base implementation performs
+ checks to ensure that all input CRS are equal
+ Returns true if ``parameters`` have passed the CRS check.
+ :rtype: bool
+%End
 
   protected:
 

diff --git a/python/core/processing/qgsprocessingcontext.sip b/python/core/processing/qgsprocessingcontext.sip
@@ -163,6 +163,28 @@ Destination project
 %End
 
 
+    void setTransformErrorCallback( SIP_PYCALLABLE / AllowNone / );
+%Docstring
+ Sets a callback function to use when encountering a transform error when iterating
+ features. This function will be
+ called using the feature which encountered the transform error as a parameter.
+.. versionadded:: 3.0
+.. seealso:: transformErrorCallback()
+%End
+%MethodCode
+    Py_BEGIN_ALLOW_THREADS
+
+    sipCpp->setTransformErrorCallback( [a0]( const QgsFeature &arg )
+    {
+      SIP_BLOCK_THREADS
+      Py_XDECREF( sipCallMethod( NULL, a0, "D", &arg, sipType_QgsFeature, NULL ) );
+      SIP_UNBLOCK_THREADS
+    } );
+
+    Py_END_ALLOW_THREADS
+%End
+
+
     QString defaultEncoding() const;
 %Docstring
  Returns the default encoding to use for newly created files.

diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml
@@ -314,6 +314,8 @@ qgis:mergevectorlayers: >
 
   If attributes tables are different, the attribute table of the resulting layer will contain the attributes from both input layers.
 
+  The layers will all be reprojected to match the coordinate reference system of the first input layer.
+
 qgis:multiparttosingleparts: >
   This algorithm takes a vector layer with multipart geometries and generates a new one in which all geometries contain a single part. Features with multipart geometries are divided in as many different features as parts the geometry contain, and the same attributes are used for each of them.
 

diff --git a/python/plugins/processing/algs/qgis/BasicStatistics.py b/python/plugins/processing/algs/qgis/BasicStatistics.py
@@ -94,7 +94,7 @@ def __init__(self):
                                                            self.tr('Field to calculate statistics on'),
                                                            None, self.INPUT_LAYER, QgsProcessingParameterTableField.Any))
 
-        self.addParameter(QgsProcessingParameterFileOutput(self.OUTPUT_HTML_FILE, self.tr('Statistics'), self.tr('HTML files (*.html)')))
+        self.addParameter(QgsProcessingParameterFileOutput(self.OUTPUT_HTML_FILE, self.tr('Statistics'), self.tr('HTML files (*.html)'), None, True))
         self.addOutput(QgsProcessingOutputHtml(self.OUTPUT_HTML_FILE, self.tr('Statistics')))
 
         self.addOutput(QgsProcessingOutputNumber(self.COUNT, self.tr('Count')))
@@ -149,9 +149,10 @@ def processAlgorithm(self, parameters, context, feedback):
             d, results = self.calcStringStats(features, feedback, field, count)
             data.extend(d)
 
-        self.createHTML(output_file, data)
+        if output_file:
+            self.createHTML(output_file, data)
+            results[self.OUTPUT_HTML_FILE] = output_file
 
-        results[self.OUTPUT_HTML_FILE] = output_file
         return results
 
     def calcNumericStats(self, features, feedback, field, count):

diff --git a/python/plugins/processing/algs/qgis/CheckValidity.py b/python/plugins/processing/algs/qgis/CheckValidity.py
@@ -41,7 +41,8 @@
                        QgsProcessingParameterEnum,
                        QgsProcessingParameterFeatureSink,
                        QgsProcessingOutputVectorLayer,
-                       QgsProcessingParameterDefinition
+                       QgsProcessingParameterDefinition,
+                       QgsProcessingOutputNumber
                        )
 from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm
 
@@ -54,8 +55,11 @@ class CheckValidity(QgisAlgorithm):
     INPUT_LAYER = 'INPUT_LAYER'
     METHOD = 'METHOD'
     VALID_OUTPUT = 'VALID_OUTPUT'
+    VALID_COUNT = 'VALID_COUNT'
     INVALID_OUTPUT = 'INVALID_OUTPUT'
+    INVALID_COUNT = 'INVALID_COUNT'
     ERROR_OUTPUT = 'ERROR_OUTPUT'
+    ERROR_COUNT = 'ERROR_COUNT'
 
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'ftools', 'check_geometry.png'))
@@ -76,10 +80,15 @@ def __init__(self):
 
         self.addParameter(QgsProcessingParameterFeatureSink(self.VALID_OUTPUT, self.tr('Valid output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
         self.addOutput(QgsProcessingOutputVectorLayer(self.VALID_OUTPUT, self.tr('Valid output')))
+        self.addOutput(QgsProcessingOutputNumber(self.VALID_COUNT, self.tr('Count of valid features')))
+
         self.addParameter(QgsProcessingParameterFeatureSink(self.INVALID_OUTPUT, self.tr('Invalid output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
         self.addOutput(QgsProcessingOutputVectorLayer(self.INVALID_OUTPUT, self.tr('Invalid output')))
+        self.addOutput(QgsProcessingOutputNumber(self.INVALID_COUNT, self.tr('Count of invalid features')))
+
         self.addParameter(QgsProcessingParameterFeatureSink(self.ERROR_OUTPUT, self.tr('Error output'), QgsProcessingParameterDefinition.TypeVectorAny, '', True))
         self.addOutput(QgsProcessingOutputVectorLayer(self.ERROR_OUTPUT, self.tr('Error output')))
+        self.addOutput(QgsProcessingOutputNumber(self.ERROR_COUNT, self.tr('Count of errors')))
 
     def name(self):
         return 'checkvalidity'
@@ -168,7 +177,11 @@ def doCheck(self, method, parameters, context, feedback):
 
             feedback.setProgress(int(current * total))
 
-        results = {}
+        results = {
+            self.VALID_COUNT: valid_count,
+            self.INVALID_COUNT: invalid_count,
+            self.ERROR_COUNT: error_count
+        }
         if valid_output_sink:
             results[self.VALID_OUTPUT] = valid_output_dest_id
         if invalid_output_sink:

diff --git a/python/plugins/processing/algs/qgis/Clip.py b/python/plugins/processing/algs/qgis/Clip.py
@@ -84,7 +84,7 @@ def processAlgorithm(self, parameters, context, feedback):
 
         # first build up a list of clip geometries
         clip_geoms = []
-        for mask_feature in mask_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([])):
+        for mask_feature in mask_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(feature_source.sourceCrs())):
             clip_geoms.append(mask_feature.geometry())
 
         # are we clipping against a single feature? if so, we can show finer progress reports

diff --git a/python/plugins/processing/algs/qgis/ExtractByExpression.py b/python/plugins/processing/algs/qgis/ExtractByExpression.py
@@ -27,7 +27,12 @@
 from qgis.core import (QgsExpression,
                        QgsFeatureRequest,
                        QgsApplication,
-                       QgsProcessingUtils)
+                       QgsProcessingUtils,
+                       QgsProcessingParameterFeatureSource,
+                       QgsProcessingParameterExpression,
+                       QgsProcessingParameterFeatureSink,
+                       QgsProcessingOutputVectorLayer,
+                       QgsProcessingParameterDefinition)
 
 from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
 from processing.core.parameters import ParameterVector
@@ -41,6 +46,7 @@ class ExtractByExpression(QgisAlgorithm):
     INPUT = 'INPUT'
     EXPRESSION = 'EXPRESSION'
     OUTPUT = 'OUTPUT'
+    FAIL_OUTPUT = 'FAIL_OUTPUT'
 
     def icon(self):
         return QgsApplication.getThemeIcon("/providerQgis.svg")
@@ -56,11 +62,16 @@ def group(self):
 
     def __init__(self):
         super().__init__()
-        self.addParameter(ParameterVector(self.INPUT,
-                                          self.tr('Input Layer')))
-        self.addParameter(ParameterExpression(self.EXPRESSION,
-                                              self.tr("Expression"), parent_layer=self.INPUT))
-        self.addOutput(OutputVector(self.OUTPUT, self.tr('Extracted (expression)')))
+        self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT,
+                                                              self.tr('Input layer')))
+        self.addParameter(QgsProcessingParameterExpression(self.EXPRESSION,
+                                                           self.tr('Expression'), None, self.INPUT))
+
+        self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Matching features')))
+        self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Matching (expression)')))
+        self.addParameter(QgsProcessingParameterFeatureSink(self.FAIL_OUTPUT, self.tr('Non-matching'),
+                                                            QgsProcessingParameterDefinition.TypeVectorAny, None, True))
+        self.addOutput(QgsProcessingOutputVectorLayer(self.FAIL_OUTPUT, self.tr('Non-matching (expression)')))
 
     def name(self):
         return 'extractbyexpression'
@@ -69,18 +80,48 @@ def displayName(self):
         return self.tr('Extract by expression')
 
     def processAlgorithm(self, parameters, context, feedback):
-        layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context)
-        expression_string = self.getParameterValue(self.EXPRESSION)
-        writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
-                                                                     context)
+        source = self.parameterAsSource(parameters, self.INPUT, context)
+        expression_string = self.parameterAsExpression(parameters, self.EXPRESSION, context)
+
+        (matching_sink, matching_sink_id) = self.parameterAsSink(parameters, self.OUTPUT, context,
+                                                                 source.fields(), source.wkbType(), source.sourceCrs())
+        (nonmatching_sink, non_matching_sink_id) = self.parameterAsSink(parameters, self.FAIL_OUTPUT, context,
+                                                                        source.fields(), source.wkbType(), source.sourceCrs())
 
         expression = QgsExpression(expression_string)
-        if not expression.hasParserError():
+        if expression.hasParserError():
+            raise GeoAlgorithmExecutionException(expression.parserErrorString())
+        expression_context = self.createExpressionContext(parameters, context)
+
+        if not nonmatching_sink:
+            # not saving failing features - so only fetch good features
             req = QgsFeatureRequest().setFilterExpression(expression_string)
+            req.setExpressionContext(expression_context)
+
+            for f in source.getFeatures(req):
+                if feedback.isCanceled():
+                    break
+                matching_sink.addFeature(f)
         else:
-            raise GeoAlgorithmExecutionException(expression.parserErrorString())
+            # saving non-matching features, so we need EVERYTHING
+            expression_context.setFields(source.fields())
+            expression.prepare(expression_context)
+
+            total = 100.0 / source.featureCount()
+
+            for current, f in enumerate(source.getFeatures()):
+                if feedback.isCanceled():
+                    break
+
+                expression_context.setFeature(f)
+                if expression.evaluate(expression_context):
+                    matching_sink.addFeature(f)
+                else:
+                    nonmatching_sink.addFeature(f)
 
-        for f in layer.getFeatures(req):
-            writer.addFeature(f)
+                feedback.setProgress(int(current * total))
 
-        del writer
+        results = {self.OUTPUT: matching_sink_id}
+        if nonmatching_sink:
+            results[self.FAIL_OUTPUT] = non_matching_sink_id
+        return results