[processing] Add test suite

qgis · Feb 4, 2016 · ff04fd3 · ff04fd3
1 parent 042a3db
commit ff04fd3
Show file tree

Hide file tree

Showing 15 changed files with 436 additions and 96 deletions.
diff --git a/python/plugins/processing/core/ProcessingLog.py b/python/plugins/processing/core/ProcessingLog.py
@@ -60,7 +60,7 @@ def addToLog(msgtype, msg):
             # It seems that this fails sometimes depending on the msg
             # added. To avoid it stopping the normal functioning of the
             # algorithm, we catch all errors, assuming that is better
-            # to miss some log info that breaking the algorithm.
+            # to miss some log info than breaking the algorithm.
             if msgtype == ProcessingLog.LOG_ALGORITHM:
                 line = msgtype + '|' + datetime.datetime.now().strftime(
                     ProcessingLog.DATE_FORMAT) + '|' \

diff --git a/python/plugins/processing/tests/AlgorithmsTest.py b/python/plugins/processing/tests/AlgorithmsTest.py
@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+
+"""
+***************************************************************************
+    test_algorithms.py
+    ---------------------
+    Date                 : January 2016
+    Copyright            : (C) 2016 by Matthias Kuhn
+    Email                : matthias@opengis.ch
+***************************************************************************
+*                                                                         *
+*   This program is free software; you can redistribute it and/or modify  *
+*   it under the terms of the GNU General Public License as published by  *
+*   the Free Software Foundation; either version 2 of the License, or     *
+*   (at your option) any later version.                                   *
+*                                                                         *
+***************************************************************************
+"""
+
+__author__ = 'Matthias Kuhn'
+__date__ = 'January 2016'
+__copyright__ = '(C) 2016, Matthias Kuhn'
+
+# This will get replaced with a git SHA1 when you do a git archive
+
+__revision__ = ':%H$'
+
+import qgis
+import os
+import shutil
+import yaml
+import nose2
+import gdal
+import hashlib
+import tempfile
+
+from osgeo.gdalconst import GA_ReadOnly
+
+import processing
+
+from processing.gui import AlgorithmExecutor
+
+from qgis.core import (
+    QgsVectorLayer,
+    QgsRasterLayer,
+    QgsMapLayerRegistry
+)
+
+from qgis.testing import (
+    start_app,
+    unittest
+)
+
+from utilities import (
+    unitTestDataPath
+)
+
+
+def processingTestDataPath():
+    return os.path.join(os.path.dirname(__file__), 'testdata')
+
+
+class TestAlgorithms(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        start_app()
+        from processing.core.Processing import Processing
+        Processing.initialize()
+        cls.cleanup_paths = []
+
+    @classmethod
+    def tearDownClass(cls):
+        for path in cls.cleanup_paths:
+            shutil.rmtree(path)
+
+    def test_algorithms(self):
+        """
+        This is the main test function. All others will be executed based on the definitions in testdata/algorithm_tests.yaml
+        """
+        with open(os.path.join(processingTestDataPath(), 'algorithm_tests.yaml'), 'r') as stream:
+            algorithm_tests = yaml.load(stream)
+
+        for algtest in algorithm_tests['tests']:
+            yield self.check_algorithm, algtest['name'], algtest
+
+    def check_algorithm(self, name, defs):
+        """
+        Will run an algorithm definition and check if it generates the expected result
+        :param name: The identifier name used in the test output heading
+        :param defs: A python dict containing a test algorithm definition
+        """
+        params = self.load_params(defs['params'])
+
+        alg = processing.Processing.getAlgorithm(defs['algorithm']).getCopy()
+
+        if isinstance(params, list):
+            for param in zip(alg.parameters, params):
+                param[0].setValue(param[1])
+        else:
+            for k, p in params.iteritems():
+                alg.setParameterValue(k, p)
+
+        for r, p in defs['results'].iteritems():
+            alg.setOutputValue(r, self.load_result_param(p))
+
+        self.assertTrue(AlgorithmExecutor.runalg(alg))
+        print(alg.getAsCommand())
+        self.check_results(alg.getOutputValuesAsDictionary(), defs['results'])
+
+    def load_params(self, params):
+        """
+        Loads an array of parameters
+        """
+        if type(params) == list:
+            return [self.load_param(p) for p in params]
+        elif type(params) == dict:
+            return {key: self.load_param(p) for key, p in params.iteritems()}
+        else:
+            return params
+
+    def load_param(self, param):
+        """
+        Loads a parameter. If it's not a map, the parameter will be returned as-is. If it is a map, it will process the
+        parameter based on its key `type` and return the appropriate parameter to pass to the algorithm.
+        """
+        try:
+            if param['type'] == 'vector' or param['type'] == 'raster':
+                return self.load_layer(param)
+            if param['type'] == 'multi':
+                return [self.load_param(p) for p in param['params']]
+        except TypeError:
+            # No type specified, use whatever is there
+            return param
+
+        raise KeyError("Unknown type '{}' specified for parameter '{}'".format(param['type'], param['name']))
+
+    def load_result_param(self, param):
+        """
+        Lodas a result parameter. Creates a temporary destination where the result should go to and returns this location
+        so it can be sent to the algorithm as parameter.
+        """
+        if param['type'] == 'vector':
+            outdir = tempfile.mkdtemp()
+            self.cleanup_paths.append(outdir)
+            basename = os.path.basename(param['name'])
+            filepath = os.path.join(outdir, basename)
+            return filepath
+
+        raise KeyError("Unknown type '{}' specified for parameter '{}'".format(param['type'], param['name']))
+
+    def load_layer(self, param):
+        """
+        Loads a layer which was specified as parameter.
+        """
+        prefix = processingTestDataPath()
+        try:
+            if param['location'] == 'qgs':
+                prefix = unitTestDataPath()
+        except KeyError:
+            pass
+
+        filepath = os.path.join(prefix, param['name'])
+
+        if param['type'] == 'vector':
+            lyr = QgsVectorLayer(filepath, param['name'], 'ogr')
+        elif param['type'] == 'raster':
+            lyr = QgsRasterLayer(filepath, param['name'], 'ogr')
+
+        self.assertTrue(lyr.isValid(), 'Could not load layer "{}"'.format(filepath))
+        QgsMapLayerRegistry.instance().addMapLayer(lyr)
+        return lyr
+
+    def check_results(self, results, expected):
+        """
+        Checks if result produced by an algorithm matches with the expected specification.
+        """
+        for id, expected_result in expected.iteritems():
+            if 'vector' == expected_result['type']:
+                expected_lyr = self.load_layer(expected_result)
+                try:
+                    results[id]
+                except KeyError as e:
+                    raise KeyError('Expected result {} does not exist in {}'.format(e.message, results.keys()))
+
+                result_lyr = QgsVectorLayer(results[id], id, 'ogr')
+
+                try:
+                    compare = expected_result['compare']
+                except KeyError:
+                    compare = {}
+
+                self.assertLayersEqual(expected_lyr, result_lyr, compare=compare)
+
+            elif 'rasterhash' == expected_result['type']:
+                dataset = gdal.Open(results[id], GA_ReadOnly)
+                strhash = hashlib.sha224(dataset.ReadAsArray(0).data).hexdigest()
+
+                self.assertEqual(strhash, expected_result['hash'])
+
+
+if __name__ == '__main__':
+    nose2.main()
diff --git a/python/plugins/processing/tests/CMakeLists.txt b/python/plugins/processing/tests/CMakeLists.txt
@@ -7,4 +7,5 @@ PLUGIN_INSTALL(processing tests/data ${TEST_DATA_FILES})
 IF(ENABLE_TESTS)
   INCLUDE(UsePythonTest)
   ADD_PYTHON_TEST(ProcessingParametersTest ParametersTest.py)
+  ADD_PYTHON_TEST(ProcessingAlgorithmsTest AlgorithmsTest.py)
 ENDIF(ENABLE_TESTS)
diff --git a/python/plugins/processing/tests/ParametersTest.py b/python/plugins/processing/tests/ParametersTest.py
@@ -2,7 +2,7 @@
 
 """
 ***************************************************************************
-    ParametersTest.py
+    ParametersTest
     ---------------------
     Date                 : March 2013
     Copyright            : (C) 2013 by Victor Olaya
@@ -25,9 +25,7 @@
 
 __revision__ = '$Format:%H$'
 
-import unittest
-from utilities import getQgisTestApp, unittest
-QGISAPP, CANVAS, IFACE, PARENT = getQgisTestApp()
+from qgis.testing import start_app, unittest
 
 from processing.core.parameters import (Parameter,
                                         ParameterBoolean,
@@ -42,6 +40,8 @@
 from qgis.core import (QgsRasterLayer,
                        QgsVectorLayer)
 
+start_app()
+
 
 class ParameterTest(unittest.TestCase):
 

diff --git a/python/plugins/processing/tests/README.md b/python/plugins/processing/tests/README.md
@@ -0,0 +1,118 @@
+Algorithm tests
+===============
+
+To test algorithms you can add entries into `testdata/algorithm_tests.yaml`.
+
+This file is structured with [yaml syntax](http://www.yaml.org/start.html).
+
+A basic test appears under the toplevel key `tests` and looks like this:
+
+  - name: centroid
+    algorithm: qgis:polygoncentroids
+    params:
+      - type: vector
+        location: qgs
+        name: polys.shp
+    results:
+      - id: OUTPUT_LAYER
+        type: vector
+        location: proc
+        name: polys_centroid.geojson
+
+How To
+------
+
+To add a new test you can follow these steps:
+
+Run the algorithm you want to test in QGIS from the processing toolbox. If the
+result is a vector layer prefer geojson as output for its support of mixed
+geometry types and good readability. Redirect output to
+`python/plugins/processing/tests/testdata/expected`
+
+When you have run the algorithm, go to "Processing" > "History" and find the
+algorithm which you have just run. This looks like
+
+    processing.runalg("qgis:densifygeometries","/home/mku/dev/cpp/qgis/QGIS/tests/testdata/polys.shp",2,"/home/mku/dev/cpp/qgis/QGIS/python/plugins/processing/tests/testdata/polys_densify.geojson")
+
+Open the file `python/plugins/processing/tests/testdata/algorithm_tests.yaml`,
+copy an existing test block and adjust it to your needs based on the
+information found in the history.
+
+The first string from the command goes to the key `algorithm`, the subsequent
+ones to params and the last one(s) to results.
+
+The above translates to
+
+    - name: densify
+      algorithm: qgis:densifygeometriesgivenaninterval
+      params:
+        - type: vector
+          location: qgs
+          name: polys.shp
+        - 2 # Interval
+      results:
+        - id: OUTPUT
+          type: vector
+          location: proc
+          name: expected/polys_densify.geojson
+
+Params and results
+------------------
+
+Trivial type parameters
+.......................
+
+Params and results are specified as lists:
+
+    params:
+      - 2
+      - string
+      - another param
+
+As in the example above they can be plain variables.
+
+Layer type parameters
+.....................
+
+To specify layers you will have to specify
+
+ * the type
+   * `vector` or `raster`
+ * a location to allow using files from the shared qgis test data
+   * `qgs` will look for the file in the src/tests/testdata
+   * `proc` will look for the file in python/plugins/processing/tests/testdata
+     you should use this location for expected data.
+ * a name
+   * relative path like `expected/polys_centroid.geojson`
+
+    params:
+      - 2
+      - string
+      - type: vector
+        location: qgs
+        name: polys.shp
+      - another param
+
+Results
+.......
+
+Results have a special key `id` which is required because an algorithm can
+produce multiple results. If you don't know the `id`, just start with `OUTPUT`
+and run the test. You will be told if it was wrong and about the possible
+values.
+
+To deal with a certain tolerance for output values you can specify a
+`compare` property for an output.
+
+For a vector layer this means
+
+    OUTPUT:
+      type: vector
+      name: expected/abcd.geojson
+    compare:
+      fields:
+        __all__:
+          precision: 5 # compare to a precision of .00001 on all fields
+        A: skip # skip field A
+      geometry:
+        precision: 5 # compare coordinates with a precision of 5 digits
diff --git a/python/plugins/processing/tests/testdata/algorithm_tests.yaml b/python/plugins/processing/tests/testdata/algorithm_tests.yaml
@@ -5,8 +5,8 @@ tests:
     algorithm: qgis:polygoncentroids # Algorithm name
     params: # A list of parameters (only 1 here)
       - type: vector    # Param is a vector layer
-        location: proc   # file is in the qgis tests/testdata directory
-        name: polys.geojson # file name
+        location: qgs   # file is in the qgis tests/testdata directory
+        name: polys.shp # file name
     results: # A map of results (only one here)
       OUTPUT_LAYER:
         type: vector   # Expected result is a vector layer

diff --git a/python/plugins/processing/tests/testdata/expected/frequency.csv b/python/plugins/processing/tests/testdata/expected/frequency.csv
@@ -0,0 +1,3 @@
+Name,FREQ
+Lake,6
+Dam,4