[processing] Fixed SAGA handling of non-ascii output files

qgis · Feb 25, 2019 · a887b7d · a887b7d
1 parent 46f0e5d
commit a887b7d
Show file tree

Hide file tree

Showing 5 changed files with 155 additions and 12 deletions.
diff --git a/python/plugins/processing/algs/saga/SagaAlgorithm.py b/python/plugins/processing/algs/saga/SagaAlgorithm.py
@@ -27,6 +27,7 @@
 __revision__ = '$Format:%H$'
 
 import os
+import shutil
 import importlib
 from qgis.core import (Qgis,
                        QgsApplication,
@@ -305,14 +306,22 @@ def processAlgorithm(self, parameters, context, feedback):
 
         output_layers = []
         output_files = {}
+        #If the user has entered an output file that has non-ascii chars, we use a different path with only ascii chars
+        output_files_nonascii = {}
         for out in self.destinationParameterDefinitions():
             filePath = self.parameterAsOutputLayer(parameters, out.name(), context)
             if isinstance(out, (QgsProcessingParameterRasterDestination, QgsProcessingParameterVectorDestination)):
                 output_layers.append(filePath)
+                try:
+                    filePath.encode('ascii')
+                except UnicodeEncodeError:
+                    nonAsciiFilePath = filePath
+                    filePath = QgsProcessingUtils.generateTempFilename(out.name() + os.path.splitext(filePath)[1])
+                    output_files_nonascii[filePath] = nonAsciiFilePath
+
             output_files[out.name()] = filePath
             command += ' -{} "{}"'.format(out.name(), filePath)
-
-        commands.append(command)
+            commands.append(command)
 
         # special treatment for RGB algorithm
         # TODO: improve this and put this code somewhere else
@@ -341,6 +350,17 @@ def processAlgorithm(self, parameters, context, feedback):
                 with open(prjFile, 'w') as f:
                     f.write(crs.toWkt())
 
+        for old, new in output_files_nonascii.items():
+            oldFolder = os.path.dirname(old)
+            newFolder = os.path.dirname(new)
+            newName = os.path.splitext(os.path.basename(new))[0]
+            files = [f for f in os.listdir(oldFolder)]
+            for f in files:
+                ext = os.path.splitext(f)[1]
+                newPath = os.path.join(newFolder, newName + ext)
+                oldPath = os.path.join(oldFolder, f)
+                shutil.move(oldPath, newPath)
+
         result = {}
         for o in self.outputDefinitions():
             if o.name() in output_files:
@@ -421,8 +441,8 @@ def checkParameterValues(self, parameters, context):
 
             if isinstance(param, QgsProcessingParameterRasterLayer):
                 raster_layer_params.append(param.name())
-            elif (isinstance(param, QgsProcessingParameterMultipleLayers) and
-                    param.layerType() == QgsProcessing.TypeRaster):
+            elif (isinstance(param, QgsProcessingParameterMultipleLayers)
+                    and param.layerType() == QgsProcessing.TypeRaster):
                 raster_layer_params.extend(param.name())
 
         for layer_param in raster_layer_params:

diff --git a/python/plugins/processing/algs/saga/SagaUtils.py b/python/plugins/processing/algs/saga/SagaUtils.py
@@ -97,7 +97,7 @@ def sagaDescriptionPath():
 
 def createSagaBatchJobFileFromSagaCommands(commands):
 
-    with open(sagaBatchJobFilename(), 'w') as fout:
+    with open(sagaBatchJobFilename(), 'w', encoding="utf8") as fout:
         if isWindows():
             fout.write('set SAGA=' + sagaPath() + '\n')
             fout.write('set SAGA_MLB=' + os.path.join(sagaPath(), 'modules') + '\n')
@@ -108,12 +108,7 @@ def createSagaBatchJobFileFromSagaCommands(commands):
         else:
             pass
         for command in commands:
-            try:
-                # Python 2
-                fout.write('saga_cmd ' + command.encode('utf8') + '\n')
-            except TypeError:
-                # Python 3
-                fout.write('saga_cmd ' + command + '\n')
+            fout.write('saga_cmd ' + command + '\n')
 
         fout.write('exit')
 

diff --git a/python/plugins/processing/tests/SagaAlgorithmsTest.py b/python/plugins/processing/tests/SagaAlgorithmsTest.py
@@ -25,11 +25,22 @@
 
 __revision__ = ':%H$'
 
+import os
 import nose2
 import shutil
+import tempfile
 
 from qgis.core import (QgsProcessingParameterNumber,
-                       QgsProcessingParameterDefinition)
+                       QgsProcessingParameterDefinition,
+                       QgsVectorLayer,
+                       QgsApplication,
+                       QgsFeature,
+                       QgsGeometry,
+                       QgsPointXY,
+                       QgsProcessingContext,
+                       QgsProject,
+                       QgsProcessingFeedback,
+                       QgsProcessingFeatureSourceDefinition)
 from qgis.testing import start_app, unittest
 
 from processing.algs.saga.SagaParameters import Parameters, SagaImageOutputParam
@@ -45,6 +56,9 @@ def setUpClass(cls):
         Processing.initialize()
         cls.cleanup_paths = []
 
+        cls.temp_dir = tempfile.mkdtemp()
+        cls.cleanup_paths.append(cls.temp_dir)
+
     @classmethod
     def tearDownClass(cls):
         from processing.core.Processing import Processing
@@ -82,6 +96,48 @@ def test_param_line(self):
         self.assertEqual(param.defaultFileExtension(), 'tif')
         self.assertEqual(param.supportedOutputRasterLayerExtensions(), ['tif'])
 
+    def test_non_ascii_output(self):
+        # create a memory layer and add to project and context
+        layer = QgsVectorLayer("Point?crs=epsg:3857&field=fldtxt:string&field=fldint:integer",
+                               "testmem", "memory")
+        self.assertTrue(layer.isValid())
+        pr = layer.dataProvider()
+        f = QgsFeature()
+        f.setAttributes(["test", 123])
+        f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(100, 200)))
+        f2 = QgsFeature()
+        f2.setAttributes(["test2", 457])
+        f2.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(110, 200)))
+        self.assertTrue(pr.addFeatures([f, f2]))
+        self.assertEqual(layer.featureCount(), 2)
+        QgsProject.instance().addMapLayer(layer)
+        context = QgsProcessingContext()
+        context.setProject(QgsProject.instance())
+
+        alg = QgsApplication.processingRegistry().createAlgorithmById('saga:fixeddistancebuffer')
+        self.assertIsNotNone(alg)
+
+        temp_file = os.path.join(self.temp_dir, 'non_ascii_ñññ.shp')
+        parameters = {'SHAPES': 'testmem',
+                      'DIST_FIELD_DEFAULT': 5,
+                      'NZONES': 1,
+                      'DARC': 5,
+                      'DISSOLVE': False,
+                      'POLY_INNER': False,
+                      'BUFFER': temp_file}
+        feedback = QgsProcessingFeedback()
+
+        results, ok = alg.run(parameters, context, feedback)
+        self.assertTrue(ok)
+        self.assertTrue(os.path.exists(temp_file))
+
+        # make sure that layer has correct features
+        res = QgsVectorLayer(temp_file, 'res')
+        self.assertTrue(res.isValid())
+        self.assertEqual(res.featureCount(), 2)
+
+        QgsProject.instance().removeMapLayer(layer)
+
 
 if __name__ == '__main__':
     nose2.main()
diff --git a/python/plugins/processing/tests/testdata/polys_non_ascii_ñññ.gml b/python/plugins/processing/tests/testdata/polys_non_ascii_ñññ.gml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<ogr:FeatureCollection
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation=""
+     xmlns:ogr="http://ogr.maptools.org/"
+     xmlns:gml="http://www.opengis.net/gml">
+  <gml:boundedBy>
+    <gml:Box>
+      <gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
+      <gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
+    </gml:Box>
+  </gml:boundedBy>
+
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.0">
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:name>aaaaa</ogr:name>
+      <ogr:intval>33</ogr:intval>
+      <ogr:floatval>44.123456</ogr:floatval>
+    </ogr:polys2>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.1">
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:name>Aaaaa</ogr:name>
+      <ogr:intval>-33</ogr:intval>
+      <ogr:floatval>0</ogr:floatval>
+    </ogr:polys2>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.2">
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:name>bbaaa</ogr:name>
+      <ogr:floatval>0.123</ogr:floatval>
+    </ogr:polys2>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.3">
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:name>ASDF</ogr:name>
+      <ogr:intval>0</ogr:intval>
+    </ogr:polys2>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.4">
+      <ogr:intval>120</ogr:intval>
+      <ogr:floatval>-100291.43213</ogr:floatval>
+    </ogr:polys2>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:polys2 fid="polys.5">
+      <ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
+      <ogr:name>elim</ogr:name>
+      <ogr:intval>2</ogr:intval>
+      <ogr:floatval>3.33</ogr:floatval>
+    </ogr:polys2>
+  </gml:featureMember>
+</ogr:FeatureCollection>
diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
@@ -16,6 +16,20 @@ tests:
           geometry:
             precision: 7
 
+  - name: Centroid with non-ascii input
+    algorithm: native:centroids
+    params:
+      INPUT:
+        type: vector
+        name: polys_non_ascii_ñññ.gml
+    results:
+      OUTPUT:
+        type: vector
+        name: expected/polys_centroid.gml
+        compare:
+          geometry:
+            precision: 7
+
   - name: Aggregate all
     algorithm: qgis:aggregate
     params: