[processing] Use QgsBlockingProcess instead of subprocess to run GDAL…

… algorithms This gives us much more flexibility when running the process, as we can now do things like report feedback messages directly without having to resort to potentially unsafe event loops regardless of whether the algorithm is run in the main or a background thread.
qgis · Jan 8, 2021 · 50b5e56 · 50b5e56
1 parent e79922b
commit 50b5e56
Showing 1 changed file with 48 additions and 27 deletions.
diff --git a/python/plugins/processing/algs/gdal/GdalUtils.py b/python/plugins/processing/algs/gdal/GdalUtils.py
@@ -34,6 +34,8 @@
     from osgeo import ogr
 
 from qgis.core import (Qgis,
+                       QgsBlockingProcess,
+                       QgsRunProcess,
                        QgsApplication,
                        QgsVectorFileWriter,
                        QgsProcessingFeedback,
@@ -43,9 +45,13 @@
                        QgsCredentials,
                        QgsDataSourceUri,
                        QgsProjUtils,
-                       QgsCoordinateReferenceSystem)
+                       QgsCoordinateReferenceSystem,
+                       QgsProcessingException)
 
-from qgis.PyQt.QtCore import QCoreApplication
+from qgis.PyQt.QtCore import (
+    QCoreApplication,
+    QProcess
+)
 
 from processing.core.ProcessingConfig import ProcessingConfig
 from processing.tools.system import isWindows, isMac
@@ -94,32 +100,47 @@ def runGdal(commands, feedback=None):
         feedback.pushInfo(GdalUtils.tr('GDAL command:'))
         feedback.pushCommandInfo(fused_command)
         feedback.pushInfo(GdalUtils.tr('GDAL command output:'))
-        success = False
-        retry_count = 0
-        while not success:
-            loglines = [GdalUtils.tr('GDAL execution console output')]
-            try:
-                with subprocess.Popen(
-                    fused_command,
-                    shell=True,
-                    stdout=subprocess.PIPE,
-                    stdin=subprocess.DEVNULL,
-                    stderr=subprocess.STDOUT,
-                    universal_newlines=True,
-                ) as proc:
-                    for line in proc.stdout:
-                        feedback.pushConsoleInfo(line)
-                        loglines.append(line)
-                    success = True
-            except IOError as e:
-                if retry_count < 5:
-                    retry_count += 1
-                else:
-                    raise IOError(
-                        str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format(
-                            len(loglines), u'\n'.join(loglines[-10:])))
 
-            QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
+        loglines = [GdalUtils.tr('GDAL execution console output')]
+
+        def on_stdout(ba):
+            val = ba.data().decode('UTF-8')
+            on_stdout.buffer += val
+            if on_stdout.buffer.endswith('\n') or on_stdout.buffer.endswith('\r'):
+                # flush buffer
+                feedback.pushConsoleInfo(on_stdout.buffer.rstrip())
+                loglines.append(on_stdout.buffer.rstrip())
+                on_stdout.buffer = ''
+
+        on_stdout.buffer = ''
+
+        def on_stderr(ba):
+            val = ba.data().decode('UTF-8')
+            on_stderr.buffer += val
+
+            if on_stderr.buffer.endswith('\n') or on_stderr.buffer.endswith('\r'):
+                # flush buffer
+                feedback.reportError(on_stderr.buffer.rstrip())
+                loglines.append(on_stderr.buffer.rstrip())
+                on_stderr.buffer = ''
+
+        on_stderr.buffer = ''
+
+        command, *arguments = QgsRunProcess.splitCommand(fused_command)
+        proc = QgsBlockingProcess(command, arguments)
+        proc.setStdOutHandler(on_stdout)
+        proc.setStdErrHandler(on_stderr)
+
+        res = proc.run(feedback)
+        if feedback.isCanceled() and res != 0:
+            feedback.pushInfo(GdalUtils.tr('Process was canceled and did not complete'))
+        elif not feedback.isCanceled() and proc.exitStatus() == QProcess.CrashExit:
+            raise QgsProcessingException(GdalUtils.tr('Process was unexpectedly terminated'))
+        elif res == 0:
+            feedback.pushInfo(GdalUtils.tr('Process completed successfully'))
+        else:
+            feedback.reportError(GdalUtils.tr('Process returned error code {}').format(res))
+
         return loglines
 
     @staticmethod