Improvementes to SEXTANTE batch processing interface

Better progress handling in SEXTANTE script algorithms
qgis · Nov 19, 2012 · 8cb001d · 8cb001d
1 parent 6173fa3
commit 8cb001d
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 52 deletions.
diff --git a/python/plugins/sextante/algs/SextanteAlgorithmProvider.py b/python/plugins/sextante/algs/SextanteAlgorithmProvider.py
@@ -16,14 +16,13 @@
 *                                                                         *
 ***************************************************************************
 """
-from sextante.algs.EquivalentNumField import EquivalentNumField
-
 __author__ = 'Victor Olaya'
 __date__ = 'August 2012'
 __copyright__ = '(C) 2012, Victor Olaya'
 # This will get replaced with a git SHA1 when you do a git archive
 __revision__ = '$Format:%H$'
 
+from sextante.algs.EquivalentNumField import EquivalentNumField
 from sextante.core.AlgorithmProvider import AlgorithmProvider
 from sextante.algs.AddTableField import AddTableField
 from PyQt4 import QtGui

diff --git a/python/plugins/sextante/core/Sextante.py b/python/plugins/sextante/core/Sextante.py
@@ -258,7 +258,7 @@ def runAlgorithm(algOrName, onFinish, *args):
             return
         if len(args) != alg.getVisibleParametersCount() + alg.getVisibleOutputsCount():
             print ("Error: Wrong number of parameters")
-            Sextante.alghelp(algOrName)
+            alghelp(algOrName)
             return
 
         alg = alg.getCopy()#copy.deepcopy(alg)
@@ -294,7 +294,14 @@ def runAlgorithm(algOrName, onFinish, *args):
 
         SextanteLog.addToLog(SextanteLog.LOG_ALGORITHM, alg.getAsCommand())
 
-        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+        # don't set the wait cursor twice, because then when you restore it
+        # it will still be a wait cursor
+        cursor = QApplication.overrideCursor()
+        if cursor == None or cursor == 0:
+            QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) 
+        elif cursor.shape() != Qt.WaitCursor:
+            QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
         if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
             algEx = AlgorithmExecutor(alg)
             progress = QProgressDialog()

diff --git a/python/plugins/sextante/core/SextanteUtils.py b/python/plugins/sextante/core/SextanteUtils.py
@@ -26,9 +26,8 @@
 import os
 import time
 import sys
-
+import uuid
 from PyQt4.QtCore import *
-
 from qgis.core import *
 
 class SextanteUtils:
@@ -64,9 +63,9 @@ def setTempOutput(out, alg):
         ext = out.getDefaultFileExtension(alg)
         validChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
         safeCmdName = ''.join(c for c in alg.commandLineName() if c in validChars)
-        filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + str(SextanteUtils.NUM_EXPORTED) + "." + ext
-        out.value = filename
-        SextanteUtils.NUM_EXPORTED += 1
+        uniqueSufix = str(uuid.uuid4()).replace("-","");
+        filename = SextanteUtils.tempFolder() + os.sep + safeCmdName + uniqueSufix + "." + ext
+        out.value = filename                
 
     @staticmethod
     def getTempFilename(ext):

diff --git a/python/plugins/sextante/grass/GrassUtils.py b/python/plugins/sextante/grass/GrassUtils.py
@@ -325,7 +325,7 @@ def startGrassSession():
     # End session by removing the temporary GRASS mapset and all the layers.
     @staticmethod    
     def endGrassSession():
-        #shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
+        shutil.rmtree(GrassUtils.grassMapsetFolder(), True)
         GrassUtils.sessionRunning = False
         GrassUtils.sessionLayers = {}
 

diff --git a/python/plugins/sextante/grass/description/r.mapcalculator.txt b/python/plugins/sextante/grass/description/r.mapcalculator.txt
@@ -2,10 +2,10 @@ r.mapcalculator
 r.mapcalculator - Calculate new raster map from a r.mapcalc expression.
 Raster (r.*)
 ParameterRaster|amap|amap|False
-ParameterRaster|bmap|bmap|False
-ParameterRaster|cmap|cmap|False
-ParameterRaster|dmap|dmap|False
-ParameterRaster|emap|emap|False
-ParameterRaster|fmap|fmap|False
-ParameterString|formula|Formula (e.g. A-B or A*C+B|
+ParameterRaster|bmap|bmap|True
+ParameterRaster|cmap|cmap|True
+ParameterRaster|dmap|dmap|True
+ParameterRaster|emap|emap|True
+ParameterRaster|fmap|fmap|True
+ParameterString|formula|Formula (e.g. A-B or A*C+B)|
 OutputRaster|outfile|Name for output raster map
diff --git a/python/plugins/sextante/grass/description/r.series.txt b/python/plugins/sextante/grass/description/r.series.txt
@@ -4,7 +4,7 @@ Raster (r.*)
 ParameterMultipleInput|input|Name of input raster map(s)|3.0|False
 ParameterBoolean|-n|Propagate NULLs|True
 ParameterSelection|method|Aggregate operation|average;count;median;mode;minimum;min_raster;maximum;max_raster;stddev;range;sum;threshold;variance;diversity;slope;offset;detcoeff;quart1;quart3;perc90;quantile;skewness;kurtosis
-ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
-ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
-ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
+*ParameterNumber|quantile|Quantile to calculate for method=quantile|0.0|1.0|0.0
+*ParameterNumber|threshold|Threshold to calculate for method=threshold|None|None|0.0
+*ParameterString|range|Ignore values outside this range (lo,hi)|-10000000000,10000000000
 OutputRaster|output|Name for output raster map
diff --git a/python/plugins/sextante/grass/description/r.univar.txt b/python/plugins/sextante/grass/description/r.univar.txt
@@ -2,6 +2,9 @@ r.univar
 r.univar - Calculates univariate statistics from the non-null cells of a raster map.
 Raster (r.*)
 ParameterMultipleInput|map|Name of input raster map(s)|3.0|False
-ParameterRaster|zones|Raster map used for zoning, must be of type CELL|False
-ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
-ParameterBoolean|-e|Calculate extended statistics|False
+ParameterRaster|zones|Raster map used for zoning, must be of type CELL|True
+*ParameterBoolean|-e|Calculate extended statistics|False
+*ParameterBoolean|-g|Print the stats in shell script style|False
+*ParameterBoolean|-t|Table output format instead of standard output format|False
+*ParameterNumber|percentile|Percentile to calculate (requires extended statistics flag)|0.0|100.0|90
+OutputFile|output|Name for output text file
diff --git a/python/plugins/sextante/gui/BatchProcessingDialog.py b/python/plugins/sextante/gui/BatchProcessingDialog.py
@@ -17,6 +17,9 @@
 ***************************************************************************
 """
 from sextante.gui.CrsSelectionPanel import CrsSelectionPanel
+from sextante.outputs.OutputNumber import OutputNumber
+from sextante.outputs.OutputString import OutputString
+from sextante.core.SextanteUtils import SextanteUtils
 
 __author__ = 'Victor Olaya'
 __date__ = 'August 2012'
@@ -79,12 +82,14 @@ def __init__(self, alg):
         self.deleteRowButton.setText("Delete row")
         self.buttonBox.addButton(self.addRowButton, QtGui.QDialogButtonBox.ActionRole)
         self.buttonBox.addButton(self.deleteRowButton, QtGui.QDialogButtonBox.ActionRole)
-        self.table.setColumnCount(len(self.alg.parameters) + len(self.alg.outputs))
+
+        self.table.setColumnCount(self.alg.getVisibleParametersCount() + self.alg.getVisibleOutputsCount())
         self.setTableContent()
         self.table.horizontalHeader().setStretchLastSection(True)
         self.table.verticalHeader().setVisible(False)
         self.table.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
-        self.progress = QtGui.QProgressBar()
+        #self.progress = QtGui.QProgressBar()
+        #self.progress.setMaximum(100)
         self.addRowButton.clicked.connect(self.addRow)
         self.deleteRowButton.clicked.connect(self.deleteRow)
 
@@ -110,13 +115,17 @@ def accept(self):
             alg = self.alg.getCopy()#copy.deepcopy(self.alg)
             col = 0
             for param in alg.parameters:
+                if param.hidden:
+                    continue
                 widget = self.table.cellWidget(row, col)
                 if not self.setParameterValueFromWidget(param, widget, alg):
                     QMessageBox.critical(self.dialog, "Unable to execute batch process", "Wrong or missing parameter values")
                     self.algs = None
                     return
                 col+=1
             for out in alg.outputs:
+                if out.hidden:
+                    continue
                 widget = self.table.cellWidget(row, col)
                 text = widget.getValue()
                 if text.strip() != "":
@@ -129,18 +138,19 @@ def accept(self):
             self.algs.append(alg)
 
         QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
-        self.progress.setMaximum(len(self.algs))
+        #self.progress.setMaximum(len(self.algs))
         self.table.setEnabled(False)
         if SextanteConfig.getSetting(SextanteConfig.USE_THREADS):
-            self.progress.setValue(0)
+            #self.progress.setValue(0)
             self.nextAlg(0)
         else:
             i=1
             self.progress.setMaximum(len(self.algs))
             for alg in self.algs:
-                if UnthreadedAlgorithmExecutor.runalg(alg, SilentProgress()):
-                    self.progress.setValue(i)
-                    self.loadHTMLResults(alg, i)
+                self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
+                if UnthreadedAlgorithmExecutor.runalg(alg, self):#SilentProgress()):
+                    #self.progress.setValue(i)                    
+                    #self.loadHTMLResults(alg, i)
                     i+=1
                 else:
                     QApplication.restoreOverrideCursor()
@@ -164,31 +174,53 @@ def cancel(self):
     @pyqtSlot()
     def finish(self, i):
         i += 1
-        self.progress.setValue(i)
+        self.progress.setValue(i)        
         if len(self.algs) == i:
             self.finishAll()
             self.algEx = None
         else:
             self.nextAlg(i)
-
+ 
     @pyqtSlot()
     def error(self, msg):
         QApplication.restoreOverrideCursor()
         QMessageBox.critical(self, "Error", msg)
         SextanteLog.addToLog(SextanteLog.LOG_ERROR, msg)
         self.close()
-
+
+
     def nextAlg(self, i):
+        self.setBaseText("Processing algorithm " + str(i) + "/" + str(len(self.algs)) + "...")
         self.algEx = AlgorithmExecutor(self.algs[i]);
+        self.algEx.percentageChanged.connect(self.setPercentage)
+        self.algEx.textChanged.connect(self.setText)
         self.algEx.error.connect(self.error)
         self.algEx.finished.connect(lambda: self.finish(i))
         self.algEx.start()
 
+    def createSummaryTable(self): 
+        createTable = False           
+        for out in self.algs[0].outputs:
+            if isinstance(out, (OutputNumber,OutputString)):
+                createTable = True
+                break
+        if not createTable:
+            return    
+        outputFile = SextanteUtils.getTempFilename("html")
+        f = open(outputFile, "w")                                
+        for alg in self.algs:
+            for out in alg.outputs:            
+                if isinstance(out, (OutputNumber,OutputString)):
+                    f.write("<p>" + out.description + ": " + str(out.value) + "</p>\n")
+        f.close()
+        SextanteResults.addResult(self.algs[0].name + "[summary]", outputFile)                            
+
     def finishAll(self):
         i = 0
         for alg in self.algs:
             self.loadHTMLResults(alg, i)
             i = i + 1
+        self.createSummaryTable()
         QApplication.restoreOverrideCursor()
         self.table.setEnabled(True)
         QMessageBox.information(self, "Batch processing", "Batch processing successfully completed!")
@@ -266,4 +298,10 @@ def showAdvancedParametersClicked(self):
         for param in self.alg.parameters:
             if param.isAdvanced:
                 self.table.setColumnHidden(i, not self.showAdvanced)
-            i+=1
+            i+=1
+
+    def setText(self, text):
+        self.progressLabel.setText(self.baseText + "   --- [" + text + "]")
+
+    def setBaseText(self, text):
+        self.baseText = text
diff --git a/python/plugins/sextante/gui/FileSelectionPanel.py b/python/plugins/sextante/gui/FileSelectionPanel.py
@@ -56,5 +56,5 @@ def showSelectionDialog(self):
     def getValue(self):
         s = str(self.text.text())
         if SextanteUtils.isWindows():
-            s = s.replace("/", "\\")
+            s = s.replace("\\", "/")
         return s
diff --git a/python/plugins/sextante/script/ScriptAlgorithm.py b/python/plugins/sextante/script/ScriptAlgorithm.py
@@ -45,6 +45,8 @@
 from sextante.parameters.ParameterExtent import ParameterExtent
 from sextante.parameters.ParameterFile import ParameterFile
 from sextante.outputs.OutputFile import OutputFile
+from sextante.parameters.ParameterFactory import ParameterFactory
+from sextante.outputs.OutputFactory import OutputFactory
 import sys
 from sextante.gui.Help2Html import Help2Html
 
@@ -87,7 +89,6 @@ def defineCharacteristicsFromFile(self):
             line = lines.readline()
         lines.close()
 
-
     def defineCharacteristicsFromScript(self):
         lines = self.script.split("\n")
         self.silentOutputs = []
@@ -107,11 +108,19 @@ def processParameterLine(self,line):
         param = None
         out = None
         line = line.replace("#", "");
+        # If the line is in the format of the text description files for normal algorithms,
+        # then process it using parameter and output factories
+        if '|' in line:
+            self.processDescriptionParameterLine(line)
+            return
         tokens = line.split("=");
         desc = self.createDescriptiveName(tokens[0])
         if tokens[1].lower().strip() == "group":
             self.group = tokens[0]
             return
+        if tokens[1].lower().strip() == "name":
+            self.name = tokens[0]
+            return
         if tokens[1].lower().strip() == "raster":
             param = ParameterRaster(tokens[0], desc, False)
         elif tokens[1].lower().strip() == "vector":
@@ -174,13 +183,27 @@ def processParameterLine(self,line):
             self.addOutput(out)
         else:
             raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")
+
+    def processDescriptionParameterLine(self, line):
+        try:
+            if line.startswith("Parameter"):
+                self.addParameter(ParameterFactory.getFromString(line))
+            elif line.startswith("*Parameter"):
+                param = ParameterFactory.getFromString(line[1:])
+                param.isAdvanced = True
+                self.addParameter(param)
+            else:
+                self.addOutput(OutputFactory.getFromString(line))
+        except Exception:
+            raise WrongScriptException("Could not load script:" + self.descriptionFile + ".\n Problem with line \"" + line + "\"")
 
     def processAlgorithm(self, progress):
 
         script = "import sextante\n"
 
         ns = {}
-
+        ns['progress'] = progress
+
         for param in self.parameters:
             #script += param.name + "=" + param.getValueAsCommandLineParameter() + "\n"
             ns[param.name] = param.value
@@ -189,11 +212,8 @@ def processAlgorithm(self, progress):
             ns[out.name] = out.value
             #script += out.name + "=" + out.getValueAsCommandLineParameter() + "\n"
 
-        script+=self.script
-        redirection = Redirection(progress)
-        sys.stdout = redirection
-        exec(script) in ns
-        sys.stdout = sys.__stdout__
+        script+=self.script        
+        exec(script) in ns        
         for out in self.outputs:
             out.setValue(ns[out.name])
 
@@ -205,14 +225,3 @@ def helpFile(self):
         else:
             return None
 
-class Redirection():
-    def __init__(self, progress):
-        self.progress = progress
-
-    def write(self, string):
-        try:
-            n = int(string)
-            self.progress.setPercentage(n)
-        except:
-            self.progress.setText(string)
-            pass