Skip to content

Commit

Permalink
Merge pull request #5592 from nyalldawson/batch
Browse files Browse the repository at this point in the history 
[processing] Fixes for batch processing
  • Loading branch information
@alexbruy
alexbruy committed Nov 10, 2017
2 parents 019a4d2 + feacb76 commit 9b85e5b
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 59 deletions.
61 changes: 61 additions & 0 deletions python/core/conversions.sip
View file
Expand Up @@ -2028,6 +2028,67 @@ bool null_from_qvariant_converter( const QVariant *varp, PyObject **objp )
}
%End

// Mapped type for QList<QgsWkbTypes::GeometryType>.
%MappedType QList<QgsWkbTypes::GeometryType>
{
%TypeHeaderCode
#include <qgswkbtypes.h>
%End

%ConvertFromTypeCode
// Create the list.
PyObject *l;

if ((l = PyList_New(sipCpp->size())) == NULL)
return NULL;

// Set the list elements.
for (int i = 0; i < sipCpp->size(); ++i)
{
QgsWkbTypes::GeometryType e = sipCpp->at(i);
PyObject *eobj;

if ((eobj = sipConvertFromEnum(e, sipType_QgsWkbTypes_GeometryType)) == NULL)
{
Py_DECREF(l);

return NULL;
}

PyList_SET_ITEM(l, i, eobj);
}

return l;
%End

%ConvertToTypeCode
// Check the type if that is all that is required.
if (sipIsErr == NULL)
{
if (!PyList_Check(sipPy))
return 0;

for (SIP_SSIZE_T i = 0; i < PyList_GET_SIZE(sipPy); ++i)
if (PyObject_TypeCheck(PyList_GET_ITEM(sipPy, i), sipTypeAsPyTypeObject(sipType_QgsWkbTypes_GeometryType)))
return 0;

return 1;
}

QList<QgsWkbTypes::GeometryType> *ql = new QList<QgsWkbTypes::GeometryType>;

for (SIP_SSIZE_T i = 0; i < PyList_GET_SIZE(sipPy); ++i)
{
long l = SIPLong_AsLong(PyList_GET_ITEM(sipPy, i));
ql->append(static_cast<QgsWkbTypes::GeometryType>(l));
}

*sipCppPtr = ql;

return sipGetState(sipTransferObj);
%End
};

%PostInitialisationCode //#spellok

// Import the Chimera helper registration functions.
Expand Down
76 changes: 38 additions & 38 deletions python/plugins/processing/gui/BatchAlgorithmDialog.py
View file
Expand Up @@ -43,6 +43,7 @@
QgsProject)

from qgis.gui import QgsMessageBar
from qgis.utils import OverrideCursor

from processing.gui.BatchPanel import BatchPanel
from processing.gui.AlgorithmDialogBase import AlgorithmDialogBase
Expand Down Expand Up @@ -103,7 +104,7 @@ def accept(self):
count_visible_outputs += 1
widget = self.mainWidget.tblParameters.cellWidget(row, col)
text = widget.getValue()
if param.checkValueIsAcceptable(text, context):
if out.checkValueIsAcceptable(text, context):
if isinstance(out, (QgsProcessingParameterRasterDestination,
QgsProcessingParameterFeatureSink)):
# load rasters and sinks on completion
Expand All @@ -119,47 +120,48 @@ def accept(self):

alg_parameters.append(parameters)

QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
self.mainWidget.setEnabled(False)
self.buttonCancel.setEnabled(True)
with OverrideCursor(Qt.WaitCursor):

# Make sure the Log tab is visible before executing the algorithm
try:
self.tabWidget.setCurrentIndex(1)
self.repaint()
except:
pass
self.mainWidget.setEnabled(False)
self.buttonCancel.setEnabled(True)

start_time = time.time()
# Make sure the Log tab is visible before executing the algorithm
try:
self.tabWidget.setCurrentIndex(1)
self.repaint()
except:
pass

algorithm_results = []
for count, parameters in enumerate(alg_parameters):
if feedback.isCanceled():
break
self.setText(self.tr('\nProcessing algorithm {0}/{1}...').format(count + 1, len(alg_parameters)))
self.setInfo(self.tr('<b>Algorithm {0} starting...</b>').format(self.alg.displayName()), escape_html=False)

feedback.pushInfo(self.tr('Input parameters:'))
feedback.pushCommandInfo(pformat(parameters))
feedback.pushInfo('')

alg_start_time = time.time()
ret, results = execute(self.alg, parameters, context, feedback)
if ret:
self.setInfo(self.tr('Algorithm {0} correctly executed...').format(self.alg.displayName()), escape_html=False)
feedback.setProgress(100)
feedback.pushInfo(
self.tr('Execution completed in {0:0.2f} seconds'.format(time.time() - alg_start_time)))
feedback.pushInfo(self.tr('Results:'))
feedback.pushCommandInfo(pformat(results))
start_time = time.time()

algorithm_results = []
for count, parameters in enumerate(alg_parameters):
if feedback.isCanceled():
break
self.setText(self.tr('\nProcessing algorithm {0}/{1}...').format(count + 1, len(alg_parameters)))
self.setInfo(self.tr('<b>Algorithm {0} starting...</b>').format(self.alg.displayName()), escape_html=False)

feedback.pushInfo(self.tr('Input parameters:'))
feedback.pushCommandInfo(pformat(parameters))
feedback.pushInfo('')
algorithm_results.append(results)
else:
break

feedback.pushInfo(self.tr('Batch execution completed in {0:0.2f} seconds'.format(time.time() - start_time)))
alg_start_time = time.time()
ret, results = execute(self.alg, parameters, context, feedback)
if ret:
self.setInfo(self.tr('Algorithm {0} correctly executed...').format(self.alg.displayName()), escape_html=False)
feedback.setProgress(100)
feedback.pushInfo(
self.tr('Execution completed in {0:0.2f} seconds'.format(time.time() - alg_start_time)))
feedback.pushInfo(self.tr('Results:'))
feedback.pushCommandInfo(pformat(results))
feedback.pushInfo('')
algorithm_results.append(results)
else:
break

feedback.pushInfo(self.tr('Batch execution completed in {0:0.2f} seconds'.format(time.time() - start_time)))

handleAlgorithmResults(self.alg, context, feedback, False)
handleAlgorithmResults(self.alg, context, feedback, False)

self.finish(algorithm_results)
self.buttonCancel.setEnabled(False)
Expand All @@ -169,8 +171,6 @@ def finish(self, algorithm_results):
self.loadHTMLResults(results, count)

self.createSummaryTable(algorithm_results)
QApplication.restoreOverrideCursor()

self.mainWidget.setEnabled(True)
QMessageBox.information(self, self.tr('Batch processing'),
self.tr('Batch processing completed'))
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/gui/BatchInputSelectionPanel.py
View file
Expand Up @@ -94,7 +94,7 @@ def showPopupMenu(self):
popupmenu.addAction(selectLayerAction)

selectFileAction = QAction(
self.tr('Select from filesystem'), self.pushButton)
self.tr('Select from file system'), self.pushButton)
selectFileAction.triggered.connect(self.showFileSelectionDialog)
popupmenu.addAction(selectFileAction)

Expand Down Expand Up @@ -150,7 +150,7 @@ def showFileSelectionDialog(self):
path = ''

ret, selected_filter = QFileDialog.getOpenFileNames(self, self.tr('Open file'), path,
self.tr('All files (*.*);;') + getFileFilter(self.param))
getFileFilter(self.param))
if ret:
files = list(ret)
settings.setValue('/Processing/LastInputPath',
Expand Down
54 changes: 36 additions & 18 deletions python/plugins/processing/gui/BatchPanel.py
View file
Expand Up @@ -31,8 +31,9 @@

from qgis.PyQt import uic
from qgis.PyQt.QtWidgets import QTableWidgetItem, QComboBox, QHeaderView, QFileDialog, QMessageBox

from qgis.PyQt.QtCore import QDir, QFileInfo
from qgis.core import (QgsApplication,
QgsSettings,
QgsProcessingParameterDefinition)
from qgis.gui import QgsMessageBar
from processing.gui.wrappers import WidgetWrapperFactory
Expand Down Expand Up @@ -87,6 +88,10 @@ def __init__(self, parent, alg):
self.tblParameters.horizontalHeader().sectionDoubleClicked.connect(
self.fillParameterValues)

self.tblParameters.horizontalHeader().resizeSections(QHeaderView.ResizeToContents)
self.tblParameters.horizontalHeader().setDefaultSectionSize(250)
self.tblParameters.horizontalHeader().setMinimumSectionSize(150)

self.initWidgets()

def layerRegistryChanged(self):
Expand Down Expand Up @@ -133,18 +138,19 @@ def initWidgets(self):
for i in range(2):
self.addRow()

self.tblParameters.horizontalHeader().setSectionResizeMode(QHeaderView.Interactive)
self.tblParameters.horizontalHeader().setDefaultSectionSize(250)
self.tblParameters.horizontalHeader().setMinimumSectionSize(150)
self.tblParameters.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
self.tblParameters.horizontalHeader().resizeSections(QHeaderView.ResizeToContents)
self.tblParameters.verticalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
self.tblParameters.horizontalHeader().setStretchLastSection(True)

def load(self):
settings = QgsSettings()
last_path = settings.value("/Processing/LastBatchPath", QDir.homePath())
filename, selected_filter = QFileDialog.getOpenFileName(self,
self.tr('Open Batch'), None,
self.tr('Open Batch'), last_path,
self.tr('JSON files (*.json)'))
if filename:
last_path = QFileInfo(filename).path()
settings.setValue('/Processing/LastBatchPath', last_path)
with open(filename) as f:
values = json.load(f)
else:
Expand All @@ -161,17 +167,19 @@ def load(self):
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if param.isDestination():
continue
if param.name() in params:
value = params[param.name()].strip('"')
value = params[param.name()].strip("'")
wrapper = self.wrappers[row][column]
wrapper.setValue(value)
column += 1

for out in self.alg.outputs:
for out in self.alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if out.name() in outputs:
value = outputs[out.name()].strip('"')
value = outputs[out.name()].strip("'")
widget = self.tblParameters.cellWidget(row, column)
widget.setValue(value)
column += 1
Expand All @@ -192,36 +200,42 @@ def save(self):
for param in alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if param.isDestination():
continue
wrapper = self.wrappers[row][col]
if not param.checkValueIsAcceptable(wrapper.value, context):
if not param.checkValueIsAcceptable(wrapper.value(), context):
self.parent.bar.pushMessage("", self.tr('Wrong or missing parameter value: {0} (row {1})').format(
param.description(), row + 1),
level=QgsMessageBar.WARNING, duration=5)
param.description(), row + 1),
level=QgsMessageBar.WARNING, duration=5)
return
algParams[param.name()] = param.getValueAsCommandLineParameter()
algParams[param.name()] = param.valueAsPythonString(wrapper.value(), context)
col += 1
for out in alg.outputs:
for out in alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
widget = self.tblParameters.cellWidget(row, col)
text = widget.getValue()
if text.strip() != '':
algOutputs[out.name] = text.strip()
algOutputs[out.name()] = text.strip()
col += 1
else:
self.parent.bar.pushMessage("", self.tr('Wrong or missing output value: {0} (row {1})').format(
out.description(), row + 1),
level=QgsMessageBar.WARNING, duration=5)
out.description(), row + 1),
level=QgsMessageBar.WARNING, duration=5)
return
toSave.append({self.PARAMETERS: algParams, self.OUTPUTS: algOutputs})

settings = QgsSettings()
last_path = settings.value("/Processing/LastBatchPath", QDir.homePath())
filename, __ = QFileDialog.getSaveFileName(self,
self.tr('Save Batch'),
None,
last_path,
self.tr('JSON files (*.json)'))
if filename:
if not filename.endswith('.json'):
filename += '.json'
last_path = QFileInfo(filename).path()
settings.setValue('/Processing/LastBatchPath', last_path)
with open(filename, 'w') as f:
json.dump(toSave, f)

Expand Down Expand Up @@ -271,6 +285,10 @@ def removeRows(self):

def fillParameterValues(self, column):
wrapper = self.wrappers[0][column]
if wrapper is None:
# e.g. double clicking on a destination header
return

for row in range(1, self.tblParameters.rowCount()):
self.wrappers[row][column].setValue(wrapper.value())

Expand Down
3 changes: 2 additions & 1 deletion python/plugins/processing/gui/Postprocessing.py
View file
Expand Up @@ -82,7 +82,8 @@ def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers.append(str(l))
i += 1

QApplication.restoreOverrideCursor()
feedback.setProgress(100)

if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
Expand Down

0 comments on commit 9b85e5b

Please sign in to comment.