Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Steps towards executing models
  • Loading branch information
nyalldawson committed Jun 21, 2017
1 parent 1df9f6b commit d1ed7d1
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 85 deletions.
2 changes: 1 addition & 1 deletion python/plugins/processing/gui/NumberInputPanel.py
Expand Up @@ -125,7 +125,7 @@ def showExpressionsBuilder(self):
def getValue(self):
value = self.leText.text()
values = []
for param in self.modelParametersDialog.model.parameters:
for param in self.modelParametersDialog.model.parameterDefinitions():
if isinstance(param, QgsProcessingParameterNumber):
if "@" + param.name() in value:
values.append(ValueFromInput(param.name()))
Expand Down
55 changes: 0 additions & 55 deletions python/plugins/processing/modeler/ModelerAlgorithm.py
Expand Up @@ -197,17 +197,6 @@ def prepareAlgorithm(self, alg):
self.tr("Parameter {0} in algorithm {1} in the model is run with default value! Edit the model to make sure that this is correct.").format(param.name(), alg.displayName()),
QgsMessageBar.WARNING, 4)
value = param.defaultValue()
# We allow unexistent filepaths, since that allows
# algorithms to skip some conversion routines

# TODO
#if not param.checkValueIsAcceptable(value) and not isinstance(param,
# ParameterDataObject):
# raise GeoAlgorithmExecutionException(
# self.tr('Wrong value {0} for {1} {2}', 'ModelerAlgorithm').format(
# value, param.__class__.__name__, param.name()
# )
# )

# note to self - these are parameters, not outputs
for out in algInstance.outputDefinitions():
Expand Down Expand Up @@ -240,50 +229,6 @@ def resolveValue(self, value, param):
v = value
return param.evaluateForModeler(v, self)

def processAlgorithm(self, parameters, context, feedback):
executed = []
toExecute = [alg for alg in list(self.childAlgorithms().values()) if alg.isActive()]
while len(executed) < len(toExecute):
for alg in toExecute:
if alg.childId() not in executed:
canExecute = True
required = self.dependsOnChildAlgorithms(alg.childId())
for requiredAlg in required:
if requiredAlg not in executed:
canExecute = False
break
if canExecute:
try:
feedback.pushDebugInfo(
self.tr('Prepare algorithm: {0}', 'ModelerAlgorithm').format(alg.childId()))
self.prepareAlgorithm(alg)
feedback.setProgressText(
self.tr('Running {0} [{1}/{2}]', 'ModelerAlgorithm').format(alg.description, len(executed) + 1, len(toExecute)))
feedback.pushDebugInfo('Parameters: ' + ', '.join([str(p).strip() +
'=' + str(p.value) for p in alg.algorithm.parameters]))
t0 = time.time()
alg.algorithm().execute(parameters, context, feedback)
dt = time.time() - t0

# copy algorithm output value(s) back to model in case the algorithm modified those
for out in alg.algorithm().outputs:
if not out.flags() & QgsProcessingParameterDefinition.FlagHidden:
if out.name() in alg.modelOutputs():
modelOut = self.getOutputFromName(self.getSafeNameForOutput(alg.childId(), out.name()))
if modelOut:
modelOut.value = out.value

executed.append(alg.childId())
feedback.pushDebugInfo(
self.tr('OK. Execution took %{0:.3f} ms ({1} outputs).', 'ModelerAlgorithm').format(dt, len(alg.algorithm.modelOutputs())))
except GeoAlgorithmExecutionException as e:
feedback.pushDebugInfo(self.tr('Failed', 'ModelerAlgorithm'))
raise GeoAlgorithmExecutionException(
self.tr('Error executing algorithm {0}\n{1}', 'ModelerAlgorithm').format(alg.description, e.msg))

feedback.pushDebugInfo(
self.tr('Model processed ok. Executed {0} algorithms total', 'ModelerAlgorithm').format(len(executed)))

def asPythonCommand(self, parameters, context):
if self.descriptionFile:
return QgsProcessingAlgorithm.asPythonCommand(self, parameters, context)
Expand Down
121 changes: 92 additions & 29 deletions src/core/processing/qgsprocessingmodelalgorithm.cpp
Expand Up @@ -284,20 +284,81 @@ QString QgsProcessingModelAlgorithm::svgIconPath() const
return QgsApplication::iconPath( QStringLiteral( "processingModel.svg" ) );
}

QVariantMap QgsProcessingModelAlgorithm::parametersForChildAlgorithm( const ChildAlgorithm &child, const QVariantMap &modelParameters, const QMap< QString, QVariantMap > &results ) const
{
QVariantMap childParams;
Q_FOREACH ( const QgsProcessingParameterDefinition *def, child.algorithm()->parameterDefinitions() )
{
if ( def->flags() & QgsProcessingParameterDefinition::FlagHidden )
continue;

if ( !def->isDestination() )
{
if ( !child.parameterSources().contains( def->name() ) )
continue; // use default value

ChildParameterSource paramSource = child.parameterSources().value( def->name() );
switch ( paramSource.source() )
{
case ChildParameterSource::StaticValue:
childParams.insert( def->name(), paramSource.staticValue() );
break;

case ChildParameterSource::ModelParameter:
childParams.insert( def->name(), modelParameters.value( paramSource.parameterName() ) );
break;

case ChildParameterSource::ChildOutput:
{
QVariantMap linkedChildResults = results.value( paramSource.outputChildId() );
childParams.insert( def->name(), linkedChildResults.value( paramSource.outputName() ) );
break;
}
}
}
else
{
// is destination linked to one of the final outputs from this model?
if ( child.modelOutputs().contains( def->name() ) )
{
QString outputName = child.modelOutputs().value( def->name() ).outputName();
QString paramName = child.childId() + ':' + outputName;
if ( modelParameters.contains( paramName ) )
childParams.insert( def->name(), modelParameters.value( paramName ) );
}
else
{
// output is temporary
// TODO - skip if this output is optional and not used elsewhere in the model
childParams.insert( def->name(), "memory:" );


}
}
}
return childParams;
}

QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const
{
QSet< QString > toExecute;
QMap< QString, ChildAlgorithm >::const_iterator childIt = mChildAlgorithms.constBegin();
for ( ; childIt != mChildAlgorithms.constEnd(); ++childIt )
{
if ( childIt->isActive() )
if ( childIt->isActive() && childIt->algorithm() )
toExecute.insert( childIt->childId() );
}

QMap< QString, QVariantMap > resultsMap;
QTime totalTime;
totalTime.start();

QMap< QString, QVariantMap > childResults;
QVariantMap finalResults;
QSet< QString > executed;
while ( executed.count() < toExecute.count() )
bool executedAlg = true;
while ( executedAlg && executed.count() < toExecute.count() )
{
executedAlg = false;
Q_FOREACH ( const QString &childId, toExecute )
{
if ( executed.contains( childId ) )
Expand All @@ -316,47 +377,47 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
if ( !canExecute )
continue;

executedAlg = true;
feedback->pushDebugInfo( QObject::tr( "Prepare algorithm: %1" ).arg( childId ) );

const ChildAlgorithm &child = mChildAlgorithms[ childId ];

// self.prepareAlgorithm( alg )

QVariantMap childParams = parametersForChildAlgorithm( child, parameters, childResults );
feedback->setProgressText( QObject::tr( "Running %1 [%2/%3]" ).arg( child.description() ).arg( executed.count() + 1 ).arg( toExecute.count() ) );
//feedback->pushDebugInfo( "Parameters: " + ', '.join( [str( p ).strip() +
// '=' + str( p.value ) for p in alg.algorithm.parameters] ) )
//t0 = time.time()
QVariantMap results = child.algorithm()->run( parameters, context, feedback );
resultsMap.insert( childId, results );

//dt = time.time() - t0
#if 0
QTime childTime;
childTime.start();

# copy algorithm output value(s) back to model in case the algorithm modified those
for out in alg.algorithm().outputs :
if not out.flags() & QgsProcessingParameterDefinition.FlagHidden:
if out.name() in alg.modelOutputs():
modelOut = self.getOutputFromName( self.getSafeNameForOutput( alg.childId(), out.name() ) )
if modelOut:
modelOut.value = out.value
#endif
QVariantMap results = child.algorithm()->run( childParams, context, feedback );
childResults.insert( childId, results );

executed.insert( childId );
//feedback->pushDebugInfo( QObject::tr( "OK. Execution took %1 ms (%2 outputs)." ).arg( dt, len( alg.algorithm.modelOutputs() ) ) )
// look through child alg's outputs to determine whether any of these should be copied
// to the final model outputs
QMap<QString, QgsProcessingModelAlgorithm::ModelOutput> outputs = child.modelOutputs();
QMap<QString, QgsProcessingModelAlgorithm::ModelOutput>::const_iterator outputIt = outputs.constBegin();
for ( ; outputIt != outputs.constEnd(); ++outputIt )
{
finalResults.insert( childId + ':' + outputIt->outputName(), results.value( outputIt->outputName() ) );
}

executed.insert( childId );
feedback->pushDebugInfo( QObject::tr( "OK. Execution took %1 s (%2 outputs)." ).arg( childTime.elapsed() / 1000.0 ).arg( results.count() ) );
#if 0
except GeoAlgorithmExecutionException as e:
feedback.pushDebugInfo( self.tr( 'Failed', 'ModelerAlgorithm' ) )
raise GeoAlgorithmExecutionException(
self.tr( 'Error executing algorithm {0}\n{1}', 'ModelerAlgorithm' ).format( alg.description, e.msg ) )
except GeoAlgorithmExecutionException as e:
feedback.pushDebugInfo( self.tr( 'Failed', 'ModelerAlgorithm' ) )
raise GeoAlgorithmExecutionException(
self.tr( 'Error executing algorithm {0}\n{1}', 'ModelerAlgorithm' ).format( alg.description, e.msg ) )
#endif
}
}
feedback->pushDebugInfo( QObject::tr( "Model processed ok. Executed %1 algorithms total" ).arg( executed.count() ) );

return QVariantMap();
}
feedback->pushDebugInfo( QObject::tr( "Model processed ok. Executed %1 algorithms total in %2 s." ).arg( executed.count() ).arg( totalTime.elapsed() / 1000.0 ) );

return finalResults;
}

void QgsProcessingModelAlgorithm::setName( const QString &name )
void QgsProcessingModelAlgorithm::setName( const QString &name )
{
mModelName = name;
}
Expand Down Expand Up @@ -434,7 +495,9 @@ void QgsProcessingModelAlgorithm::updateDestinationParameters()
{
QgsProcessingOutputDefinition *output = destParam->toOutputDefinition();
if ( output )
{
addOutput( output );
}
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/core/processing/qgsprocessingmodelalgorithm.h
Expand Up @@ -763,6 +763,8 @@ class CORE_EXPORT QgsProcessingModelAlgorithm : public QgsProcessingAlgorithm
void dependsOnChildAlgorithmsRecursive( const QString &childId, QSet<QString> &depends ) const;
void dependentChildAlgorithmsRecursive( const QString &childId, QSet<QString> &depends ) const;

QVariantMap parametersForChildAlgorithm( const ChildAlgorithm &child, const QVariantMap &modelParameters, const QMap<QString, QVariantMap> &results ) const;

/**
* Saves this model to a QVariantMap, wrapped in a QVariant.
* You can use QgsXmlUtils::writeVariant to save it to an XML document.
Expand Down

0 comments on commit d1ed7d1

Please sign in to comment.