Skip to content

Commit

Permalink
[feature] Allow input parameter values for qgis_process to be
Browse files Browse the repository at this point in the history 
specified as a JSON object passed via stdin to qgis_process

This provides a mechanism to support complex input parameters
for algorithms, and a way for qgis_process to gain support
for parameter types which are themselves specified as a dictionary
type object.

To indicate that parameters will be specified via stdin then
the qgis_process command must follow the format

    qgis_process run algid -

(with a trailing - in place of the usual arguments list).

The JSON object must contain an "inputs" key, which is a map
of the input parameter values.

E.g.

    echo "{"inputs": {\"INPUT\": \"my_shape.shp\", DISTANCE: 5}}" | qgis_process run native:buffer -

Specifying input parameters via stdin implies automatically
the --json output format for results.

One big motivation behind this enhancement is to provide a way for
the qgisprocess R libraries to support parameter types such as
aggregates.

Refs r-spatial/qgisprocess#56
Refs r-spatial/qgisprocess#44

Sponsored by the Research Institute for Nature and Forest, Flemish Govt
  • Loading branch information
@nyalldawson
nyalldawson committed Dec 20, 2021
1 parent 2406286 commit 5fd9b20
Show file tree
Hide file tree
Showing 3 changed files with 213 additions and 60 deletions.
1 change: 1 addition & 0 deletions python/core/auto_generated/qgsjsonutils.sip.in
View file
Expand Up @@ -345,6 +345,7 @@ Parse a simple array (depth=1)




};

/************************************************************************
Expand Down
152 changes: 92 additions & 60 deletions src/process/qgsprocess.cpp
View file
Expand Up @@ -342,42 +342,112 @@ int QgsProcessingExec::run( const QStringList &constArgs )
QgsUnitTypes::AreaUnit areaUnit = QgsUnitTypes::AreaUnknownUnit;
QString projectPath;
QVariantMap params;
int i = 3;
for ( ; i < args.count(); i++ )

if ( args.size() == 4 && args.at( 3 ) == '-' )
{
QString arg = args.at( i );
// read arguments as JSON value from stdin
std::string stdin;
for ( std::string line; std::getline( std::cin, line ); )
{
stdin.append( line + '\n' );
}

if ( arg == QLatin1String( "--" ) )
QString error;
const QVariantMap json = QgsJsonUtils::parseJson( stdin, error ).toMap();
if ( !error.isEmpty() )
{
break;
std::cerr << QStringLiteral( "Could not parse JSON parameters: %1" ).arg( error ).toLocal8Bit().constData() << std::endl;
return 1;
}
if ( !json.contains( QStringLiteral( "inputs" ) ) )
{
std::cerr << QStringLiteral( "JSON parameters object must contain an \"inputs\" key." ).toLocal8Bit().constData() << std::endl;
return 1;
}

if ( arg.startsWith( QLatin1String( "--" ) ) )
arg = arg.mid( 2 );
params = json.value( QStringLiteral( "inputs" ) ).toMap();

const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
// JSON format for input parameters implies JSON output format
useJson = true;
}
else
{
int i = 3;
for ( ; i < args.count(); i++ )
{
const QString name = parts.at( 0 );
QString arg = args.at( i );

if ( name.compare( QLatin1String( "ellipsoid" ), Qt::CaseInsensitive ) == 0 )
if ( arg == QLatin1String( "--" ) )
{
ellipsoid = parts.mid( 1 ).join( '=' );
break;
}
else if ( name.compare( QLatin1String( "distance_units" ), Qt::CaseInsensitive ) == 0 )
{
distanceUnit = QgsUnitTypes::decodeDistanceUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "area_units" ), Qt::CaseInsensitive ) == 0 )

if ( arg.startsWith( QLatin1String( "--" ) ) )
arg = arg.mid( 2 );

const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
areaUnit = QgsUnitTypes::decodeAreaUnit( parts.mid( 1 ).join( '=' ) );
const QString name = parts.at( 0 );

if ( name.compare( QLatin1String( "ellipsoid" ), Qt::CaseInsensitive ) == 0 )
{
ellipsoid = parts.mid( 1 ).join( '=' );
}
else if ( name.compare( QLatin1String( "distance_units" ), Qt::CaseInsensitive ) == 0 )
{
distanceUnit = QgsUnitTypes::decodeDistanceUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "area_units" ), Qt::CaseInsensitive ) == 0 )
{
areaUnit = QgsUnitTypes::decodeAreaUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "project_path" ), Qt::CaseInsensitive ) == 0 )
{
projectPath = parts.mid( 1 ).join( '=' );
}
else
{
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
// parameter specified multiple times, store all of them in a list...
if ( params.value( name ).type() == QVariant::StringList )
{
// append to existing list
QStringList listValue = params.value( name ).toStringList();
listValue << value;
params.insert( name, listValue );
}
else
{
// upgrade previous value to list
QStringList listValue = QStringList() << params.value( name ).toString()
<< value;
params.insert( name, listValue );
}
}
else
{
params.insert( name, value );
}
}
}
else if ( name.compare( QLatin1String( "project_path" ), Qt::CaseInsensitive ) == 0 )
else
{
projectPath = parts.mid( 1 ).join( '=' );
std::cerr << QStringLiteral( "Invalid parameter value %1. Parameter values must be entered after \"--\" e.g.\n Example:\n qgis_process run algorithm_name -- PARAM1=VALUE PARAM2=42\"\n" ).arg( arg ).toLocal8Bit().constData();
return 1;
}
else
}

// After '--' we only have params
for ( ; i < args.count(); i++ )
{
const QString arg = args.at( i );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
const QString name = parts.first();
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
Expand All @@ -403,45 +473,6 @@ int QgsProcessingExec::run( const QStringList &constArgs )
}
}
}
else
{
std::cerr << QStringLiteral( "Invalid parameter value %1. Parameter values must be entered after \"--\" e.g.\n Example:\n qgis_process run algorithm_name -- PARAM1=VALUE PARAM2=42\"\n" ).arg( arg ).toLocal8Bit().constData();
return 1;
}
}

// After '--' we only have params
for ( ; i < args.count(); i++ )
{
const QString arg = args.at( i );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
const QString name = parts.first();
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
// parameter specified multiple times, store all of them in a list...
if ( params.value( name ).type() == QVariant::StringList )
{
// append to existing list
QStringList listValue = params.value( name ).toStringList();
listValue << value;
params.insert( name, listValue );
}
else
{
// upgrade previous value to list
QStringList listValue = QStringList() << params.value( name ).toString()
<< value;
params.insert( name, listValue );
}
}
else
{
params.insert( name, value );
}
}
}

return execute( algId, params, ellipsoid, distanceUnit, areaUnit, logLevel, useJson, projectPath );
Expand Down Expand Up @@ -473,6 +504,7 @@ void QgsProcessingExec::showUsage( const QString &appName )
<< "\tlist\t\tlist all available processing algorithms\n"
<< "\thelp\t\tshow help for an algorithm. The algorithm id or a path to a model file must be specified.\n"
<< "\trun\t\truns an algorithm. The algorithm id or a path to a model file and parameter values must be specified. Parameter values are specified after -- with PARAMETER=VALUE syntax. Ordered list values for a parameter can be created by specifying the parameter multiple times, e.g. --LAYERS=layer1.shp --LAYERS=layer2.shp\n"
<< "\t\t\tAlternatively, a '-' character in place of the parameters argument indicates that the parameters should be read from STDIN as a JSON object. The JSON should be structured as a map containing at least the \"inputs\" key specifying a map of input parameter values. This implies the --json option for output as a JSON object.\n"
<< "\t\t\tIf required, the ellipsoid to use for distance and area calculations can be specified via the \"--ELLIPSOID=name\" argument.\n"
<< "\t\t\tIf required, an existing QGIS project to use during the algorithm execution can be specified via the \"--PROJECT_PATH=path\" argument.\n";

Expand Down
120 changes: 120 additions & 0 deletions tests/src/python/test_qgsprocessexecutable.py
View file
Expand Up @@ -56,6 +56,19 @@ def run_process(self, arguments):

return rc, output.decode(), err.decode()

def run_process_stdin(self, arguments, stdin_string: str):
call = [QGIS_PROCESS_BIN] + arguments
print(' '.join(call))

myenv = os.environ.copy()
myenv["QGIS_DEBUG"] = '0'

p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, env=myenv)
output, err = p.communicate(input=stdin_string.encode())
rc = p.returncode

return rc, output.decode(), err.decode()

def testNoArgs(self):
rc, output, err = self.run_process([])
self.assertIn('Available commands', output)
Expand Down Expand Up @@ -187,6 +200,60 @@ def testAlgorithmRun(self):
self.assertTrue(os.path.exists(output_file))
self.assertEqual(rc, 0)

def testAlgorithmRunStdIn(self):
output_file = self.TMP_DIR + '/polygon_centroid_json.shp'

params = {
'inputs': {
'INPUT': TEST_DATA_DIR + '/polys.shp',
'OUTPUT': output_file
}
}

rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)

res = json.loads(output)

self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)

self.assertEqual(res['algorithm_details']['name'], 'Centroids')
self.assertEqual(res['inputs']['INPUT'], TEST_DATA_DIR + '/polys.shp')
self.assertEqual(res['inputs']['OUTPUT'], output_file)
self.assertEqual(res['results']['OUTPUT'], output_file)

self.assertTrue(os.path.exists(output_file))
self.assertEqual(rc, 0)

def testAlgorithmRunStdInMissingInputKey(self):
output_file = self.TMP_DIR + '/polygon_centroid_json.shp'

params = {
'INPUT': TEST_DATA_DIR + '/polys.shp',
'OUTPUT': output_file
}

rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], json.dumps(params))
self.assertEqual(rc, 1)
self.assertIn('JSON parameters object must contain an "inputs" key.', err)

def testAlgorithmRunStdInNoInput(self):
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], '')
self.assertEqual(rc, 1)
self.assertIn('Could not parse JSON parameters', err)

def testAlgorithmRunStdInBadInput(self):
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], '{"not valid json"}')
self.assertEqual(rc, 1)
self.assertIn('Could not parse JSON parameters', err)

def testAlgorithmRunJson(self):
output_file = self.TMP_DIR + '/polygon_centroid2.shp'
rc, output, err = self.run_process(['run', '--json', 'native:centroids', '--', 'INPUT={}'.format(TEST_DATA_DIR + '/polys.shp'), 'OUTPUT={}'.format(output_file)])
Expand Down Expand Up @@ -254,6 +321,32 @@ def testModelRun(self):
self.assertIn('results', output.lower())
self.assertTrue(os.path.exists(output_file))

def testModelRunStdIn(self):
output_file = self.TMP_DIR + '/model_output_stdin.shp'

params = {
'inputs': {
'FEATS': TEST_DATA_DIR + '/polys.shp',
'native:centroids_1:CENTROIDS': output_file
}
}

rc, output, err = self.run_process_stdin(['run', TEST_DATA_DIR + '/test_model.model3', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)
self.assertEqual(rc, 0)

res = json.loads(output)
self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)
self.assertEqual(res['algorithm_details']['id'], 'Test model')
self.assertTrue(os.path.exists(output_file))

def testModelRunJson(self):
output_file = self.TMP_DIR + '/model_output2.shp'
rc, output, err = self.run_process(['run', TEST_DATA_DIR + '/test_model.model3', '--json', '--', 'FEATS={}'.format(TEST_DATA_DIR + '/polys.shp'), 'native:centroids_1:CENTROIDS={}'.format(output_file)])
Expand Down Expand Up @@ -320,6 +413,33 @@ def testPythonScriptRunJson(self):
self.assertEqual(res['algorithm_details']['id'], 'script:converttouppercase')
self.assertEqual(res['results']['OUTPUT'], 'ABC')

def testScriptRunStdIn(self):
output_file = self.TMP_DIR + '/model_output_stdin.shp'

params = {
'inputs':
{
'INPUT': 'abc def'
}
}

rc, output, err = self.run_process_stdin(['run', TEST_DATA_DIR + '/convert_to_upper.py', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)

self.assertEqual(rc, 0)

res = json.loads(output)
self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)
self.assertEqual(res['algorithm_details']['id'], 'script:converttouppercase')
self.assertEqual(res['results']['OUTPUT'], 'ABC DEF')

def testPythonScriptRunNotAlgorithm(self):
rc, output, err = self.run_process(['run', TEST_DATA_DIR + '/not_a_processing_script.py'])
self.assertEqual(rc, 1)
Expand Down

0 comments on commit 5fd9b20

Please sign in to comment.