Skip to content

Commit

Permalink
Reduce appends
Browse files Browse the repository at this point in the history
  • Loading branch information
@dericke @nyalldawson
dericke authored and nyalldawson committed Feb 20, 2021
1 parent dcde81f commit 0e63799
Show file tree
Hide file tree
Showing 26 changed files with 205 additions and 199 deletions.
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py
View file
Expand Up @@ -126,7 +126,7 @@ def name(self):

def longName(self):
version = GdalUtils.readableVersion()
return 'GDAL ({})'.format(version)
return f'GDAL ({version})'

def id(self):
return 'gdal'
Expand Down
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/gdal/GdalUtils.py
View file
Expand Up @@ -358,7 +358,7 @@ def ogrConnectionStringAndFormatFromLayer(layer):
if dsUri.host() != "":
ogrstr += delim + dsUri.host()
delim = ""
if dsUri.port() != "" and dsUri.port() != '1521':
if dsUri.port() not in ["", '1521']:
ogrstr += ":" + dsUri.port()
ogrstr += "/"
if dsUri.database() != "":
Expand Down
21 changes: 8 additions & 13 deletions python/plugins/processing/algs/gdal/PointsAlongLines.py
View file
Expand Up @@ -100,31 +100,26 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)

other_fields = []
for f in fields:
if f.name() == geometry:
continue
other_fields.append('"{}"'.format(f.name()))

if other_fields:
other_fields = ',*'
else:
other_fields = ''
other_fields = [
f'"{f.name()}"'
for f in fields
if f.name() != geometry
]

other_fields = ',*' if other_fields else ''
arguments = [
output,
ogrLayer,
'-dialect',
'sqlite',
'-sql'
f'SELECT ST_Line_Interpolate_Point({geometry}, {distance}) AS {geometry}{other_fields} FROM "{layerName}"'
]
sql = 'SELECT ST_Line_Interpolate_Point({}, {}) AS {}{} FROM "{}"'.format(geometry, distance, geometry, other_fields, layerName)
arguments.append(sql)

if options:
arguments.append(options)

if outputFormat:
arguments.append('-f {}'.format(outputFormat))
arguments.append(f'-f {outputFormat}')

return ['ogr2ogr', GdalUtils.escapeAndJoin(arguments)]
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/gdal/extractprojection.py
View file
Expand Up @@ -88,7 +88,7 @@ def processAlgorithm(self, parameters, context, feedback):
context)
raster = self.parameterAsRasterLayer(parameters, self.INPUT,
context)
if not raster.dataProvider().name() == 'gdal':
if raster.dataProvider().name() != 'gdal':
raise QgsProcessingException(self.tr('This algorithm can '
'only be used with '
'GDAL raster layers'))
Expand Down
14 changes: 8 additions & 6 deletions python/plugins/processing/algs/gdal/gdalcalc.py
View file
Expand Up @@ -190,12 +190,14 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
else:
noData = None

arguments = []
arguments.append('--calc "{}"'.format(formula))
arguments.append('--format')
arguments.append(GdalUtils.getFormatShortNameFromFilename(out))
arguments.append('--type')
arguments.append(self.TYPE[self.parameterAsEnum(parameters, self.RTYPE, context)])
arguments = [
f'--calc "{formula}"',
'--format',
GdalUtils.getFormatShortNameFromFilename(out),
'--type',
self.TYPE[self.parameterAsEnum(parameters, self.RTYPE, context)]
]

if noData is not None:
arguments.append('--NoDataValue')
arguments.append(noData)
Expand Down
41 changes: 29 additions & 12 deletions python/plugins/processing/algs/otb/OtbAlgorithm.py
View file
Expand Up @@ -191,7 +191,10 @@ def preprocessParameters(self, parameters):
# - metadata without a 'group_key'
# - metadata with 'group_key' and 'group_key' is not in list of parameters. see ParameterGroup in OTB
# - metadata with 'group_key' and 'group_key' is a valid parameter and it's value == metadata()['group_value']
if 'group_key' in param.metadata() and not param.metadata()['group_key'] in parameters:
if (
'group_key' in param.metadata()
and param.metadata()['group_key'] not in parameters
):
valid_params[k] = v
elif 'group_key' not in param.metadata() or parameters[param.metadata()['group_key']] == param.metadata()[
'group_value']:
Expand Down Expand Up @@ -219,9 +222,14 @@ def processAlgorithm(self, parameters, context, feedback):
if isinstance(param, QgsProcessingParameterEnum) and param.name() == "outputpixeltype":
value = self.parameterAsEnum(parameters, param.name(), context)
elif isinstance(param, QgsProcessingParameterEnum):
value = " ".join([param.options()[i]
for i in self.parameterAsEnums(parameters, param.name(), context)
if i >= 0 and i < len(param.options())])
value = " ".join(
param.options()[i]
for i in self.parameterAsEnums(
parameters, param.name(), context
)
if i >= 0 and i < len(param.options())
)

elif isinstance(param, QgsProcessingParameterField):
value = " ".join(self.parameterAsFields(parameters, param.name(), context))
elif isinstance(param, QgsProcessingParameterBoolean):
Expand All @@ -242,7 +250,11 @@ def processAlgorithm(self, parameters, context, feedback):
layers = self.parameterAsLayerList(parameters, param.name(), context)
if layers is None or len(layers) == 0:
continue
value = ' '.join(['"{}"'.format(self.getLayerSource(param.name(), layer)) for layer in layers])
value = ' '.join(
'"{}"'.format(self.getLayerSource(param.name(), layer))
for layer in layers
)

elif isinstance(param, QgsProcessingParameterNumber):
if param.dataType() == QgsProcessingParameterNumber.Integer:
value = self.parameterAsInt(parameters, param.name(), context)
Expand All @@ -253,8 +265,13 @@ def processAlgorithm(self, parameters, context, feedback):
elif isinstance(param, QgsProcessingParameterString):
value = '"{}"'.format(self.parameterAsString(parameters, param.name(), context))
elif isinstance(param, QgsProcessingParameterBand):
value = ' '.join(['"Channel{}"'.format(index) for index in
self.parameterAsInts(parameters, param.name(), context)])
value = ' '.join(
'"Channel{}"'.format(index)
for index in self.parameterAsInts(
parameters, param.name(), context
)
)

else:
# Use whatever is given
value = '"{}"'.format(parameters[param.name()])
Expand All @@ -276,11 +293,11 @@ def processAlgorithm(self, parameters, context, feedback):

OtbUtils.executeOtb(command, feedback)

result = {}
for o in self.outputDefinitions():
if o.name() in output_files:
result[o.name()] = output_files[o.name()]
return result
return {
o.name(): output_files[o.name()]
for o in self.outputDefinitions()
if o.name() in output_files
}

def getLayerSource(self, name, layer):
providerName = layer.dataProvider().name()
Expand Down
12 changes: 6 additions & 6 deletions python/plugins/processing/algs/otb/OtbAlgorithmProvider.py
View file
Expand Up @@ -104,12 +104,12 @@ def createAlgsList(self):
algs = []
try:
folder = OtbUtils.otbFolder()
alg_names = []
algs_txt = self.algsFile(folder)
with open(algs_txt) as lines:
line = lines.readline().strip('\n').strip()
if line != '' and line.startswith('#'):
line = lines.readline().strip('\n').strip()
alg_names = []
while line != '' and not line.startswith('#'):
data = line.split('|')
descriptionFile = self.descrFile(folder, str(data[1]) + '.txt')
Expand Down Expand Up @@ -237,11 +237,11 @@ def descrFile(self, d, f):
return os.path.join(self.descrFolder(d), f)

def appDirs(self, v):
app_dirs = []
for f in v.split(';'):
if f is not None and os.path.exists(f):
app_dirs.append(self.normalize_path(f))
return app_dirs
return [
self.normalize_path(f)
for f in v.split(';')
if f is not None and os.path.exists(f)
]

def name(self):
return 'OTB'
Expand Down
25 changes: 11 additions & 14 deletions python/plugins/processing/algs/otb/OtbUtils.py
View file
Expand Up @@ -142,10 +142,8 @@ def getAuxiliaryDataDirectories():
if 'PROJ_LIB=' in line:
proj_dir = line.split("PROJ_LIB=")[1]
except BaseException as exc:
errmsg = "Cannot find gdal and geotiff data directory." + str(exc)
errmsg = 'Cannot find gdal and geotiff data directory.' + str(exc)
QgsMessageLog.logMessage(errmsg, OtbUtils.tr('Processing'), Qgis.Info)
pass

return gdal_data_dir, gtiff_csv_dir, proj_dir

@staticmethod
Expand All @@ -167,21 +165,20 @@ def executeOtb(commands, feedback, addToLog=True):
if max_ram_hint and int(max_ram_hint) > 256:
otb_env['OTB_MAX_RAM_HINT'] = max_ram_hint

kw = {}
kw['env'] = otb_env
kw = {'env': otb_env}
if os.name == 'nt' and sys.version_info >= (3, 6):
kw['encoding'] = "cp{}".format(OtbUtils.getWindowsCodePage())

QgsMessageLog.logMessage("{}".format(kw), OtbUtils.tr('Processing'), Qgis.Info)
QgsMessageLog.logMessage("cmd={}".format(commands), OtbUtils.tr('Processing'), Qgis.Info)
with subprocess.Popen(
commands,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
**kw
commands,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
**kw
) as proc:

for line in iter(proc.stdout.readline, ''):
Expand All @@ -200,9 +197,9 @@ def executeOtb(commands, feedback, addToLog=True):
if feedback is None:
QgsMessageLog.logMessage(line, OtbUtils.tr('Processing'), Qgis.Info)
else:
if any([l in line for l in ['(WARNING)', 'WARNING:']]):
if any(l in line for l in ['(WARNING)', 'WARNING:']):
feedback.reportError(line, False)
elif any([l in line for l in ['(FATAL)', 'ERROR:', 'ERROR']]):
elif any(l in line for l in ['(FATAL)', 'ERROR:', 'ERROR']):
feedback.reportError(line, True)
else:
feedback.pushConsoleInfo(line.strip())
Expand Down
76 changes: 37 additions & 39 deletions python/plugins/processing/algs/qgis/BasicStatistics.py
View file
Expand Up @@ -145,19 +145,16 @@ def processAlgorithm(self, parameters, context, feedback):
features = source.getFeatures(request, QgsProcessingFeatureSource.FlagSkipGeometryValidityChecks)
count = source.featureCount()

data = []
data.append(self.tr('Analyzed field: {}').format(field_name))
data = [self.tr('Analyzed field: {}').format(field_name)]
results = {}

if field.isNumeric():
d, results = self.calcNumericStats(features, feedback, field, count)
data.extend(d)
elif field.type() in (QVariant.Date, QVariant.Time, QVariant.DateTime):
d, results = self.calcDateTimeStats(features, feedback, field, count)
data.extend(d)
else:
d, results = self.calcStringStats(features, feedback, field, count)
data.extend(d)
data.extend(d)

if output_file:
self.createHTML(output_file, data)
Expand Down Expand Up @@ -195,23 +192,24 @@ def calcNumericStats(self, features, feedback, field, count):
self.THIRDQUARTILE: stat.thirdQuartile(),
self.IQR: stat.interQuartileRange()}

data = []
data.append(self.tr('Count: {}').format(stat.count()))
data.append(self.tr('Unique values: {}').format(stat.variety()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(self.tr('Minimum value: {}').format(stat.min()))
data.append(self.tr('Maximum value: {}').format(stat.max()))
data.append(self.tr('Range: {}').format(stat.range()))
data.append(self.tr('Sum: {}').format(stat.sum()))
data.append(self.tr('Mean value: {}').format(stat.mean()))
data.append(self.tr('Median value: {}').format(stat.median()))
data.append(self.tr('Standard deviation: {}').format(stat.stDev()))
data.append(self.tr('Coefficient of Variation: {}').format(cv))
data.append(self.tr('Minority (rarest occurring value): {}').format(stat.minority()))
data.append(self.tr('Majority (most frequently occurring value): {}').format(stat.majority()))
data.append(self.tr('First quartile: {}').format(stat.firstQuartile()))
data.append(self.tr('Third quartile: {}').format(stat.thirdQuartile()))
data.append(self.tr('Interquartile Range (IQR): {}').format(stat.interQuartileRange()))
data = [
self.tr('Count: {}').format(stat.count()),
self.tr('Unique values: {}').format(stat.variety()),
self.tr('NULL (missing) values: {}').format(stat.countMissing()),
self.tr('Minimum value: {}').format(stat.min()),
self.tr('Maximum value: {}').format(stat.max()),
self.tr('Range: {}').format(stat.range()),
self.tr('Sum: {}').format(stat.sum()),
self.tr('Mean value: {}').format(stat.mean()),
self.tr('Median value: {}').format(stat.median()),
self.tr('Standard deviation: {}').format(stat.stDev()),
self.tr('Coefficient of Variation: {}').format(cv),
self.tr('Minority (rarest occurring value): {}').format(stat.minority()),
self.tr('Majority (most frequently occurring value): {}').format(stat.majority()),
self.tr('First quartile: {}').format(stat.firstQuartile()),
self.tr('Third quartile: {}').format(stat.thirdQuartile()),
self.tr('Interquartile Range (IQR): {}').format(stat.interQuartileRange())
]
return data, results

def calcStringStats(self, features, feedback, field, count):
Expand All @@ -234,15 +232,16 @@ def calcStringStats(self, features, feedback, field, count):
self.MAX_LENGTH: stat.maxLength(),
self.MEAN_LENGTH: stat.meanLength()}

data = []
data.append(self.tr('Count: {}').format(count))
data.append(self.tr('Unique values: {}').format(stat.countDistinct()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(self.tr('Minimum value: {}').format(stat.min()))
data.append(self.tr('Maximum value: {}').format(stat.max()))
data.append(self.tr('Minimum length: {}').format(stat.minLength()))
data.append(self.tr('Maximum length: {}').format(stat.maxLength()))
data.append(self.tr('Mean length: {}').format(stat.meanLength()))
data = [
self.tr('Count: {}').format(count),
self.tr('Unique values: {}').format(stat.countDistinct()),
self.tr('NULL (missing) values: {}').format(stat.countMissing()),
self.tr('Minimum value: {}').format(stat.min()),
self.tr('Maximum value: {}').format(stat.max()),
self.tr('Minimum length: {}').format(stat.minLength()),
self.tr('Maximum length: {}').format(stat.maxLength()),
self.tr('Mean length: {}').format(stat.meanLength())
]

return data, results

Expand All @@ -263,14 +262,13 @@ def calcDateTimeStats(self, features, feedback, field, count):
self.MIN: stat.statistic(QgsDateTimeStatisticalSummary.Min),
self.MAX: stat.statistic(QgsDateTimeStatisticalSummary.Max)}

data = []
data.append(self.tr('Count: {}').format(count))
data.append(self.tr('Unique values: {}').format(stat.countDistinct()))
data.append(self.tr('NULL (missing) values: {}').format(stat.countMissing()))
data.append(
self.tr('Minimum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Min))))
data.append(
self.tr('Maximum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Max))))
data = [
self.tr('Count: {}').format(count),
self.tr('Unique values: {}').format(stat.countDistinct()),
self.tr('NULL (missing) values: {}').format(stat.countMissing()),
self.tr('Minimum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Min))),
self.tr('Maximum value: {}').format(field.displayString(stat.statistic(QgsDateTimeStatisticalSummary.Max)))
]

return data, results

Expand Down
8 changes: 4 additions & 4 deletions python/plugins/processing/algs/qgis/CheckValidity.py
View file
Expand Up @@ -116,13 +116,13 @@ def processAlgorithm(self, parameters, context, feedback):
if method_param == 0:
settings = QgsSettings()
method = int(settings.value(settings_method_key, 0)) - 1
if method < 0:
method = 0
method = max(method, 0)
else:
method = method_param - 1

results = self.doCheck(method, parameters, context, feedback, ignore_ring_self_intersection)
return results
return self.doCheck(
method, parameters, context, feedback, ignore_ring_self_intersection
)

def doCheck(self, method, parameters, context, feedback, ignore_ring_self_intersection):
flags = QgsGeometry.FlagAllowSelfTouchingHoles if ignore_ring_self_intersection else QgsGeometry.ValidityFlags()
Expand Down
7 changes: 3 additions & 4 deletions python/plugins/processing/algs/qgis/ExecuteSQL.py
View file
Expand Up @@ -145,10 +145,9 @@ def processAlgorithm(self, parameters, context, feedback):
if query == '':
raise QgsProcessingException(
self.tr('Empty SQL. Please enter valid SQL expression and try again.'))
else:
localContext = self.createExpressionContext(parameters, context)
expandedQuery = QgsExpression.replaceExpressionText(query, localContext)
df.setQuery(expandedQuery)
localContext = self.createExpressionContext(parameters, context)
expandedQuery = QgsExpression.replaceExpressionText(query, localContext)
df.setQuery(expandedQuery)

if uid_field:
df.setUid(uid_field)
Expand Down

0 comments on commit 0e63799

Please sign in to comment.