Skip to content

Commit

Permalink
processing: add windows support to exportRasterLayersIntoDirectory (f…
Browse files Browse the repository at this point in the history
…ixes #20146)
  • Loading branch information
jef-n committed Nov 11, 2018
1 parent 3a4a3af commit b39e5a0
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 16 deletions.
33 changes: 18 additions & 15 deletions python/plugins/processing/algs/grass7/Grass7Algorithm.py
Expand Up @@ -61,7 +61,6 @@
QgsProcessingParameterFile,
QgsProcessingParameterFolderDestination,
QgsProcessingOutputHtml,
QgsProcessingUtils,
QgsVectorLayer,
QgsProviderRegistry)
from qgis.utils import iface
Expand All @@ -73,7 +72,6 @@

from .Grass7Utils import Grass7Utils

#from processing.tools import dataobjects, system
from processing.tools.system import isWindows, getTempFilename

pluginPath = os.path.normpath(os.path.join(
Expand Down Expand Up @@ -187,7 +185,7 @@ def initAlgorithm(self, config=None):
"""
for p in self.params:
# We use createOutput argument for automatic output creation
res = self.addParameter(p, True)
self.addParameter(p, True)

def defineCharacteristicsFromFile(self):
"""
Expand Down Expand Up @@ -441,7 +439,7 @@ def processInputs(self, parameters, context, feedback):
QgsProcessingParameterMultipleLayers))]
for param in inputs:
paramName = param.name()
if not paramName in parameters:
if paramName not in parameters:
continue
# Handle Null parameter
if parameters[paramName] is None:
Expand Down Expand Up @@ -640,7 +638,6 @@ def processCommand(self, parameters, context, feedback, delOutputs=False):
if outName in parameters and parameters[outName] is not None:
# We add an output name to make sure it is unique if the session
# uses this algorithm several times.
#value = self.parameterAsOutputLayer(parameters, outName, context)
uniqueOutputName = outName + self.uniqueSuffix
command += ' {}={}'.format(outName, uniqueOutputName)

Expand Down Expand Up @@ -669,7 +666,7 @@ def processOutputs(self, parameters, context, feedback):

for out in self.destinationParameterDefinitions():
outName = out.name()
if not outName in parameters:
if outName not in parameters:
# skipped output
continue

Expand Down Expand Up @@ -785,15 +782,22 @@ def exportRasterLayersIntoDirectory(self, name, parameters, context, colorTable=

# Add a loop export from the basename
for cmd in [self.commands, self.outputCommands]:
# TODO Windows support
# TODO Format/options support
cmd.append("for r in $(g.list type=rast pattern='{}*'); do".format(basename))
cmd.append(" r.out.gdal -m{0} input=${{r}} output={1}/${{r}}.tif {2}".format(
' -t' if colorTable else '', outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
)
)
cmd.append("done")
if isWindows():
cmd.append("if not exist {0} mkdir {0}".format(outDir))
cmd.append("for /F %%r IN ('g.list type^=rast pattern^=\"{0}*\"') do r.out.gdal -m{1} input=%%r output={2}/%%r.tif {3}".format(
basename,
' -t' if colorTable else '',
outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
))
else:
cmd.append("for r in $(g.list type=rast pattern='{}*'); do".format(basename))
cmd.append(" r.out.gdal -m{0} input=${{r}} output={1}/${{r}}.tif {2}".format(
' -t' if colorTable else '', outDir,
'--overwrite -c createopt="TFW=YES,COMPRESS=LZW"'
))
cmd.append("done")

def loadVectorLayerFromParameter(self, name, parameters, context, feedback, external=False):
"""
Expand Down Expand Up @@ -999,7 +1003,6 @@ def setSessionProjectionFromLayer(self, layer):
def convertToHtml(self, fileName):
# Read HTML contents
lines = []
html = False
with open(fileName, 'r', encoding='utf-8') as f:
lines = f.readlines()

Expand Down
3 changes: 2 additions & 1 deletion python/plugins/processing/algs/grass7/Grass7Utils.py
Expand Up @@ -353,7 +353,7 @@ def executeGrass(commands, feedback, outputCommands=None):
loglines.append(Grass7Utils.tr('GRASS GIS 7 execution console output'))
grassOutDone = False
command, grassenv = Grass7Utils.prepareGrassExecution(commands)
#QgsMessageLog.logMessage('exec: {}'.format(command), 'DEBUG', Qgis.Info)
# QgsMessageLog.logMessage('exec: {}'.format(command), 'DEBUG', Qgis.Info)

# For MS-Windows, we need to hide the console window.
if isWindows():
Expand All @@ -369,6 +369,7 @@ def executeGrass(commands, feedback, outputCommands=None):
stderr=subprocess.STDOUT,
universal_newlines=True,
env=grassenv,
encoding="cp{}".format(Grass7Utils.getWindowsCodePage()) if isWindows() else None,
startupinfo=si if isWindows() else None
) as proc:
for line in iter(proc.stdout.readline, ''):
Expand Down

7 comments on commit b39e5a0

@EZUSoft
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getWindowsCodePage() return 1252-ansi (only correct for windows itself). But you need/mean the cmd.exe-console and it speaks MSDOS cp850.
https://gis.stackexchange.com/questions/302457/qgis-3-4-error-utf-8-codec-cant-decode-byte-0xe8-in-position-50-invalid-cont/302526?noredirect=1#comment488995_302526

@jef-n
Copy link
Member Author

@jef-n jef-n commented on b39e5a0 Nov 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

corresponds to the chcp in line 256.

@EZUSoft
Copy link

@EZUSoft EZUSoft commented on b39e5a0 Nov 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, but grass74.bat contains the line
if %ERRORLEVEL% GEQ 1 pause
This Output

"Drücken Sie eine beliebige Taste ..."

is cp850 and the code failed

Traceback (most recent call last):
File "C:/INTERW~1/QGIS 3.4.0/apps/qgis/./python/plugins\processing\algs\grass7\Grass7Algorithm.py", line 411, in processAlgorithm
Grass7Utils.executeGrass(self.commands, feedback, self.outputCommands)
File "C:/INTERW~1/QGIS 3.4.0/apps/qgis/./python/plugins\processing\algs\grass7\Grass7Utils.py", line 375, in executeGrass
for line in iter(proc.stdout.readline, ''):
File "C:\INTERW~1\QGIS 3.4.0\apps\Python37\lib\encodings\cp1252.py", line 23, in decode
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 2: character maps to <undefined>

@jef-n
Copy link
Member Author

@jef-n jef-n commented on b39e5a0 Nov 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, not really sure why the chcp was introduced in 6c81895 - guess to handle filenames with non-ascii characters in the produced batch job - but there's apparently no recoding of filename from UTF-8. After the chcp pause also produces cp1252. Just the message from pause after an error is still cp850 - everything else will be cp1252 (or whatever codepage GetACP() returns). So changing back to cp850 also doesn't seem to be a better choice. Maybe we should start grass74.bat from another batchfile and run chcp upfront (maybe even with 65001 - ie. UTF-8).

@PedroVenancio
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm also getting an error, due to this issue with portuguese regional settings:

Traceback (most recent call last):
File "C:/OSGEO4~1/apps/qgis-rel-dev/./python/plugins\processing\algs\grass7\Grass7Algorithm.py", line 413, in processAlgorithm
Grass7Utils.executeGrass(self.commands, feedback, self.outputCommands)
File "C:/OSGEO4~1/apps/qgis-rel-dev/./python/plugins\processing\algs\grass7\Grass7Utils.py", line 375, in executeGrass
for line in iter(proc.stdout.readline, ''):
File "C:\OSGEO4~1\apps\Python37\lib\codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'CP_UTF8' codec can't decode byte 0xc7 in position 4: No mapping for the Unicode character exists in the target code page.

@PedroVenancio
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@EZUSoft

Can you check which Active Code Page do you have in Windows Console?

I tried the updated QGIS 3.4.2-11 in another machine, and it worked without this issue. So I checked the Code Page and it is 850.

In the buggy machine, it was 65001. I had changed the default code page in this machine long time ago (Regional Settings -> Language for non-Unicode programs -> and marked the beta option "Use Unicode UTF-8 for worldwide language support"). Unchecking it, the problem goes away.

@somm-doe
Copy link

@somm-doe somm-doe commented on b39e5a0 Jan 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the buggy machine, it was 65001. I had changed the default code page in this machine long time ago (Regional Settings -> Language for non-Unicode programs -> and marked the beta option "Use Unicode UTF-8 for worldwide language support"). Unchecking it, the problem goes away.

I can confirm that removing the "Use Unicode UTF-8 for worldwide language support" option solves the issue.

Please sign in to comment.