Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #3728 from nyalldawson/open
Cleanup processing file handling
  • Loading branch information
nyalldawson committed Nov 7, 2016
2 parents 9b667d1 + 0484769 commit 4e4bb2b
Show file tree
Hide file tree
Showing 41 changed files with 536 additions and 585 deletions.
2 changes: 1 addition & 1 deletion python/plugins/processing/algs/gdal/GdalUtils.py
Expand Up @@ -87,7 +87,7 @@ def runGdal(commands, progress=None):
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=open(os.devnull),
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
).stdout
Expand Down
26 changes: 12 additions & 14 deletions python/plugins/processing/algs/gdal/extractprojection.py
Expand Up @@ -75,17 +75,15 @@ def processAlgorithm(self, progress):
crs = tmp.ExportToWkt()
tmp = None

prj = open(outFileName + '.prj', 'wt')
prj.write(crs)
prj.close()

wld = open(outFileName + '.wld', 'wt')
wld.write('%0.8f\n' % geotransform[1])
wld.write('%0.8f\n' % geotransform[4])
wld.write('%0.8f\n' % geotransform[2])
wld.write('%0.8f\n' % geotransform[5])
wld.write('%0.8f\n' % (geotransform[0] + 0.5 * geotransform[1] + 0.5
* geotransform[2]))
wld.write('%0.8f\n' % (geotransform[3] + 0.5 * geotransform[4] + 0.5
* geotransform[5]))
wld.close()
with open(outFileName + '.prj', 'wt') as prj:
prj.write(crs)

with open(outFileName + '.wld', 'wt') as wld:
wld.write('%0.8f\n' % geotransform[1])
wld.write('%0.8f\n' % geotransform[4])
wld.write('%0.8f\n' % geotransform[2])
wld.write('%0.8f\n' % geotransform[5])
wld.write('%0.8f\n' % (geotransform[0] + 0.5 * geotransform[1] + 0.5
* geotransform[2]))
wld.write('%0.8f\n' % (geotransform[3] + 0.5 * geotransform[4] + 0.5
* geotransform[5]))
11 changes: 5 additions & 6 deletions python/plugins/processing/algs/gdal/information.py
Expand Up @@ -76,9 +76,8 @@ def getConsoleCommands(self):
def processAlgorithm(self, progress):
GdalUtils.runGdal(self.getConsoleCommands(), progress)
output = self.getOutputValue(information.OUTPUT)
f = open(output, 'w')
f.write('<pre>')
for s in GdalUtils.getConsoleOutput()[1:]:
f.write(str(s))
f.write('</pre>')
f.close()
with open(output, 'w') as f:
f.write('<pre>')
for s in GdalUtils.getConsoleOutput()[1:]:
f.write(str(s))
f.write('</pre>')
117 changes: 57 additions & 60 deletions python/plugins/processing/algs/grass/GrassAlgorithm.py
Expand Up @@ -108,71 +108,68 @@ def getParameterDescriptions(self):
descs = {}
_, helpfile = self.help()
try:
infile = open(helpfile)
lines = infile.readlines()
for i in range(len(lines)):
if lines[i].startswith('<DT><b>'):
for param in self.parameters:
searchLine = '<b>' + param.name + '</b>'
if searchLine in lines[i]:
i += 1
descs[param.name] = (lines[i])[4:-6]
break

infile.close()
with open(helpfile) as infile:
lines = infile.readlines()
for i in range(len(lines)):
if lines[i].startswith('<DT><b>'):
for param in self.parameters:
searchLine = '<b>' + param.name + '</b>'
if searchLine in lines[i]:
i += 1
descs[param.name] = (lines[i])[4:-6]
break
except Exception:
pass
return descs

def defineCharacteristicsFromFile(self):
lines = open(self.descriptionFile)
line = lines.readline().strip('\n').strip()
self.grassName = line
line = lines.readline().strip('\n').strip()
self.name = line
self.i18n_name = QCoreApplication.translate("GrassAlgorithm", line)
if " - " not in self.name:
self.name = self.grassName + " - " + self.name
self.i18n_name = self.grassName + " - " + self.i18n_name
line = lines.readline().strip('\n').strip()
self.group = line
self.i18n_group = QCoreApplication.translate("GrassAlgorithm", line)
hasRasterOutput = False
hasVectorInput = False
vectorOutputs = 0
line = lines.readline().strip('\n').strip()
while line != '':
try:
line = line.strip('\n').strip()
if line.startswith('Hardcoded'):
self.hardcodedStrings.append(line[len('Hardcoded|'):])
parameter = getParameterFromString(line)
if parameter is not None:
self.addParameter(parameter)
if isinstance(parameter, ParameterVector):
hasVectorInput = True
if isinstance(parameter, ParameterMultipleInput) \
and parameter.datatype < 3:
hasVectorInput = True
else:
output = getOutputFromString(line)
self.addOutput(output)
if isinstance(output, OutputRaster):
hasRasterOutput = True
elif isinstance(output, OutputVector):
vectorOutputs += 1
if isinstance(output, OutputHTML):
self.addOutput(OutputFile("rawoutput", output.description +
" (raw output)", "txt"))
line = lines.readline().strip('\n').strip()
except Exception as e:

ProcessingLog.addToLog(
ProcessingLog.LOG_ERROR,
traceback.format_exc())
#self.tr('Could not open GRASS algorithm: %s.\n%s' % (self.descriptionFile, line)))
raise e
lines.close()
with open(self.descriptionFile) as lines:
line = lines.readline().strip('\n').strip()
self.grassName = line
line = lines.readline().strip('\n').strip()
self.name = line
self.i18n_name = QCoreApplication.translate("GrassAlgorithm", line)
if " - " not in self.name:
self.name = self.grassName + " - " + self.name
self.i18n_name = self.grassName + " - " + self.i18n_name
line = lines.readline().strip('\n').strip()
self.group = line
self.i18n_group = QCoreApplication.translate("GrassAlgorithm", line)
hasRasterOutput = False
hasVectorInput = False
vectorOutputs = 0
line = lines.readline().strip('\n').strip()
while line != '':
try:
line = line.strip('\n').strip()
if line.startswith('Hardcoded'):
self.hardcodedStrings.append(line[len('Hardcoded|'):])
parameter = getParameterFromString(line)
if parameter is not None:
self.addParameter(parameter)
if isinstance(parameter, ParameterVector):
hasVectorInput = True
if isinstance(parameter, ParameterMultipleInput) \
and parameter.datatype < 3:
hasVectorInput = True
else:
output = getOutputFromString(line)
self.addOutput(output)
if isinstance(output, OutputRaster):
hasRasterOutput = True
elif isinstance(output, OutputVector):
vectorOutputs += 1
if isinstance(output, OutputHTML):
self.addOutput(OutputFile("rawoutput", output.description +
" (raw output)", "txt"))
line = lines.readline().strip('\n').strip()
except Exception as e:

ProcessingLog.addToLog(
ProcessingLog.LOG_ERROR,
traceback.format_exc())
#self.tr('Could not open GRASS algorithm: %s.\n%s' % (self.descriptionFile, line)))
raise e

self.addParameter(ParameterExtent(
self.GRASS_REGION_EXTENT_PARAMETER,
Expand Down
109 changes: 52 additions & 57 deletions python/plugins/processing/algs/grass/GrassUtils.py
Expand Up @@ -136,56 +136,53 @@ def createGrassScript(commands):

encoding = locale.getpreferredencoding()
# Temporary gisrc file
output = codecs.open(gisrc, 'w', encoding=encoding)
location = 'temp_location'
gisdbase = GrassUtils.grassDataFolder()

output.write('GISDBASE: ' + gisdbase + '\n')
output.write('LOCATION_NAME: ' + location + '\n')
output.write('MAPSET: PERMANENT \n')
output.write('GRASS_GUI: text\n')
output.close()

output = codecs.open(script, 'w', encoding=encoding)
output.write('set HOME=' + os.path.expanduser('~') + '\n')
output.write('set GISRC=' + gisrc + '\n')
output.write('set GRASS_SH=' + shell + '\\bin\\sh.exe\n')
output.write('set PATH=' + os.path.join(shell, 'bin') + ';' + os.path.join(shell, 'lib') + ';' + '%PATH%\n')
output.write('set WINGISBASE=' + folder + '\n')
output.write('set GISBASE=' + folder + '\n')
output.write('set GRASS_PROJSHARE=' + os.path.join(folder, 'share', 'proj') + '\n')
output.write('set GRASS_MESSAGE_FORMAT=gui\n')

# Replacement code for etc/Init.bat
output.write('if "%GRASS_ADDON_PATH%"=="" set PATH=%WINGISBASE%\\bin;%WINGISBASE%\\lib;%PATH%\n')
output.write('if not "%GRASS_ADDON_PATH%"=="" set PATH=%WINGISBASE%\\bin;%WINGISBASE%\\lib;%GRASS_ADDON_PATH%;%PATH%\n')
output.write('\n')
output.write('set GRASS_VERSION=' + GrassUtils.getGrassVersion() + '\n')
output.write('if not "%LANG%"=="" goto langset\n')
output.write('FOR /F "usebackq delims==" %%i IN (`"%WINGISBASE%\\etc\\winlocale"`) DO @set LANG=%%i\n')
output.write(':langset\n')
output.write('\n')
output.write('set PATHEXT=%PATHEXT%;.PY\n')
output.write('set PYTHONPATH=%PYTHONPATH%;%WINGISBASE%\\etc\\python;%WINGISBASE%\\etc\\wxpython\\n')
output.write('\n')
output.write('g.gisenv.exe set="MAPSET=PERMANENT"\n')
output.write('g.gisenv.exe set="LOCATION=' + location + '"\n')
output.write('g.gisenv.exe set="LOCATION_NAME=' + location + '"\n')
output.write('g.gisenv.exe set="GISDBASE=' + gisdbase + '"\n')
output.write('g.gisenv.exe set="GRASS_GUI=text"\n')
for command in commands:
output.write(command + u'\n')
output.write('\n')
output.write('exit\n')
output.close()
with codecs.open(gisrc, 'w', encoding=encoding) as output:
location = 'temp_location'
gisdbase = GrassUtils.grassDataFolder()

output.write('GISDBASE: ' + gisdbase + '\n')
output.write('LOCATION_NAME: ' + location + '\n')
output.write('MAPSET: PERMANENT \n')
output.write('GRASS_GUI: text\n')

with codecs.open(script, 'w', encoding=encoding) as output:
output.write('set HOME=' + os.path.expanduser('~') + '\n')
output.write('set GISRC=' + gisrc + '\n')
output.write('set GRASS_SH=' + shell + '\\bin\\sh.exe\n')
output.write('set PATH=' + os.path.join(shell, 'bin') + ';' + os.path.join(shell, 'lib') + ';' + '%PATH%\n')
output.write('set WINGISBASE=' + folder + '\n')
output.write('set GISBASE=' + folder + '\n')
output.write('set GRASS_PROJSHARE=' + os.path.join(folder, 'share', 'proj') + '\n')
output.write('set GRASS_MESSAGE_FORMAT=gui\n')

# Replacement code for etc/Init.bat
output.write('if "%GRASS_ADDON_PATH%"=="" set PATH=%WINGISBASE%\\bin;%WINGISBASE%\\lib;%PATH%\n')
output.write('if not "%GRASS_ADDON_PATH%"=="" set PATH=%WINGISBASE%\\bin;%WINGISBASE%\\lib;%GRASS_ADDON_PATH%;%PATH%\n')
output.write('\n')
output.write('set GRASS_VERSION=' + GrassUtils.getGrassVersion() + '\n')
output.write('if not "%LANG%"=="" goto langset\n')
output.write('FOR /F "usebackq delims==" %%i IN (`"%WINGISBASE%\\etc\\winlocale"`) DO @set LANG=%%i\n')
output.write(':langset\n')
output.write('\n')
output.write('set PATHEXT=%PATHEXT%;.PY\n')
output.write('set PYTHONPATH=%PYTHONPATH%;%WINGISBASE%\\etc\\python;%WINGISBASE%\\etc\\wxpython\\n')
output.write('\n')
output.write('g.gisenv.exe set="MAPSET=PERMANENT"\n')
output.write('g.gisenv.exe set="LOCATION=' + location + '"\n')
output.write('g.gisenv.exe set="LOCATION_NAME=' + location + '"\n')
output.write('g.gisenv.exe set="GISDBASE=' + gisdbase + '"\n')
output.write('g.gisenv.exe set="GRASS_GUI=text"\n')
for command in commands:
output.write(command + u'\n')
output.write('\n')
output.write('exit\n')

@staticmethod
def createGrassBatchJobFileFromGrassCommands(commands):
fout = codecs.open(GrassUtils.grassBatchJobFilename(), 'w', encoding='utf-8')
for command in commands:
fout.write(command + u'\n')
fout.write('exit')
fout.close()
with codecs.open(GrassUtils.grassBatchJobFilename(), 'w', encoding='utf-8') as fout:
for command in commands:
fout.write(command + u'\n')
fout.write('exit')

@staticmethod
def grassMapsetFolder():
Expand Down Expand Up @@ -213,17 +210,15 @@ def createTempMapset():
mkdir(os.path.join(folder, 'PERMANENT'))
mkdir(os.path.join(folder, 'PERMANENT', '.tmp'))
GrassUtils.writeGrassWindow(os.path.join(folder, 'PERMANENT', 'DEFAULT_WIND'))
outfile = codecs.open(os.path.join(folder, 'PERMANENT', 'MYNAME'), 'w', encoding='utf-8')
outfile.write(
'QGIS GRASS interface: temporary data processing location.\n')
outfile.close()
with codecs.open(os.path.join(folder, 'PERMANENT', 'MYNAME'), 'w', encoding='utf-8') as outfile:
outfile.write(
'QGIS GRASS interface: temporary data processing location.\n')

GrassUtils.writeGrassWindow(os.path.join(folder, 'PERMANENT', 'WIND'))
mkdir(os.path.join(folder, 'PERMANENT', 'dbf'))
outfile = codecs.open(os.path.join(folder, 'PERMANENT', 'VAR'), 'w', encoding='utf-8')
outfile.write('DB_DRIVER: dbf\n')
outfile.write('DB_DATABASE: $GISDBASE/$LOCATION_NAME/$MAPSET/dbf/\n')
outfile.close()
with codecs.open(os.path.join(folder, 'PERMANENT', 'VAR'), 'w', encoding='utf-8') as outfile:
outfile.write('DB_DRIVER: dbf\n')
outfile.write('DB_DATABASE: $GISDBASE/$LOCATION_NAME/$MAPSET/dbf/\n')

@staticmethod
def writeGrassWindow(filename):
Expand Down Expand Up @@ -282,7 +277,7 @@ def executeGrass(commands, progress, outputCommands=None):
command,
shell=True,
stdout=subprocess.PIPE,
stdin=open(os.devnull),
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
env=grassenv
Expand Down Expand Up @@ -312,7 +307,7 @@ def executeGrass(commands, progress, outputCommands=None):
command,
shell=True,
stdout=subprocess.PIPE,
stdin=open(os.devnull),
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
env=grassenv
Expand Down

0 comments on commit 4e4bb2b

Please sign in to comment.