Merge pull request #5825 from nyalldawson/gdal

[processing] Some GDAL algorithm fixes
qgis · Dec 9, 2017 · 53ebe05 · 53ebe05
2 parents 297138f + b578f8a
commit 53ebe05
Show file tree

Hide file tree

Showing 57 changed files with 273 additions and 246 deletions.
diff --git a/python/core/processing/qgsprocessingutils.sip b/python/core/processing/qgsprocessingutils.sip
@@ -273,6 +273,10 @@ class QgsProcessingFeatureSource : QgsFeatureSource
 
     virtual QVariant maximumValue( int fieldIndex ) const;
 
+    virtual QgsRectangle sourceExtent() const;
+
+    virtual QgsFeatureIds allFeatureIds() const;
+
 
     QgsExpressionContextScope *createExpressionContextScope() const /Factory/;
 %Docstring

diff --git a/python/plugins/processing/algs/gdal/AssignProjection.py b/python/plugins/processing/algs/gdal/AssignProjection.py
@@ -70,7 +70,7 @@ def icon(self):
     def group(self):
         return self.tr('Raster projections')
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
         fileName = inLayer.source()
 

diff --git a/python/plugins/processing/algs/gdal/Buffer.py b/python/plugins/processing/algs/gdal/Buffer.py
@@ -98,9 +98,9 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         fields = self.parameterAsSource(parameters, self.INPUT, context).fields()
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
         distance = self.parameterAsDouble(parameters, self.DISTANCE, context)
         fieldName = self.parameterAsString(parameters, self.FIELD, context)

diff --git a/python/plugins/processing/algs/gdal/ClipRasterByExtent.py b/python/plugins/processing/algs/gdal/ClipRasterByExtent.py
@@ -105,7 +105,7 @@ def icon(self):
     def commandName(self):
         return "gdal_translate"
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
         bbox = self.parameterAsExtent(parameters, self.EXTENT, context, inLayer.crs())
         nodata = self.parameterAsDouble(parameters, self.NODATA, context)

diff --git a/python/plugins/processing/algs/gdal/ClipRasterByMask.py b/python/plugins/processing/algs/gdal/ClipRasterByMask.py
@@ -116,10 +116,10 @@ def icon(self):
     def group(self):
         return self.tr('Raster extraction')
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
 
-        maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback)
+        maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)
 
         nodata = self.parameterAsDouble(parameters, self.NODATA, context)
         options = self.parameterAsString(parameters, self.OPTIONS, context)

diff --git a/python/plugins/processing/algs/gdal/ClipVectorByExtent.py b/python/plugins/processing/algs/gdal/ClipVectorByExtent.py
@@ -74,8 +74,8 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         source = self.parameterAsSource(parameters, self.INPUT, context)
         extent = self.parameterAsExtent(parameters, self.EXTENT, context, source.sourceCrs())
         options = self.parameterAsString(parameters, self.OPTIONS, context)

diff --git a/python/plugins/processing/algs/gdal/ClipVectorByMask.py b/python/plugins/processing/algs/gdal/ClipVectorByMask.py
@@ -73,9 +73,9 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        inLayer, inLayerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
-        maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        inLayer, inLayerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
+        maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)
         options = self.parameterAsString(parameters, self.OPTIONS, context)
         outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
 

diff --git a/python/plugins/processing/algs/gdal/ColorRelief.py b/python/plugins/processing/algs/gdal/ColorRelief.py
@@ -92,7 +92,7 @@ def displayName(self):
     def group(self):
         return self.tr('Raster analysis')
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         arguments = ['color-relief']
         inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
         arguments.append(inLayer.source())

diff --git a/python/plugins/processing/algs/gdal/Dissolve.py b/python/plugins/processing/algs/gdal/Dissolve.py
@@ -109,9 +109,9 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         fields = self.parameterAsSource(parameters, self.INPUT, context).fields()
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
         fieldName = self.parameterAsString(parameters, self.FIELD, context)
 

diff --git a/python/plugins/processing/algs/gdal/ExecuteSql.py b/python/plugins/processing/algs/gdal/ExecuteSql.py
@@ -82,8 +82,8 @@ def displayName(self):
     def group(self):
         return self.tr('Vector miscellaneous')
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         sql = self.parameterAsString(parameters, self.SQL, context)
         options = self.parameterAsString(parameters, self.OPTIONS, context)
         outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)

diff --git a/python/plugins/processing/algs/gdal/GdalAlgorithm.py b/python/plugins/processing/algs/gdal/GdalAlgorithm.py
@@ -59,30 +59,44 @@ def createInstance(self, config={}):
     def createCustomParametersWidget(self, parent):
         return GdalAlgorithmDialog(self)
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         return None
 
-    def getOgrCompatibleSource(self, parameter_name, parameters, context, feedback):
+    def getOgrCompatibleSource(self, parameter_name, parameters, context, feedback, executing):
         """
         Interprets a parameter as an OGR compatible source and layer name
+        :param executing:
         """
         input_layer = self.parameterAsVectorLayer(parameters, parameter_name, context)
         ogr_data_path = None
         ogr_layer_name = None
         if input_layer is None:
-            # parameter is not a vector layer - try to convert to a source compatible with OGR
-            # and extract selection if required
-            ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
-                                                                      QgsVectorFileWriter.supportedFormatExtensions(),
-                                                                      feedback=feedback)
-            ogr_layer_name = GdalUtils.ogrLayerName(ogr_data_path)
+            if executing:
+                # parameter is not a vector layer - try to convert to a source compatible with OGR
+                # and extract selection if required
+                ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
+                                                                          QgsVectorFileWriter.supportedFormatExtensions(),
+                                                                          feedback=feedback)
+                ogr_layer_name = GdalUtils.ogrLayerName(ogr_data_path)
+            else:
+                #not executing - don't waste time converting incompatible sources, just return dummy strings
+                #for the command preview (since the source isn't compatible with OGR, it has no meaning anyway and can't
+                #be run directly in the command line)
+                ogr_data_path = 'path_to_data_file'
+                ogr_layer_name = 'layer_name'
         elif input_layer.dataProvider().name() == 'ogr':
-            # parameter is a vector layer, with OGR data provider
-            # so extract selection if required
-            ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
-                                                                      QgsVectorFileWriter.supportedFormatExtensions(),
-                                                                      feedback=feedback)
-            ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
+            if executing:
+                # parameter is a vector layer, with OGR data provider
+                # so extract selection if required
+                ogr_data_path = self.parameterAsCompatibleSourceLayerPath(parameters, parameter_name, context,
+                                                                          QgsVectorFileWriter.supportedFormatExtensions(),
+                                                                          feedback=feedback)
+                ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
+            else:
+                #not executing - don't worry about 'selected features only' handling. It has no meaning
+                #for the command line preview since it has no meaning outside of a QGIS session!
+                ogr_data_path = GdalUtils.ogrConnectionString(input_layer.dataProvider().dataSourceUri(), context)[1:-1]
+                ogr_layer_name = GdalUtils.ogrLayerName(input_layer.dataProvider().dataSourceUri())
         else:
             # vector layer, but not OGR - get OGR compatible path
             # TODO - handle "selected features only" mode!!
@@ -94,7 +108,7 @@ def setOutputValue(self, name, value):
         self.output_values[name] = value
 
     def processAlgorithm(self, parameters, context, feedback):
-        commands = self.getConsoleCommands(parameters, context, feedback)
+        commands = self.getConsoleCommands(parameters, context, feedback, executing=True)
         GdalUtils.runGdal(commands, feedback)
 
         # auto generate outputs

diff --git a/python/plugins/processing/algs/gdal/GdalAlgorithmDialog.py b/python/plugins/processing/algs/gdal/GdalAlgorithmDialog.py
@@ -118,7 +118,7 @@ def parametersHaveChanged(self):
                     self.text.setPlainText('')
                     return
 
-            commands = self.alg.getConsoleCommands(parameters, context, feedback)
+            commands = self.alg.getConsoleCommands(parameters, context, feedback, executing=False)
             commands = [c for c in commands if c not in ['cmd.exe', '/C ']]
             self.text.setPlainText(" ".join(commands))
         except AlgorithmDialogBase.InvalidParameterValue as e:

diff --git a/python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py b/python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py
@@ -82,10 +82,10 @@
 from .ogr2ogr import ogr2ogr
 from .ogrinfo import ogrinfo
 from .OgrToPostGis import OgrToPostGis
+from .ogr2ogrtopostgislist import Ogr2OgrToPostGisList
 from .OneSideBuffer import OneSideBuffer
 from .PointsAlongLines import PointsAlongLines
 
-# from .ogr2ogrtopostgislist import Ogr2OgrToPostGisList
 # from .ogr2ogrtabletopostgislist import Ogr2OgrTableToPostGisList
 
 pluginPath = os.path.normpath(os.path.join(
@@ -187,9 +187,9 @@ def loadAlgorithms(self):
             ogr2ogr(),
             ogrinfo(),
             OgrToPostGis(),
+            Ogr2OgrToPostGisList(),
             OneSideBuffer(),
             PointsAlongLines(),
-            # Ogr2OgrToPostGisList(),
             # Ogr2OgrTableToPostGisList(),
         ]
         for a in self.algs:

diff --git a/python/plugins/processing/algs/gdal/GridAverage.py b/python/plugins/processing/algs/gdal/GridAverage.py
@@ -135,8 +135,8 @@ def group(self):
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'gdaltools', 'grid.png'))
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/GridDataMetrics.py b/python/plugins/processing/algs/gdal/GridDataMetrics.py
@@ -149,8 +149,8 @@ def icon(self):
     def group(self):
         return self.tr('Raster analysis')
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/GridInverseDistance.py b/python/plugins/processing/algs/gdal/GridInverseDistance.py
@@ -155,8 +155,8 @@ def group(self):
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'gdaltools', 'grid.png'))
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/GridInverseDistanceNearestNeighbor.py b/python/plugins/processing/algs/gdal/GridInverseDistanceNearestNeighbor.py
@@ -142,8 +142,8 @@ def group(self):
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'gdaltools', 'grid.png'))
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/GridLinear.py b/python/plugins/processing/algs/gdal/GridLinear.py
@@ -117,8 +117,8 @@ def group(self):
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'gdaltools', 'grid.png'))
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/GridNearestNeighbor.py b/python/plugins/processing/algs/gdal/GridNearestNeighbor.py
@@ -130,8 +130,8 @@ def group(self):
     def icon(self):
         return QIcon(os.path.join(pluginPath, 'images', 'gdaltools', 'grid.png'))
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
 
         arguments = ['-l']
         arguments.append(layerName)

diff --git a/python/plugins/processing/algs/gdal/OffsetCurve.py b/python/plugins/processing/algs/gdal/OffsetCurve.py
@@ -81,9 +81,9 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         fields = self.parameterAsSource(parameters, self.INPUT, context).fields()
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
         distance = self.parameterAsDouble(parameters, self.DISTANCE, context)
         options = self.parameterAsString(parameters, self.OPTIONS, context)

diff --git a/python/plugins/processing/algs/gdal/OgrToPostGis.py b/python/plugins/processing/algs/gdal/OgrToPostGis.py
@@ -122,7 +122,7 @@ def initAlgorithm(self, config=None):
                                                        self.tr('Maximum distance between 2 nodes (densification)'),
                                                        defaultValue='', optional=True))
         self.addParameter(QgsProcessingParameterExtent(self.SPAT,
-                                                       self.tr('Select features by extent (defined in input layer CRS)')))
+                                                       self.tr('Select features by extent (defined in input layer CRS)'), optional=True))
         self.addParameter(QgsProcessingParameterBoolean(self.CLIP,
                                                         self.tr('Clip the input layer using the above (rectangle) extent'),
                                                         defaultValue=False))
@@ -184,9 +184,8 @@ def getConnectionString(self, parameters, context):
             arguments.append('user=' + user)
         return GdalUtils.escapeAndJoin(arguments)
 
-    def getConsoleCommands(self, parameters, context, feedback):
-        inLayer = self.parameterAsSource(parameters, self.INPUT, context)
-        ogrLayer, layername = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
+        ogrLayer, layername = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         shapeEncoding = self.parameterAsString(parameters, self.SHAPE_ENCODING, context)
         ssrs = self.parameterAsCrs(parameters, self.S_SRS, context).authid()
         tsrs = self.parameterAsCrs(parameters, self.T_SRS, context).authid()
@@ -203,8 +202,6 @@ def getConsoleCommands(self, parameters, context, feedback):
         simplify = self.parameterAsString(parameters, self.SIMPLIFY, context)
         segmentize = self.parameterAsString(parameters, self.SEGMENTIZE, context)
         spat = self.parameterAsExtent(parameters, self.SPAT, context)
-        if spat.isNull():
-            spat = inLayer.sourceExtent()
         clip = self.parameterAsBool(parameters, self.CLIP, context)
         where = self.parameterAsString(parameters, self.WHERE, context)
         wherestring = '-where "' + where + '"'
@@ -235,7 +232,7 @@ def getConsoleCommands(self, parameters, context, feedback):
         arguments.append('"')
         arguments.append(dimstring)
         arguments.append(ogrLayer)
-        arguments.append(GdalUtils.ogrLayerName(inLayer))
+        arguments.append(layername)
         if index:
             arguments.append(indexstring)
         if launder:
@@ -256,7 +253,7 @@ def getConsoleCommands(self, parameters, context, feedback):
         elif primary_key is not None:
             arguments.append("-lco FID=" + primary_key)
         if len(table) == 0:
-            table = GdalUtils.ogrLayerName(inLayer).lower()
+            table = layername.lower()
         if schema:
             table = '{}.{}'.format(schema, table)
         arguments.append('-nln')

diff --git a/python/plugins/processing/algs/gdal/OneSideBuffer.py b/python/plugins/processing/algs/gdal/OneSideBuffer.py
@@ -107,9 +107,9 @@ def group(self):
     def commandName(self):
         return 'ogr2ogr'
 
-    def getConsoleCommands(self, parameters, context, feedback):
+    def getConsoleCommands(self, parameters, context, feedback, executing=True):
         fields = self.parameterAsSource(parameters, self.INPUT, context).fields()
-        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback)
+        ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
         geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
         distance = self.parameterAsDouble(parameters, self.DISTANCE, context)
         side = self.parameterAsEnum(parameters, self.BUFFER_SIDE, context)