[processing] Optimise feature requests within qgis algs

- don't use setFilterFid() within loops to fetch features one at time (as it's extremely slow), instead use setFilterFids() outside the loop - don't fetch unused attributes/geometry when it can be avoided
qgis · Oct 17, 2016 · 86368f3 · 86368f3 · m-kuhn · Oct 17, 2016
1 parent 55f2071
commit 86368f3
Show file tree

Hide file tree

Showing 22 changed files with 43 additions and 54 deletions.
diff --git a/python/plugins/processing/algs/qgis/Difference.py b/python/plugins/processing/algs/qgis/Difference.py
@@ -85,9 +85,9 @@ def processAlgorithm(self, progress):
             diff_geom = QgsGeometry(geom)
             attrs = inFeatA.attributes()
             intersections = index.intersects(geom.boundingBox())
-            for i in intersections:
-                request = QgsFeatureRequest().setFilterFid(i)
-                inFeatB = next(layerB.getFeatures(request))
+
+            request = QgsFeatureRequest().setFilterFids(intersections).setSubsetOfAttributes([])
+            for inFeatB in layerB.getFeatures(request):
                 tmpGeom = inFeatB.geometry()
                 if diff_geom.intersects(tmpGeom):
                     diff_geom = QgsGeometry(diff_geom.difference(tmpGeom))

diff --git a/python/plugins/processing/algs/qgis/Eliminate.py b/python/plugins/processing/algs/qgis/Eliminate.py
@@ -246,7 +246,7 @@ def processAlgorithm(self, progress):
                 geom2Eliminate = feat.geometry()
                 bbox = geom2Eliminate.boundingBox()
                 fit = processLayer.getFeatures(
-                    QgsFeatureRequest().setFilterRect(bbox))
+                    QgsFeatureRequest().setFilterRect(bbox).setSubsetOfAttributes([]))
                 mergeWithFid = None
                 mergeWithGeom = None
                 max = 0

diff --git a/python/plugins/processing/algs/qgis/ExtractByLocation.py b/python/plugins/processing/algs/qgis/ExtractByLocation.py
@@ -84,9 +84,8 @@ def processAlgorithm(self, progress):
             geom = vector.snapToPrecision(f.geometry(), precision)
             bbox = vector.bufferedBoundingBox(geom.boundingBox(), 0.51 * precision)
             intersects = index.intersects(bbox)
-            for i in intersects:
-                request = QgsFeatureRequest().setFilterFid(i)
-                feat = next(layer.getFeatures(request))
+            request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
+            for feat in layer.getFeatures(request):
                 tmpGeom = vector.snapToPrecision(feat.geometry(), precision)
                 res = False
                 for predicate in predicates:

diff --git a/python/plugins/processing/algs/qgis/HubDistanceLines.py b/python/plugins/processing/algs/qgis/HubDistanceLines.py
@@ -109,7 +109,7 @@ def processAlgorithm(self, progress):
             src = f.geometry().boundingBox().center()
 
             neighbors = index.nearestNeighbor(src, 1)
-            ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0])))
+            ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields())))
             closest = ft.geometry().boundingBox().center()
             hubDist = distance.measureLine(src, closest)
 

diff --git a/python/plugins/processing/algs/qgis/HubDistancePoints.py b/python/plugins/processing/algs/qgis/HubDistancePoints.py
@@ -109,7 +109,7 @@ def processAlgorithm(self, progress):
             src = f.geometry().boundingBox().center()
 
             neighbors = index.nearestNeighbor(src, 1)
-            ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0])))
+            ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields())))
             closest = ft.geometry().boundingBox().center()
             hubDist = distance.measureLine(src, closest)
 

diff --git a/python/plugins/processing/algs/qgis/Intersection.py b/python/plugins/processing/algs/qgis/Intersection.py
@@ -88,9 +88,8 @@ def processAlgorithm(self, progress):
             geom = inFeatA.geometry()
             atMapA = inFeatA.attributes()
             intersects = index.intersects(geom.boundingBox())
-            for i in intersects:
-                request = QgsFeatureRequest().setFilterFid(i)
-                inFeatB = next(vlayerB.getFeatures(request))
+            request = QgsFeatureRequest().setFilterFids(intersects)
+            for inFeatB in vlayerB.getFeatures(request):
                 tmpGeom = inFeatB.geometry()
                 if geom.intersects(tmpGeom):
                     atMapB = inFeatB.attributes()

diff --git a/python/plugins/processing/algs/qgis/LinesIntersection.py b/python/plugins/processing/algs/qgis/LinesIntersection.py
@@ -105,9 +105,8 @@ def processAlgorithm(self, progress):
                 hasIntersections = True
 
             if hasIntersections:
-                for i in lines:
-                    request = QgsFeatureRequest().setFilterFid(i)
-                    inFeatB = next(layerB.getFeatures(request))
+                request = QgsFeatureRequest().setFilterFids(lines)
+                for inFeatB in layerB.getFeatures(request):
                     tmpGeom = inFeatB.geometry()
 
                     points = []

diff --git a/python/plugins/processing/algs/qgis/NearestNeighbourAnalysis.py b/python/plugins/processing/algs/qgis/NearestNeighbourAnalysis.py
@@ -95,7 +95,7 @@ def processAlgorithm(self, progress):
         for current, feat in enumerate(features):
             neighbourID = spatialIndex.nearestNeighbor(
                 feat.geometry().asPoint(), 2)[1]
-            request = QgsFeatureRequest().setFilterFid(neighbourID)
+            request = QgsFeatureRequest().setFilterFid(neighbourID).setSubsetOfAttributes([])
             neighbour = next(layer.getFeatures(request))
             sumDist += distance.measureLine(neighbour.geometry().asPoint(),
                                             feat.geometry().asPoint())

diff --git a/python/plugins/processing/algs/qgis/PointDistance.py b/python/plugins/processing/algs/qgis/PointDistance.py
@@ -136,9 +136,8 @@ def linearMatrix(self, inLayer, inField, targetLayer, targetField,
             featList = index.nearestNeighbor(inGeom.asPoint(), nPoints)
             distList = []
             vari = 0.0
-            for i in featList:
-                request = QgsFeatureRequest().setFilterFid(i)
-                outFeat = next(targetLayer.getFeatures(request))
+            request = QgsFeatureRequest().setFilterFids(featList).setSubsetOfAttributes([outIdx])
+            for outFeat in targetLayer.getFeatures(request):
                 outID = outFeat.attributes()[outIdx]
                 outGeom = outFeat.geometry()
                 dist = distArea.measureLine(inGeom.asPoint(),

diff --git a/python/plugins/processing/algs/qgis/PointsDisplacement.py b/python/plugins/processing/algs/qgis/PointsDisplacement.py
@@ -86,11 +86,10 @@ def processAlgorithm(self, progress):
 
         fullPerimeter = 2 * math.pi
 
-        request = QgsFeatureRequest()
         for (geom, fids) in list(duplicates.items()):
             count = len(fids)
             if count == 1:
-                f = next(layer.getFeatures(request.setFilterFid(fids[0])))
+                f = next(layer.getFeatures(QgsFeatureRequest().setFilterFid(fids[0])))
                 writer.addFeature(f)
             else:
                 angleStep = fullPerimeter / count
@@ -100,14 +99,14 @@ def processAlgorithm(self, progress):
                     currentAngle = 0
 
                 old_point = QgsGeometry.fromWkt(geom).asPoint()
-                for fid in fids:
+
+                request = QgsFeatureRequest().setFilterFids(fids).setFlags(QgsFeatureRequest.NoGeometry)
+                for f in layer.getFeatures(request):
                     sinusCurrentAngle = math.sin(currentAngle)
                     cosinusCurrentAngle = math.cos(currentAngle)
                     dx = radius * sinusCurrentAngle
                     dy = radius * cosinusCurrentAngle
 
-                    f = next(layer.getFeatures(request.setFilterFid(fid)))
-
                     new_point = QgsPoint(old_point.x() + dx, old_point.y()
                                          + dy)
                     out_feature = QgsFeature()

diff --git a/python/plugins/processing/algs/qgis/PointsInPolygon.py b/python/plugins/processing/algs/qgis/PointsInPolygon.py
@@ -96,7 +96,7 @@ def processAlgorithm(self, progress):
             count = 0
             points = spatialIndex.intersects(geom.boundingBox())
             if len(points) > 0:
-                request = QgsFeatureRequest().setFilterFids(points)
+                request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([])
                 fit = pointLayer.getFeatures(request)
                 ftPoint = QgsFeature()
                 while fit.nextFeature(ftPoint):

diff --git a/python/plugins/processing/algs/qgis/PointsInPolygonUnique.py b/python/plugins/processing/algs/qgis/PointsInPolygonUnique.py
@@ -90,7 +90,7 @@ def processAlgorithm(self, progress):
             classes = set()
             points = spatialIndex.intersects(geom.boundingBox())
             if len(points) > 0:
-                request = QgsFeatureRequest().setFilterFids(points)
+                request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([classFieldIndex])
                 fit = pointLayer.getFeatures(request)
                 ftPoint = QgsFeature()
                 while fit.nextFeature(ftPoint):

diff --git a/python/plugins/processing/algs/qgis/PointsInPolygonWeighted.py b/python/plugins/processing/algs/qgis/PointsInPolygonWeighted.py
@@ -98,7 +98,7 @@ def processAlgorithm(self, progress):
             points = spatialIndex.intersects(geom.boundingBox())
             if len(points) > 0:
                 progress.setText(str(len(points)))
-                request = QgsFeatureRequest().setFilterFids(points)
+                request = QgsFeatureRequest().setFilterFids(points).setSubsetOfAttributes([fieldIdx])
                 fit = pointLayer.getFeatures(request)
                 ftPoint = QgsFeature()
                 while fit.nextFeature(ftPoint):

diff --git a/python/plugins/processing/algs/qgis/RandomPointsAlongLines.py b/python/plugins/processing/algs/qgis/RandomPointsAlongLines.py
@@ -88,7 +88,7 @@ def processAlgorithm(self, progress):
         while nIterations < maxIterations and nPoints < pointCount:
             # pick random feature
             fid = random.randint(0, featureCount - 1)
-            f = next(layer.getFeatures(request.setFilterFid(fid)))
+            f = next(layer.getFeatures(request.setFilterFid(fid).setSubsetOfAttributes([])))
             fGeom = f.geometry()
 
             if fGeom.isMultipart():

diff --git a/python/plugins/processing/algs/qgis/RandomPointsLayer.py b/python/plugins/processing/algs/qgis/RandomPointsLayer.py
@@ -88,8 +88,6 @@ def processAlgorithm(self, progress):
         index = QgsSpatialIndex()
         points = dict()
 
-        request = QgsFeatureRequest()
-
         random.seed()
 
         while nIterations < maxIterations and nPoints < pointCount:
@@ -101,8 +99,8 @@ def processAlgorithm(self, progress):
             ids = idxLayer.intersects(geom.buffer(5, 5).boundingBox())
             if len(ids) > 0 and \
                     vector.checkMinDistance(pnt, index, minDistance, points):
-                for i in ids:
-                    f = next(layer.getFeatures(request.setFilterFid(i)))
+                request = QgsFeatureRequest().setFilterFids(ids).setSubsetOfAttributes([])
+                for f in layer.getFeatures(request):
                     tmpGeom = f.geometry()
                     if geom.within(tmpGeom):
                         f = QgsFeature(nPoints)

diff --git a/python/plugins/processing/algs/qgis/SelectByAttribute.py b/python/plugins/processing/algs/qgis/SelectByAttribute.py
@@ -111,7 +111,7 @@ def processAlgorithm(self, progress):
 
         qExp = QgsExpression(expr)
         if not qExp.hasParserError():
-            qReq = QgsFeatureRequest(qExp)
+            qReq = QgsFeatureRequest(qExp).setSubsetOfAttributes([])
         else:
             raise GeoAlgorithmExecutionException(qExp.parserErrorString())
         selected = [f.id() for f in layer.getFeatures(qReq)]

diff --git a/python/plugins/processing/algs/qgis/SelectByAttributeSum.py b/python/plugins/processing/algs/qgis/SelectByAttributeSum.py
@@ -82,8 +82,8 @@ def processAlgorithm(self, progress):
                 progress.setInfo(self.tr('No adjacent features found.'))
                 break
 
-            for i in intersected:
-                ft = next(layer.getFeatures(req.setFilterFid(i)))
+            req = QgsFeatureRequest().setFilterFids(intersected).setSubsetOfAttributes([fieldName], layer.fields())
+            for ft in layer.getFeatures(req):
                 tmpGeom = ft.geometry()
                 if tmpGeom.touches(geom):
                     geom = tmpGeom.combine(geom)

diff --git a/python/plugins/processing/algs/qgis/SelectByLocation.py b/python/plugins/processing/algs/qgis/SelectByLocation.py
@@ -105,9 +105,8 @@ def processAlgorithm(self, progress):
             bbox = vector.bufferedBoundingBox(geom.boundingBox(), 0.51 * precision)
             intersects = index.intersects(bbox)
 
-            for i in intersects:
-                request = QgsFeatureRequest().setFilterFid(i)
-                feat = next(inputLayer.getFeatures(request))
+            request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
+            for feat in inputLayer.getFeatures(request):
                 tmpGeom = vector.snapToPrecision(feat.geometry(), precision)
 
                 res = False

diff --git a/python/plugins/processing/algs/qgis/SplitLinesWithLines.py b/python/plugins/processing/algs/qgis/SplitLinesWithLines.py
@@ -79,9 +79,8 @@ def processAlgorithm(self, progress):
             if len(lines) > 0:  # hasIntersections
                 splittingLines = []
 
-                for i in lines:
-                    request = QgsFeatureRequest().setFilterFid(i)
-                    inFeatB = next(layerB.getFeatures(request))
+                request = QgsFeatureRequest().setFilterFids(lines).setSubsetOfAttributes([])
+                for inFeatB in layerB.getFeatures(request):
                     # check if trying to self-intersect
                     if sameLayer:
                         if inFeatA.id() == inFeatB.id():

diff --git a/python/plugins/processing/algs/qgis/SumLines.py b/python/plugins/processing/algs/qgis/SumLines.py
@@ -104,9 +104,8 @@ def processAlgorithm(self, progress):
                 hasIntersections = True
 
             if hasIntersections:
-                for i in lines:
-                    request = QgsFeatureRequest().setFilterFid(i)
-                    ftLine = next(lineLayer.getFeatures(request))
+                request = QgsFeatureRequest().setFilterFids(lines).setSubsetOfAttributes([])
+                for ftLine in lineLayer.getFeatures(request):
                     tmpGeom = ftLine.geometry()
                     if inGeom.intersects(tmpGeom):
                         outGeom = inGeom.intersection(tmpGeom)

diff --git a/python/plugins/processing/algs/qgis/SymmetricalDifference.py b/python/plugins/processing/algs/qgis/SymmetricalDifference.py
@@ -88,8 +88,8 @@ def processAlgorithm(self, progress):
             diffGeom = QgsGeometry(geom)
             attrs = featA.attributes()
             intersects = indexA.intersects(geom.boundingBox())
-            for i in intersects:
-                layerB.getFeatures(QgsFeatureRequest().setFilterFid(i)).nextFeature(featB)
+            request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
+            for featB in layerB.getFeatures(request):
                 tmpGeom = featB.geometry()
                 if diffGeom.intersects(tmpGeom):
                     diffGeom = QgsGeometry(diffGeom.difference(tmpGeom))
@@ -123,8 +123,8 @@ def processAlgorithm(self, progress):
             attrs = featA.attributes()
             attrs = [NULL] * length + attrs
             intersects = indexB.intersects(geom.boundingBox())
-            for i in intersects:
-                layerA.getFeatures(QgsFeatureRequest().setFilterFid(i)).nextFeature(featB)
+            request = QgsFeatureRequest().setFilterFids(intersects).setSubsetOfAttributes([])
+            for featB in layerA.getFeatures(request):
                 tmpGeom = featB.geometry()
                 if diffGeom.intersects(tmpGeom):
                     diffGeom = QgsGeometry(diffGeom.difference(tmpGeom))

diff --git a/python/plugins/processing/algs/qgis/Union.py b/python/plugins/processing/algs/qgis/Union.py
@@ -105,10 +105,10 @@ def processAlgorithm(self, progress):
                     ProcessingLog.addToLog(ProcessingLog.LOG_INFO,
                                            self.tr('Feature geometry error: One or more output features ignored due to invalid geometry.'))
             else:
-                for id in intersects:
+                request = QgsFeatureRequest().setFilterFids(intersects)
+                for inFeatB in vlayerB.getFeatures(request):
                     count += 1
-                    request = QgsFeatureRequest().setFilterFid(id)
-                    inFeatB = next(vlayerB.getFeatures(request))
+
                     atMapB = inFeatB.attributes()
                     tmpGeom = inFeatB.geometry()
 
@@ -197,9 +197,8 @@ def processAlgorithm(self, progress):
                     ProcessingLog.addToLog(ProcessingLog.LOG_INFO,
                                            self.tr('Feature geometry error: One or more output features ignored due to invalid geometry.'))
             else:
-                for id in intersects:
-                    request = QgsFeatureRequest().setFilterFid(id)
-                    inFeatB = next(vlayerA.getFeatures(request))
+                request = QgsFeatureRequest().setFilterFids(intersects)
+                for inFeatB in vlayerA.getFeatures(request):
                     atMapB = inFeatB.attributes()
                     tmpGeom = inFeatB.geometry()