Skip to content

Commit

Permalink
Refactor processing writer to use QgsFeatureSink, and to store
Browse files Browse the repository at this point in the history 
temporary output layers in the provided processing context

Should fix loss of intermediate memory layers when running
a multi-step model
  • Loading branch information
@nyalldawson
nyalldawson committed Apr 26, 2017
1 parent 946f0fa commit fd2c18e
Show file tree
Hide file tree
Showing 123 changed files with 327 additions and 478 deletions.
8 changes: 6 additions & 2 deletions python/core/processing/qgsprocessingcontext.sip
View file
Expand Up @@ -75,8 +75,12 @@ class QgsProcessingContext
Sets the expression ``context``.
%End



QgsProject &temporaryLayerStore();
%Docstring
Returns a reference to the project used for storing temporary layers during
algorithm execution.
:rtype: QgsProject
%End

QgsFeatureRequest::InvalidGeometryCheck invalidGeometryCheck() const;
%Docstring
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/AddTableField.py
View file
Expand Up @@ -97,8 +97,7 @@ def processAlgorithm(self, context, feedback):
fields = layer.fields()
fields.append(QgsField(fieldName, self.TYPES[fieldType], '',
fieldLength, fieldPrecision))
writer = output.getVectorWriter(fields, layer.wkbType(),
layer.crs())
writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/AutoincrementalField.py
View file
Expand Up @@ -67,8 +67,7 @@ def processAlgorithm(self, context, feedback):
dataobjects.getLayerFromString(self.getParameterValue(self.INPUT))
fields = vlayer.fields()
fields.append(QgsField('AUTO', QVariant.Int))
writer = output.getVectorWriter(fields, vlayer.wkbType(),
vlayer.crs())
writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(vlayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vlayer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Boundary.py
View file
Expand Up @@ -78,10 +78,7 @@ def processAlgorithm(self, context, feedback):
output_wkb = QgsWkbTypes.addM(output_wkb)

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields(),
output_wkb,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields(), output_wkb, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/BoundingBox.py
View file
Expand Up @@ -67,10 +67,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT_LAYER))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields().toList(),
QgsWkbTypes.Polygon,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Centroids.py
View file
Expand Up @@ -69,10 +69,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT_LAYER))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields(),
QgsWkbTypes.Point,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields(), QgsWkbTypes.Point, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
15 changes: 3 additions & 12 deletions python/plugins/processing/algs/qgis/CheckValidity.py
View file
Expand Up @@ -108,32 +108,23 @@ def doCheck(self, context, feedback):

valid_output = self.getOutputFromName(self.VALID_OUTPUT)
valid_fields = layer.fields()
valid_writer = valid_output.getVectorWriter(
valid_fields,
layer.wkbType(),
layer.crs())
valid_writer = valid_output.getVectorWriter(valid_fields, layer.wkbType(), layer.crs(), context)
valid_count = 0

invalid_output = self.getOutputFromName(self.INVALID_OUTPUT)
invalid_fields = layer.fields().toList() + [
QgsField(name='_errors',
type=QVariant.String,
len=255)]
invalid_writer = invalid_output.getVectorWriter(
invalid_fields,
layer.wkbType(),
layer.crs())
invalid_writer = invalid_output.getVectorWriter(invalid_fields, layer.wkbType(), layer.crs(), context)
invalid_count = 0

error_output = self.getOutputFromName(self.ERROR_OUTPUT)
error_fields = [
QgsField(name='message',
type=QVariant.String,
len=255)]
error_writer = error_output.getVectorWriter(
error_fields,
QgsWkbTypes.Point,
layer.crs())
error_writer = error_output.getVectorWriter(error_fields, QgsWkbTypes.Point, layer.crs(), context)
error_count = 0

features = QgsProcessingUtils.getFeatures(layer, context)
Expand Down
7 changes: 3 additions & 4 deletions python/plugins/processing/algs/qgis/Clip.py
View file
Expand Up @@ -75,10 +75,9 @@ def processAlgorithm(self, context, feedback):
mask_layer = dataobjects.getLayerFromString(
self.getParameterValue(Clip.OVERLAY))

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
source_layer.fields(),
QgsWkbTypes.multiType(source_layer.wkbType()),
source_layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(source_layer.fields(),
QgsWkbTypes.multiType(source_layer.wkbType()),
source_layer.crs(), context)

# first build up a list of clip geometries
clip_geoms = []
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/qgis/ConcaveHull.py
View file
Expand Up @@ -133,8 +133,8 @@ def processAlgorithm(self, context, feedback):
feedback.setProgressText(self.tr('Saving data...'))
feat = QgsFeature()
dissolved_layer.getFeatures(QgsFeatureRequest().setFilterFid(0)).nextFeature(feat)
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields().toList(), QgsWkbTypes.Polygon,
layer.crs(), context)
geom = feat.geometry()
if no_multigeom and geom.isMultipart():
# Only singlepart geometries are allowed
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ConvexHull.py
View file
Expand Up @@ -103,8 +103,7 @@ def processAlgorithm(self, context, feedback):
QgsField('perim', QVariant.Double, '', 20, 6)
]

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
fields, QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)

outFeat = QgsFeature()
inGeom = QgsGeometry()
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/Delaunay.py
View file
Expand Up @@ -77,8 +77,7 @@ def processAlgorithm(self, context, feedback):
QgsField('POINTB', QVariant.Double, '', 24, 15),
QgsField('POINTC', QVariant.Double, '', 24, 15)]

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)

pts = []
ptDict = {}
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DeleteColumn.py
View file
Expand Up @@ -83,8 +83,7 @@ def processAlgorithm(self, context, feedback):
for index in field_indices:
fields.remove(index)

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DeleteDuplicateGeometries.py
View file
Expand Up @@ -65,8 +65,7 @@ def processAlgorithm(self, context, feedback):

fields = layer.fields()

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)

Expand Down
6 changes: 2 additions & 4 deletions python/plugins/processing/algs/qgis/DeleteHoles.py
View file
Expand Up @@ -77,10 +77,8 @@ def processAlgorithm(self, context, feedback):
if min_area == 0.0:
min_area = -1.0

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
layer.fields(),
layer.wkbType(),
layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DensifyGeometries.py
View file
Expand Up @@ -85,8 +85,7 @@ def processAlgorithm(self, context, feedback):
isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry

writer = self.getOutputFromName(
self.OUTPUT).getVectorWriter(layer.fields().toList(),
layer.wkbType(), layer.crs())
self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/DensifyGeometriesInterval.py
View file
Expand Up @@ -79,8 +79,7 @@ def processAlgorithm(self, context, feedback):
isPolygon = layer.geometryType() == QgsWkbTypes.PolygonGeometry

writer = self.getOutputFromName(
self.OUTPUT).getVectorWriter(layer.fields().toList(),
layer.wkbType(), layer.crs())
self.OUTPUT).getVectorWriter(layer.fields().toList(), layer.wkbType(), layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
4 changes: 1 addition & 3 deletions python/plugins/processing/algs/qgis/Difference.py
View file
Expand Up @@ -77,9 +77,7 @@ def processAlgorithm(self, context, feedback):

geomType = QgsWkbTypes.multiType(layerA.wkbType())
writer = self.getOutputFromName(
Difference.OUTPUT).getVectorWriter(layerA.fields(),
geomType,
layerA.crs())
Difference.OUTPUT).getVectorWriter(layerA.fields(), geomType, layerA.crs(), context)

outFeat = QgsFeature()
index = vector.spatialindex(layerB)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/Dissolve.py
View file
Expand Up @@ -82,10 +82,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(Dissolve.INPUT))

writer = self.getOutputFromName(
Dissolve.OUTPUT).getVectorWriter(
vlayerA.fields().toList(),
vlayerA.wkbType(),
vlayerA.crs())
Dissolve.OUTPUT).getVectorWriter(vlayerA.fields().toList(), vlayerA.wkbType(), vlayerA.crs(), context)

outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(vlayerA, context)
Expand Down
6 changes: 2 additions & 4 deletions python/plugins/processing/algs/qgis/DropGeometry.py
View file
Expand Up @@ -70,10 +70,8 @@ def processAlgorithm(self, context, feedback):
layer = dataobjects.getLayerFromString(
self.getParameterValue(self.INPUT_LAYER))
writer = self.getOutputFromName(
self.OUTPUT_TABLE).getVectorWriter(
layer.fields(),
QgsWkbTypes.NoGeometry,
QgsCoordinateReferenceSystem())
self.OUTPUT_TABLE).getVectorWriter(layer.fields(), QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem(),
context)

request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)
features = QgsProcessingUtils.getFeatures(layer, context, request)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/EliminateSelection.py
View file
Expand Up @@ -92,8 +92,7 @@ def processAlgorithm(self, context, feedback):
featToEliminate = []
selFeatIds = inLayer.selectedFeatureIds()
output = self.getOutputFromName(self.OUTPUT)
writer = output.getVectorWriter(inLayer.fields(),
inLayer.wkbType(), inLayer.crs())
writer = output.getVectorWriter(inLayer.fields(), inLayer.wkbType(), inLayer.crs(), context)

for aFeat in inLayer.getFeatures():
if aFeat.id() in selFeatIds:
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/EquivalentNumField.py
View file
Expand Up @@ -73,8 +73,7 @@ def processAlgorithm(self, context, feedback):
fieldindex = vlayer.fields().lookupField(fieldname)
fields = vlayer.fields()
fields.append(QgsField('NUM_FIELD', QVariant.Int))
writer = output.getVectorWriter(fields, vlayer.wkbType(),
vlayer.crs())
writer = output.getVectorWriter(fields, vlayer.wkbType(), vlayer.crs(), context)
outFeat = QgsFeature()
classes = {}

Expand Down
8 changes: 3 additions & 5 deletions python/plugins/processing/algs/qgis/ExecuteSQL.py
View file
Expand Up @@ -146,11 +146,9 @@ def processAlgorithm(self, context, feedback):
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(vLayer.dataProvider().error().message())

writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
vLayer.fields().toList(),
# Create a point layer (without any points) if 'no geometry' is chosen
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(vLayer.fields().toList(),
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs(), context)

features = QgsProcessingUtils.getFeatures(vLayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vLayer, context)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/Explode.py
View file
Expand Up @@ -68,8 +68,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT))
output = self.getOutputFromName(self.OUTPUT)
fields = vlayer.fields()
writer = output.getVectorWriter(fields, QgsWkbTypes.LineString,
vlayer.crs())
writer = output.getVectorWriter(fields, QgsWkbTypes.LineString, vlayer.crs(), context)
outFeat = QgsFeature()
features = QgsProcessingUtils.getFeatures(vlayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vlayer, context)
Expand Down
4 changes: 2 additions & 2 deletions python/plugins/processing/algs/qgis/ExportGeometryInfo.py
View file
Expand Up @@ -107,8 +107,8 @@ def processAlgorithm(self, context, feedback):
zName = vector.createUniqueFieldName('mvalue', fields)
fields.append(QgsField(zName, QVariant.Double))

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
fields.toList(), layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields.toList(), layer.wkbType(), layer.crs(),
context)

ellips = None
crs = None
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/ExtendLines.py
View file
Expand Up @@ -71,10 +71,7 @@ def processAlgorithm(self, context, feedback):
self.getParameterValue(self.INPUT_LAYER))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
layer.fields(),
layer.wkbType(),
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(), context)

start_distance = self.getParameterValue(self.START_DISTANCE)
end_distance = self.getParameterValue(self.END_DISTANCE)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ExtentFromLayer.py
View file
Expand Up @@ -89,8 +89,7 @@ def processAlgorithm(self, context, feedback):
QgsField('WIDTH', QVariant.Double),
]

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
QgsWkbTypes.Polygon, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Polygon, layer.crs(), context)

if byFeature:
self.featureExtent(layer, context, writer, feedback)
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ExtractByAttribute.py
View file
Expand Up @@ -111,8 +111,7 @@ def processAlgorithm(self, context, feedback):
value = self.getParameterValue(self.VALUE)

fields = layer.fields()
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields,
layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

idx = layer.fields().lookupField(fieldName)
fieldType = fields[idx].type()
Expand Down
3 changes: 2 additions & 1 deletion python/plugins/processing/algs/qgis/ExtractByExpression.py
View file
Expand Up @@ -70,7 +70,8 @@ def defineCharacteristics(self):
def processAlgorithm(self, context, feedback):
layer = dataobjects.getLayerFromString(self.getParameterValue(self.INPUT))
expression_string = self.getParameterValue(self.EXPRESSION)
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(),
context)

expression = QgsExpression(expression_string)
if not expression.hasParserError():
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ExtractByLocation.py
View file
Expand Up @@ -97,8 +97,7 @@ def processAlgorithm(self, context, feedback):
index = vector.spatialindex(layer)

output = self.getOutputFromName(self.OUTPUT)
writer = output.getVectorWriter(layer.fields(),
layer.wkbType(), layer.crs())
writer = output.getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(), context)

if 'disjoint' in predicates:
disjoinSet = []
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/ExtractNodes.py
View file
Expand Up @@ -75,8 +75,7 @@ def processAlgorithm(self, context, feedback):
fields.append(QgsField('distance', QVariant.Double))
fields.append(QgsField('angle', QVariant.Double))

writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
fields, QgsWkbTypes.Point, layer.crs())
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Point, layer.crs(), context)

features = QgsProcessingUtils.getFeatures(layer, context)
total = 100.0 / QgsProcessingUtils.featureCount(layer, context)
Expand Down
5 changes: 1 addition & 4 deletions python/plugins/processing/algs/qgis/ExtractSpecificNodes.py
View file
Expand Up @@ -80,10 +80,7 @@ def processAlgorithm(self, context, feedback):
fields.append(QgsField('angle', QVariant.Double))

writer = self.getOutputFromName(
self.OUTPUT_LAYER).getVectorWriter(
fields,
QgsWkbTypes.Point,
layer.crs())
self.OUTPUT_LAYER).getVectorWriter(fields, QgsWkbTypes.Point, layer.crs(), context)

node_indices_string = self.getParameterValue(self.NODES)
indices = []
Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/FieldPyculator.py
View file
Expand Up @@ -107,8 +107,7 @@ def processAlgorithm(self, context, feedback):
fields = layer.fields()
fields.append(QgsField(fieldName, self.TYPES[fieldType], '',
fieldLength, fieldPrecision))
writer = output.getVectorWriter(fields, layer.wkbType(),
layer.crs())
writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
outFeat = QgsFeature()
new_ns = {}

Expand Down
3 changes: 1 addition & 2 deletions python/plugins/processing/algs/qgis/FieldsCalculator.py
View file
Expand Up @@ -113,8 +113,7 @@ def processAlgorithm(self, context, feedback):
if newField:
fields.append(QgsField(fieldName, fieldType, '', width, precision))

writer = output.getVectorWriter(fields, layer.wkbType(),
layer.crs())
writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)

exp = QgsExpression(formula)

Expand Down

0 comments on commit fd2c18e

Please sign in to comment.