[processing] Hookup backend API to allow sinks to append to an existi…

…ng layer (respecting the sink mapping definition)
qgis · Apr 7, 2020 · bf6f017 · bf6f017
1 parent 0d7773a
commit bf6f017
Show file tree

Hide file tree

Showing 7 changed files with 117 additions and 20 deletions.
diff --git a/python/core/auto_generated/qgsremappingproxyfeaturesink.sip.in b/python/core/auto_generated/qgsremappingproxyfeaturesink.sip.in
@@ -162,12 +162,15 @@ and by coercing geometries to the format required by the destination sink.
 %End
   public:
 
+
     QgsRemappingProxyFeatureSink( const QgsRemappingSinkDefinition &mappingDefinition, QgsFeatureSink *sink );
 %Docstring
 Constructor for QgsRemappingProxyFeatureSink, using the specified ``mappingDefinition``
 to manipulate features before sending them to the destination ``sink``.
 %End
 
+    ~QgsRemappingProxyFeatureSink();
+
     void setExpressionContext( const QgsExpressionContext &context );
 %Docstring
 Sets the expression ``context`` to use when evaluating mapped field values.

diff --git a/src/core/processing/qgsprocessingparameters.cpp b/src/core/processing/qgsprocessingparameters.cpp
@@ -579,6 +579,8 @@ QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingPar
   QgsProject *destinationProject = nullptr;
   QString destName;
   QVariantMap createOptions;
+  QgsRemappingSinkDefinition remapDefinition;
+  bool useRemapDefinition = false;
   if ( val.canConvert<QgsProcessingOutputLayerDefinition>() )
   {
     // input is a QgsProcessingOutputLayerDefinition - get extra properties from it
@@ -588,6 +590,11 @@ QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingPar
 
     val = fromVar.sink;
     destName = fromVar.destinationName;
+    if ( fromVar.useRemapping() )
+    {
+      useRemapDefinition = true;
+      remapDefinition = fromVar.remappingDefinition();
+    }
   }
 
   QString dest;
@@ -622,7 +629,7 @@ QgsFeatureSink *QgsProcessingParameters::parameterAsSink( const QgsProcessingPar
   if ( dest.isEmpty() )
     return nullptr;
 
-  std::unique_ptr< QgsFeatureSink > sink( QgsProcessingUtils::createFeatureSink( dest, context, fields, geometryType, crs, createOptions, sinkFlags ) );
+  std::unique_ptr< QgsFeatureSink > sink( QgsProcessingUtils::createFeatureSink( dest, context, fields, geometryType, crs, createOptions, sinkFlags, useRemapDefinition ? &remapDefinition : nullptr ) );
   destinationIdentifier = dest;
 
   if ( destinationProject )

diff --git a/src/core/processing/qgsprocessingutils.cpp b/src/core/processing/qgsprocessingutils.cpp
@@ -675,7 +675,7 @@ void QgsProcessingUtils::parseDestinationString( QString &destination, QString &
   }
 }
 
-QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, QgsProcessingContext &context, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, const QVariantMap &createOptions, QgsFeatureSink::SinkFlags sinkFlags )
+QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, QgsProcessingContext &context, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, const QVariantMap &createOptions, QgsFeatureSink::SinkFlags sinkFlags, QgsRemappingSinkDefinition *remappingDefinition )
 {
   QVariantMap options = createOptions;
   if ( !options.contains( QStringLiteral( "fileEncoding" ) ) )
@@ -719,7 +719,7 @@ QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, Qgs
     bool useWriter = false;
     parseDestinationString( destination, providerKey, uri, layerName, format, options, useWriter, extension );
 
-    QgsFields newFields = fields;
+    QgsFields newFields = remappingDefinition ? remappingDefinition->destinationFields() : fields;
     if ( useWriter && providerKey == QLatin1String( "ogr" ) )
     {
       // use QgsVectorFileWriter for OGR destinations instead of QgsVectorLayerImport, as that allows
@@ -731,34 +731,61 @@ QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, Qgs
       saveOptions.datasourceOptions = QgsVectorFileWriter::defaultDatasetOptions( format );
       saveOptions.layerOptions = QgsVectorFileWriter::defaultLayerOptions( format );
       saveOptions.symbologyExport = QgsVectorFileWriter::NoSymbology;
-      saveOptions.actionOnExistingFile = QgsVectorFileWriter::CreateOrOverwriteFile;
+      saveOptions.actionOnExistingFile = remappingDefinition ? QgsVectorFileWriter::AppendToLayerNoNewFields : QgsVectorFileWriter::CreateOrOverwriteFile;
       std::unique_ptr< QgsVectorFileWriter > writer( QgsVectorFileWriter::create( destination, newFields, geometryType, crs, context.transformContext(), saveOptions, sinkFlags, &finalFileName ) );
       if ( writer->hasError() )
       {
         throw QgsProcessingException( QObject::tr( "Could not create layer %1: %2" ).arg( destination, writer->errorMessage() ) );
       }
       destination = finalFileName;
-      return new QgsProcessingFeatureSink( writer.release(), destination, context, true );
+      if ( remappingDefinition )
+      {
+        std::unique_ptr< QgsRemappingProxyFeatureSink > remapSink = qgis::make_unique< QgsRemappingProxyFeatureSink >( *remappingDefinition, writer.release(), true );
+        remapSink->setExpressionContext( context.expressionContext() );
+        remapSink->setTransformContext( context.transformContext() );
+        return new QgsProcessingFeatureSink( remapSink.release(), destination, context, true );
+      }
+      else
+        return new QgsProcessingFeatureSink( writer.release(), destination, context, true );
     }
     else
     {
-      //create empty layer
       const QgsVectorLayer::LayerOptions layerOptions { context.transformContext() };
-      std::unique_ptr< QgsVectorLayerExporter > exporter = qgis::make_unique<QgsVectorLayerExporter>( uri, providerKey, newFields, geometryType, crs, true, options, sinkFlags );
-      if ( exporter->errorCode() )
+      if ( remappingDefinition )
       {
-        throw QgsProcessingException( QObject::tr( "Could not create layer %1: %2" ).arg( destination, exporter->errorMessage() ) );
+        //write to existing layer
+
+        // use destination string as layer name (eg "postgis:..." )
+        std::unique_ptr< QgsVectorLayer > layer = qgis::make_unique<QgsVectorLayer>( uri, destination, providerKey, layerOptions );
+        // update destination to layer ID
+        destination = layer->id();
+
+        context.temporaryLayerStore()->addMapLayer( layer.release() );
+
+        std::unique_ptr< QgsRemappingProxyFeatureSink > remapSink = qgis::make_unique< QgsRemappingProxyFeatureSink >( *remappingDefinition, layer->dataProvider(), false );
+        remapSink->setExpressionContext( context.expressionContext() );
+        remapSink->setTransformContext( context.transformContext() );
+        return new QgsProcessingFeatureSink( remapSink.release(), destination, context, true );
       }
+      else
+      {
+        //create empty layer
+        std::unique_ptr< QgsVectorLayerExporter > exporter = qgis::make_unique<QgsVectorLayerExporter>( uri, providerKey, newFields, geometryType, crs, true, options, sinkFlags );
+        if ( exporter->errorCode() )
+        {
+          throw QgsProcessingException( QObject::tr( "Could not create layer %1: %2" ).arg( destination, exporter->errorMessage() ) );
+        }
 
-      // use destination string as layer name (eg "postgis:..." )
-      if ( !layerName.isEmpty() )
-        uri += QStringLiteral( "|layername=%1" ).arg( layerName );
-      std::unique_ptr< QgsVectorLayer > layer = qgis::make_unique<QgsVectorLayer>( uri, destination, providerKey, layerOptions );
-      // update destination to layer ID
-      destination = layer->id();
+        // use destination string as layer name (eg "postgis:..." )
+        if ( !layerName.isEmpty() )
+          uri += QStringLiteral( "|layername=%1" ).arg( layerName );
+        std::unique_ptr< QgsVectorLayer > layer = qgis::make_unique<QgsVectorLayer>( uri, destination, providerKey, layerOptions );
+        // update destination to layer ID
+        destination = layer->id();
 
-      context.temporaryLayerStore()->addMapLayer( layer.release() );
-      return new QgsProcessingFeatureSink( exporter.release(), destination, context, true );
+        context.temporaryLayerStore()->addMapLayer( layer.release() );
+        return new QgsProcessingFeatureSink( exporter.release(), destination, context, true );
+      }
     }
   }
 }

diff --git a/src/core/processing/qgsprocessingutils.h b/src/core/processing/qgsprocessingutils.h
@@ -27,6 +27,7 @@
 #include "qgsfeaturesink.h"
 #include "qgsfeaturesource.h"
 #include "qgsproxyfeaturesink.h"
+#include "qgsremappingproxyfeaturesink.h"
 
 class QgsMeshLayer;
 class QgsProject;
@@ -222,7 +223,8 @@ class CORE_EXPORT QgsProcessingUtils
         QgsWkbTypes::Type geometryType,
         const QgsCoordinateReferenceSystem &crs,
         const QVariantMap &createOptions = QVariantMap(),
-        QgsFeatureSink::SinkFlags sinkFlags = nullptr ) SIP_FACTORY;
+        QgsFeatureSink::SinkFlags sinkFlags = nullptr,
+        QgsRemappingSinkDefinition *remappingDefinition = nullptr ) SIP_FACTORY;
 #endif
 
     /**

diff --git a/src/core/qgsremappingproxyfeaturesink.cpp b/src/core/qgsremappingproxyfeaturesink.cpp
@@ -18,12 +18,19 @@
 #include "qgsremappingproxyfeaturesink.h"
 #include "qgslogger.h"
 
-QgsRemappingProxyFeatureSink::QgsRemappingProxyFeatureSink( const QgsRemappingSinkDefinition &mappingDefinition, QgsFeatureSink *sink )
+QgsRemappingProxyFeatureSink::QgsRemappingProxyFeatureSink( const QgsRemappingSinkDefinition &mappingDefinition, QgsFeatureSink *sink, bool ownsSink )
   : QgsFeatureSink()
   , mDefinition( mappingDefinition )
   , mSink( sink )
+  , mOwnsSink( ownsSink )
 {}
 
+QgsRemappingProxyFeatureSink::~QgsRemappingProxyFeatureSink()
+{
+  if ( mOwnsSink )
+    delete mSink;
+}
+
 void QgsRemappingProxyFeatureSink::setExpressionContext( const QgsExpressionContext &context )
 {
   mContext = context;
@@ -39,6 +46,7 @@ QgsFeatureList QgsRemappingProxyFeatureSink::remapFeature( const QgsFeature &fea
   QgsFeatureList res;
 
   mContext.setFeature( feature );
+  mContext.setFields( feature.fields() );
 
   // remap fields first
   QgsFeature f;

diff --git a/src/core/qgsremappingproxyfeaturesink.h b/src/core/qgsremappingproxyfeaturesink.h
@@ -179,11 +179,27 @@ class CORE_EXPORT QgsRemappingProxyFeatureSink : public QgsFeatureSink
 {
   public:
 
+#ifndef SIP_RUN
+
+    /**
+     * Constructor for QgsRemappingProxyFeatureSink, using the specified \a mappingDefinition
+     * to manipulate features before sending them to the destination \a sink.
+     *
+     * Ownership of \a sink is dictated by \a ownsSink. If \a ownsSink is FALSE,
+     * ownership is not transferred, and callers must ensure that \a sink exists for the lifetime of this object.
+     * If \a ownsSink is TRUE, then this object will take ownership of \a sink.
+     */
+    QgsRemappingProxyFeatureSink( const QgsRemappingSinkDefinition &mappingDefinition, QgsFeatureSink *sink, bool ownsSink = false );
+#else
+
     /**
      * Constructor for QgsRemappingProxyFeatureSink, using the specified \a mappingDefinition
      * to manipulate features before sending them to the destination \a sink.
      */
     QgsRemappingProxyFeatureSink( const QgsRemappingSinkDefinition &mappingDefinition, QgsFeatureSink *sink );
+#endif
+
+    ~QgsRemappingProxyFeatureSink() override;
 
     /**
      * Sets the expression \a context to use when evaluating mapped field values.
@@ -215,6 +231,7 @@ class CORE_EXPORT QgsRemappingProxyFeatureSink : public QgsFeatureSink
     QgsCoordinateTransform mTransform;
     QgsFeatureSink *mSink = nullptr;
     mutable QgsExpressionContext mContext;
+    bool mOwnsSink = false;
 };
 
 #endif // QGSREMAPPINGPROXYFEATURESINK_H

diff --git a/tests/src/analysis/testqgsprocessing.cpp b/tests/src/analysis/testqgsprocessing.cpp
@@ -1800,6 +1800,8 @@ void TestQgsProcessing::createFeatureSink()
   prevDest = QDir::tempPath() + "/create_feature_sink2.gpkg";
   sink.reset( QgsProcessingUtils::createFeatureSink( destination, context, fields, QgsWkbTypes::PointZ, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ) ) );
   QVERIFY( sink.get() );
+  f = QgsFeature( fields );
+  f.setAttributes( QgsAttributes() << "val" );
   f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "PointZ(1 2 3)" ) ) );
   QVERIFY( sink->addFeature( f ) );
   QCOMPARE( destination, prevDest );
@@ -1812,7 +1814,38 @@ void TestQgsProcessing::createFeatureSink()
   QCOMPARE( layer->fields().at( 1 ).name(), QStringLiteral( "my_field" ) );
   QCOMPARE( layer->fields().at( 1 ).type(), QVariant::String );
   QCOMPARE( layer->featureCount(), 1L );
-
+  // append to existing OGR layer
+  QgsRemappingSinkDefinition remapDef;
+  remapDef.setDestinationFields( layer->fields() );
+  remapDef.setDestinationCrs( layer->crs() );
+  remapDef.setSourceCrs( QgsCoordinateReferenceSystem( "EPSG:4326" ) );
+  remapDef.setDestinationWkbType( QgsWkbTypes::Polygon );
+  remapDef.addMappedField( QStringLiteral( "fid" ), QgsProperty::fromValue( 3 ) );
+  remapDef.addMappedField( QStringLiteral( "my_field" ), QgsProperty::fromExpression( QStringLiteral( "field2 || @extra" ) ) );
+  QgsFields fields2;
+  fields2.append( QgsField( "field2", QVariant::String ) );
+  context.expressionContext().appendScope( new QgsExpressionContextScope() );
+  context.expressionContext().scope( 0 )->setVariable( QStringLiteral( "extra" ), 2 );
+  sink.reset( QgsProcessingUtils::createFeatureSink( destination, context, fields2, QgsWkbTypes::Point, QgsCoordinateReferenceSystem::fromEpsgId( 4326 ), QVariantMap(), nullptr, &remapDef ) );
+  QVERIFY( sink.get() );
+  f = QgsFeature( fields2 );
+  f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "Point(10 0)" ) ) );
+  f.setAttributes( QgsAttributes() << "val" );
+  QVERIFY( sink->addFeature( f ) );
+  QCOMPARE( destination, prevDest );
+  sink.reset( nullptr );
+  layer = new QgsVectorLayer( destination );
+  QVERIFY( layer->isValid() );
+  QCOMPARE( layer->featureCount(), 2L );
+  QgsFeatureIterator it = layer->getFeatures();
+  QVERIFY( it.nextFeature( f ) );
+  QCOMPARE( f.attributes().at( 1 ).toString(), QStringLiteral( "val" ) );
+  QCOMPARE( f.geometry().asWkt( 1 ), QStringLiteral( "PointZ (1 2 3)" ) );
+  QVERIFY( it.nextFeature( f ) );
+  QCOMPARE( f.attributes().at( 0 ).toInt(), 3 );
+  QCOMPARE( f.attributes().at( 1 ).toString(), QStringLiteral( "val2" ) );
+  QCOMPARE( f.geometry().asWkt( 0 ), QStringLiteral( "Point (-10199761 -4017774)" ) );
+  delete layer;
   //windows style path
   destination = "d:\\temp\\create_feature_sink.tab";
   sink.reset( QgsProcessingUtils::createFeatureSink( destination, context, fields, QgsWkbTypes::Polygon, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ) ) );