Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Cache unique values when creating features
Fixes #21305 - pasting features is very slow

Aggressively optimize createFeature for speed
and introduces createFeatures for bulk creation.
  • Loading branch information
elpaso committed Feb 19, 2019
1 parent f2e745e commit ba3d9ed
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 77 deletions.
45 changes: 45 additions & 0 deletions python/core/auto_generated/qgsvectorlayerutils.sip.in
Expand Up @@ -57,6 +57,40 @@ Returns the duplicated features in the given layer

};

class QgsFeaturesData
{
%Docstring
Encapsulate geometry and attributes for new features, to be passed to createFeatures

.. seealso:: :py:func:`createFeatures`

.. versionadded:: 3.6
%End

%TypeHeaderCode
#include "qgsvectorlayerutils.h"
%End
public:

QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );
%Docstring
Constructs a new QgsFeaturesData with given ``geometry`` and ``attributes``
%End

QgsGeometry geometry() const;
%Docstring
Returns geometry
%End

QgsAttributeMap attributes() const;
%Docstring
Returns attributes
%End

};

typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;

static QgsFeatureIterator getValuesIterator( const QgsVectorLayer *layer, const QString &fieldOrExpression, bool &ok, bool selectedOnly );
%Docstring
Create a feature iterator for a specified field name or expression.
Expand Down Expand Up @@ -143,6 +177,17 @@ Creates a new feature ready for insertion into a layer. Default values and const
passed for the new feature to copy as many attribute values as possible from the map,
assuming that they respect the layer's constraints. Note that the created feature is not
automatically inserted into the layer.
%End

static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
const QgsFeaturesDataList &featuresData,
QgsExpressionContext *context = 0 );
%Docstring
Creates a set of new features ready for insertion into a layer. Default values and constraints
(e.g., unique constraints) will automatically be handled. An optional attribute map can be
passed for the new feature to copy as many attribute values as possible from the map,
assuming that they respect the layer's constraints. Note that the created features are not
automatically inserted into the layer.
%End

static QgsFeature duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext /Out/ );
Expand Down
10 changes: 6 additions & 4 deletions src/app/qgisapp.cpp
Expand Up @@ -8996,11 +8996,12 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
QgsExpressionContext context = pasteVectorLayer->createExpressionContext();

QgsFeatureList compatibleFeatures( QgsVectorLayerUtils::makeFeaturesCompatible( features, pasteVectorLayer ) );
QgsFeatureList newFeatures;
QgsVectorLayerUtils::QgsFeaturesDataList newFeaturesDataList;
newFeaturesDataList.reserve( compatibleFeatures.size() );

// Count collapsed geometries
int invalidGeometriesCount = 0;

newFeatures.reserve( compatibleFeatures.size() );
for ( const auto &feature : qgis::as_const( compatibleFeatures ) )
{

Expand All @@ -9022,8 +9023,10 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
}
// now create new feature using pasted feature as a template. This automatically handles default
// values and field constraints
newFeatures << QgsVectorLayerUtils::createFeature( pasteVectorLayer, geom, attrMap, &context );
newFeaturesDataList << QgsVectorLayerUtils::QgsFeaturesData( geom, attrMap );
}

QgsFeatureList newFeatures {QgsVectorLayerUtils::createFeatures( pasteVectorLayer, newFeaturesDataList, &context )};
pasteVectorLayer->addFeatures( newFeatures );
QgsFeatureIds newIds;
newIds.reserve( newFeatures.size() );
Expand All @@ -9032,7 +9035,6 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
newIds << f.id();
}


pasteVectorLayer->selectByIds( newIds );
pasteVectorLayer->endEditCommand();
pasteVectorLayer->updateExtents();
Expand Down
177 changes: 104 additions & 73 deletions src/core/qgsvectorlayerutils.cpp
Expand Up @@ -358,11 +358,17 @@ bool QgsVectorLayerUtils::validateAttribute( const QgsVectorLayer *layer, const

QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, const QgsGeometry &geometry,
const QgsAttributeMap &attributes, QgsExpressionContext *context )
{
return createFeatures( layer, QgsFeaturesDataList() << QgsFeaturesData( geometry, attributes ), context ).first();
}

QgsFeatureList QgsVectorLayerUtils::createFeatures( const QgsVectorLayer *layer, const QgsFeaturesDataList &featuresData, QgsExpressionContext *context )
{
if ( !layer )
{
return QgsFeature();
}
return QgsFeatureList();

QgsFeatureList result;
result.reserve( featuresData.length() );

QgsExpressionContext *evalContext = context;
std::unique_ptr< QgsExpressionContext > tempContext;
Expand All @@ -375,94 +381,104 @@ QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, cons

QgsFields fields = layer->fields();

QgsFeature newFeature( fields );
newFeature.setValid( true );
newFeature.setGeometry( geometry );
// Cache unique values
QMap<int, QSet<QVariant>> uniqueValueCaches;

// initialize attributes
newFeature.initAttributes( fields.count() );
for ( int idx = 0; idx < fields.count(); ++idx )
for ( const auto &fd : qgis::as_const( featuresData ) )
{
QVariant v;
bool checkUnique = true;

// in order of priority:
// 1. passed attribute value and if field does not have a unique constraint like primary key
if ( attributes.contains( idx ) )
QgsFeature newFeature( fields );
newFeature.setValid( true );
newFeature.setGeometry( fd.geometry() );

// initialize attributes
newFeature.initAttributes( fields.count() );
for ( int idx = 0; idx < fields.count(); ++idx )
{
v = attributes.value( idx );
}
QVariant v;
bool checkUnique = true;
const bool hasUniqueConstraint { static_cast<bool>( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique ) };

// Cache unique values
QSet<QVariant> uniqueValues { layer->uniqueValues( idx ) };
// in order of priority:
// 1. passed attribute value and if field does not have a unique constraint like primary key
if ( fd.attributes().contains( idx ) )
{
v = fd.attributes().value( idx );
}

// 2. client side default expression
// note - deliberately not using else if!
if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
&& uniqueValues.contains( v ) ) )
&& layer->defaultValueDefinition( idx ).isValid() )
{
// client side default expression set - takes precedence over all. Why? Well, this is the only default
// which QGIS users have control over, so we assume that they're deliberately overriding any
// provider defaults for some good reason and we should respect that
v = layer->defaultValue( idx, newFeature, evalContext );
}
// Cache unique values
if ( hasUniqueConstraint && ! uniqueValueCaches.contains( idx ) )
uniqueValueCaches[ idx ] = layer->uniqueValues( idx );

// 3. provider side default value clause
// note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
&& uniqueValues.contains( v ) ) )
&& fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
{
int providerIndex = fields.fieldOriginIndex( idx );
QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
if ( !providerDefault.isEmpty() )
// 2. client side default expression
// note - deliberately not using else if!
if ( ( !v.isValid() || ( hasUniqueConstraint
&& uniqueValueCaches[ idx ].contains( v ) ) )
&& layer->defaultValueDefinition( idx ).isValid() )
{
v = providerDefault;
checkUnique = false;
// client side default expression set - takes precedence over all. Why? Well, this is the only default
// which QGIS users have control over, so we assume that they're deliberately overriding any
// provider defaults for some good reason and we should respect that
v = layer->defaultValue( idx, newFeature, evalContext );
}
}

// 4. provider side default literal
// note - deliberately not using else if!
if ( ( !v.isValid() || ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
&& uniqueValues.contains( v ) ) )
&& fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
{
int providerIndex = fields.fieldOriginIndex( idx );
v = layer->dataProvider()->defaultValue( providerIndex );
if ( v.isValid() )
// 3. provider side default value clause
// note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
if ( ( !v.isValid() || ( hasUniqueConstraint
&& uniqueValueCaches[ idx ].contains( v ) ) )
&& fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
{
//trust that the provider default has been sensibly set not to violate any constraints
checkUnique = false;
int providerIndex = fields.fieldOriginIndex( idx );
QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
if ( !providerDefault.isEmpty() )
{
v = providerDefault;
checkUnique = false;
}
}
}

// 5. passed attribute value
// note - deliberately not using else if!
if ( !v.isValid() && attributes.contains( idx ) )
{
v = attributes.value( idx );
}
// 4. provider side default literal
// note - deliberately not using else if!
if ( ( !v.isValid() || ( checkUnique && hasUniqueConstraint
&& uniqueValueCaches[ idx ].contains( v ) ) )
&& fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
{
int providerIndex = fields.fieldOriginIndex( idx );
v = layer->dataProvider()->defaultValue( providerIndex );
if ( v.isValid() )
{
//trust that the provider default has been sensibly set not to violate any constraints
checkUnique = false;
}
}

// last of all... check that unique constraints are respected
// we can't handle not null or expression constraints here, since there's no way to pick a sensible
// value if the constraint is violated
if ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique )
{
if ( uniqueValues.contains( v ) )
// 5. passed attribute value
// note - deliberately not using else if!
if ( !v.isValid() && fd.attributes().contains( idx ) )
{
// unique constraint violated
QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
if ( uniqueValue.isValid() )
v = uniqueValue;
v = fd.attributes().value( idx );
}
}

newFeature.setAttribute( idx, v );
// last of all... check that unique constraints are respected
// we can't handle not null or expression constraints here, since there's no way to pick a sensible
// value if the constraint is violated
if ( checkUnique && hasUniqueConstraint )
{
if ( uniqueValueCaches[ idx ].contains( v ) )
{
// unique constraint violated
QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
if ( uniqueValue.isValid() )
v = uniqueValue;
}
}
if ( hasUniqueConstraint )
uniqueValueCaches[ idx ].insert( v );
newFeature.setAttribute( idx, v );
}
result.append( newFeature );
}

return newFeature;
return result;
}

QgsFeature QgsVectorLayerUtils::duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext )
Expand Down Expand Up @@ -772,3 +788,18 @@ QMap<QgsVectorLayer *, QgsFeatureIds> QgsVectorLayerUtils::QgsDuplicateFeatureC
return mDuplicatedFeatures;
}
*/

QgsVectorLayerUtils::QgsFeaturesData::QgsFeaturesData( const QgsGeometry &geometry, const QgsAttributeMap &attributes ):
mGeometry( geometry ),
mAttributes( attributes )
{}

QgsGeometry QgsVectorLayerUtils::QgsFeaturesData::geometry() const
{
return mGeometry;
}

QgsAttributeMap QgsVectorLayerUtils::QgsFeaturesData::attributes() const
{
return mAttributes;
}
42 changes: 42 additions & 0 deletions src/core/qgsvectorlayerutils.h
Expand Up @@ -70,6 +70,37 @@ class CORE_EXPORT QgsVectorLayerUtils
void setDuplicatedFeatures( QgsVectorLayer *layer, const QgsFeatureIds &ids );
};

/**
* \ingroup core
* \class QgsFeatureSetData
* \brief Encapsulate geometry and attributes for new features, to be passed to createFeatures
* \see createFeatures()
* \since QGIS 3.6
*/
class CORE_EXPORT QgsFeaturesData
{
public:

/**
* Constructs a new QgsFeaturesData with given \a geometry and \a attributes
*/
QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );

//! Returns geometry
QgsGeometry geometry() const;

//! Returns attributes
QgsAttributeMap attributes() const;

private:
QgsGeometry mGeometry;
QgsAttributeMap mAttributes;
};

// SIP does not lile "using", use legacy typedef
//! Alias for list of QgsFeaturesData
typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;

/**
* Create a feature iterator for a specified field name or expression.
* \param layer vector layer to retrieve values from
Expand Down Expand Up @@ -145,6 +176,17 @@ class CORE_EXPORT QgsVectorLayerUtils
const QgsAttributeMap &attributes = QgsAttributeMap(),
QgsExpressionContext *context = nullptr );

/**
* Creates a set of new features ready for insertion into a layer. Default values and constraints
* (e.g., unique constraints) will automatically be handled. An optional attribute map can be
* passed for the new feature to copy as many attribute values as possible from the map,
* assuming that they respect the layer's constraints. Note that the created features are not
* automatically inserted into the layer.
*/
static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
const QgsFeaturesDataList &featuresData,
QgsExpressionContext *context = nullptr );

/**
* Duplicates a feature and it's children (one level deep). It calls CreateFeature, so
* default values and constraints (e.g., unique constraints) will automatically be handled.
Expand Down

0 comments on commit ba3d9ed

Please sign in to comment.