Skip to content

Commit

Permalink
[FEATURE][processing] Join by field value can export unmatched records,
Browse files Browse the repository at this point in the history 
and reports counts of matched/unmatched features

This gives an explicit warning to users when features were not matched,
and optionally allows them to save non-matching features to a layer.
  • Loading branch information
@nyalldawson
nyalldawson committed Aug 17, 2018
1 parent 78aa26a commit 773371a
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 14 deletions.
70 changes: 70 additions & 0 deletions python/plugins/processing/tests/testdata/expected/join_attribute_table_unjoinable.gml
View file
@@ -0,0 +1,70 @@
<?xml version="1.0" encoding="utf-8" ?>
<ogr:FeatureCollection
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://ogr.maptools.org/ join_attribute_table_unjoinable.xsd"
xmlns:ogr="http://ogr.maptools.org/"
xmlns:gml="http://www.opengis.net/gml">
<gml:boundedBy>
<gml:Box>
<gml:coord><gml:X>0</gml:X><gml:Y>-5</gml:Y></gml:coord>
<gml:coord><gml:X>8</gml:X><gml:Y>3</gml:Y></gml:coord>
</gml:Box>
</gml:boundedBy>

<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.1">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>3,3</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>2</ogr:id>
<ogr:id2>1</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.2">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>2,2</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>3</ogr:id>
<ogr:id2>0</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.3">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>5,2</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>4</ogr:id>
<ogr:id2>2</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.4">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>4,1</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>5</ogr:id>
<ogr:id2>1</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.5">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>0,-5</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>6</ogr:id>
<ogr:id2>0</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.6">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>8,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>7</ogr:id>
<ogr:id2>0</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.7">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>7,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>8</ogr:id>
<ogr:id2>0</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
<gml:featureMember>
<ogr:join_attribute_table_unjoinable fid="points.8">
<ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>0,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
<ogr:id>9</ogr:id>
<ogr:id2>0</ogr:id2>
</ogr:join_attribute_table_unjoinable>
</gml:featureMember>
</ogr:FeatureCollection>
37 changes: 37 additions & 0 deletions python/plugins/processing/tests/testdata/expected/join_attribute_table_unjoinable.xsd
View file
@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml" elementFormDefault="qualified" version="1.0">
<xs:import namespace="http://www.opengis.net/gml" schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
<xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
<xs:complexType name="FeatureCollectionType">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureCollectionType">
<xs:attribute name="lockId" type="xs:string" use="optional"/>
<xs:attribute name="scope" type="xs:string" use="optional"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:element name="join_attribute_table_unjoinable" type="ogr:join_attribute_table_unjoinable_Type" substitutionGroup="gml:_Feature"/>
<xs:complexType name="join_attribute_table_unjoinable_Type">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureType">
<xs:sequence>
<xs:element name="geometryProperty" type="gml:PointPropertyType" nillable="true" minOccurs="0" maxOccurs="1"/>
<xs:element name="id" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:integer">
<xs:totalDigits value="10"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
<xs:element name="id2" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:integer">
<xs:totalDigits value="10"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
</xs:schema>
19 changes: 19 additions & 0 deletions python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
View file
Expand Up @@ -2991,6 +2991,25 @@ tests:
name: expected/join_attribute_table_subset.gml
type: vector

- algorithm: native:joinattributestable
name: Join attributes table, saving unjoinable features
params:
DISCARD_NONMATCHING: false
FIELD: id
FIELD_2: ID
INPUT:
name: points.gml
type: vector
INPUT_2:
name: table2.dbf
type: vector
METHOD: 1
PREFIX: ''
results:
NON_MATCHING:
name: expected/join_attribute_table_unjoinable.gml
type: vector

- algorithm: native:shortestpathpointtopoint
name: Shortest path (point to point, shortest route)
params:
Expand Down
67 changes: 53 additions & 14 deletions src/analysis/processing/qgsalgorithmjoinbyattribute.cpp
View file
Expand Up @@ -16,6 +16,7 @@
***************************************************************************/

#include "qgsalgorithmjoinbyattribute.h"
#include "qgsprocessingoutputs.h"

///@cond PRIVATE

Expand Down Expand Up @@ -75,7 +76,16 @@ void QgsJoinByAttributeAlgorithm::initAlgorithm( const QVariantMap & )
addParameter( new QgsProcessingParameterString( QStringLiteral( "PREFIX" ),
QObject::tr( "Joined field prefix" ), QVariant(), false, true ) );

addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "OUTPUT" ), QObject::tr( "Joined layer" ) ) );
addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "OUTPUT" ), QObject::tr( "Joined layer" ), QgsProcessing::TypeVectorAnyGeometry, QVariant(), true, true ) );

std::unique_ptr< QgsProcessingParameterFeatureSink > nonMatchingSink = qgis::make_unique< QgsProcessingParameterFeatureSink >(
QStringLiteral( "NON_MATCHING" ), QObject::tr( "Unjoinable features from first layer" ), QgsProcessing::TypeVectorAnyGeometry, QVariant(), true, false );
// TODO GUI doesn't support advanced outputs yet
//nonMatchingSink->setFlags(nonMatchingSink->flags() | QgsProcessingParameterDefinition::FlagAdvanced );
addParameter( nonMatchingSink.release() );

addOutput( new QgsProcessingOutputNumber( QStringLiteral( "JOINED_COUNT" ), QObject::tr( "Number of joined features from input table" ) ) );
addOutput( new QgsProcessingOutputNumber( QStringLiteral( "UNJOINABLE_COUNT" ), QObject::tr( "Number of unjoinable features from input table" ) ) );
}

QString QgsJoinByAttributeAlgorithm::shortHelpString() const
Expand Down Expand Up @@ -154,9 +164,14 @@ QVariantMap QgsJoinByAttributeAlgorithm::processAlgorithm( const QVariantMap &pa
QString dest;
std::unique_ptr< QgsFeatureSink > sink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, dest, outFields,
input->wkbType(), input->sourceCrs() ) );
if ( !sink )
if ( parameters.value( QStringLiteral( "OUTPUT" ) ).isValid() && !sink )
throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "OUTPUT" ) ) );

QString destNonMatching1;
std::unique_ptr< QgsFeatureSink > sinkNonMatching1( parameterAsSink( parameters, QStringLiteral( "NON_MATCHING" ), context, destNonMatching1, input->fields(),
input->wkbType(), input->sourceCrs() ) );
if ( parameters.value( QStringLiteral( "NON_MATCHING" ) ).isValid() && !sinkNonMatching1 )
throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "NON_MATCHING" ) ) );

// cache attributes of input2
QMultiHash< QVariant, QgsAttributes > input2AttributeCache;
Expand Down Expand Up @@ -193,6 +208,8 @@ QVariantMap QgsJoinByAttributeAlgorithm::processAlgorithm( const QVariantMap &pa
step = input->featureCount() > 0 ? 50.0 / input->featureCount() : 1;
features = input->getFeatures( QgsFeatureRequest(), QgsProcessingFeatureSource::FlagSkipGeometryValidityChecks );
i = 0;
long long joinedCount = 0;
long long unjoinedCount = 0;
while ( features.nextFeature( feat ) )
{
i++;
Expand All @@ -205,26 +222,48 @@ QVariantMap QgsJoinByAttributeAlgorithm::processAlgorithm( const QVariantMap &pa

if ( input2AttributeCache.count( feat.attribute( joinField1Index ) ) > 0 )
{
QgsAttributes attrs = feat.attributes();

QList< QgsAttributes > attributes = input2AttributeCache.values( feat.attribute( joinField1Index ) );
QList< QgsAttributes >::iterator attrsIt = attributes.begin();
for ( ; attrsIt != attributes.end(); ++attrsIt )
joinedCount++;
if ( sink )
{
QgsAttributes newAttrs = attrs;
newAttrs.append( *attrsIt );
feat.setAttributes( newAttrs );
sink->addFeature( feat, QgsFeatureSink::FastInsert );
QgsAttributes attrs = feat.attributes();

QList< QgsAttributes > attributes = input2AttributeCache.values( feat.attribute( joinField1Index ) );
QList< QgsAttributes >::iterator attrsIt = attributes.begin();
for ( ; attrsIt != attributes.end(); ++attrsIt )
{
QgsAttributes newAttrs = attrs;
newAttrs.append( *attrsIt );
feat.setAttributes( newAttrs );
sink->addFeature( feat, QgsFeatureSink::FastInsert );
}
}
}
else if ( !discardNonMatching )
else
{
sink->addFeature( feat, QgsFeatureSink::FastInsert );
// no matching for input feature
if ( sink && !discardNonMatching )
{
sink->addFeature( feat, QgsFeatureSink::FastInsert );
}
if ( sinkNonMatching1 )
{
sinkNonMatching1->addFeature( feat );
}
unjoinedCount++;
}
}

feedback->pushInfo( QObject::tr( "%1 feature(s) from input layer were successfully matched" ).arg( joinedCount ) );
if ( unjoinedCount > 0 )
feedback->reportError( QObject::tr( "%1 feature(s) from input layer could not be matched" ).arg( unjoinedCount ) );

QVariantMap outputs;
outputs.insert( QStringLiteral( "OUTPUT" ), dest );
if ( sink )
outputs.insert( QStringLiteral( "OUTPUT" ), dest );
outputs.insert( QStringLiteral( "JOINED_COUNT" ), joinedCount );
outputs.insert( QStringLiteral( "UNJOINABLE_COUNT" ), unjoinedCount );
if ( sinkNonMatching1 )
outputs.insert( QStringLiteral( "NON_MATCHING" ), destNonMatching1 );
return outputs;
}

Expand Down

0 comments on commit 773371a

Please sign in to comment.