Skip to content

Commit

Permalink
[feature][processing] Add new algorithm to retain only selected fields
Browse files Browse the repository at this point in the history 
Allows users to select a list of fields to keep, and all other fields
will be dropped from the layer. Helps with making flexible models where
input datasets may have a range of different fields and you need to drop
all but a certain subset of these
  • Loading branch information
@nyalldawson
nyalldawson committed Nov 23, 2020
1 parent b07a58c commit b6ee3e6
Show file tree
Hide file tree
Showing 10 changed files with 493 additions and 1 deletion.
49 changes: 49 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_one.gml
View file
@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="utf-8" ?>
<ogr:FeatureCollection
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://ogr.maptools.org/ retain_fields_one.xsd"
xmlns:ogr="http://ogr.maptools.org/"
xmlns:gml="http://www.opengis.net/gml">
<gml:boundedBy>
<gml:Box>
<gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
<gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
</gml:Box>
</gml:boundedBy>

<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.0">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>aaaaa</ogr:name>
</ogr:retain_fields_one>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.1">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>Aaaaa</ogr:name>
</ogr:retain_fields_one>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.2">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>bbaaa</ogr:name>
</ogr:retain_fields_one>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.3">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>ASDF</ogr:name>
</ogr:retain_fields_one>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.4">
<ogr:name xsi:nil="true"/>
</ogr:retain_fields_one>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_one fid="retain_fields_one.5">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>elim</ogr:name>
</ogr:retain_fields_one>
</gml:featureMember>
</ogr:FeatureCollection>
30 changes: 30 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_one.xsd
View file
@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml" elementFormDefault="qualified" version="1.0">
<xs:import namespace="http://www.opengis.net/gml" schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
<xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
<xs:complexType name="FeatureCollectionType">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureCollectionType">
<xs:attribute name="lockId" type="xs:string" use="optional"/>
<xs:attribute name="scope" type="xs:string" use="optional"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:element name="retain_fields_one" type="ogr:retain_fields_one_Type" substitutionGroup="gml:_Feature"/>
<xs:complexType name="retain_fields_one_Type">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureType">
<xs:sequence>
<xs:element name="geometryProperty" type="gml:PolygonPropertyType" nillable="true" minOccurs="0" maxOccurs="1"/>
<xs:element name="name" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:maxLength value="5"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
</xs:schema>
61 changes: 61 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_three.gml
View file
@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="utf-8" ?>
<ogr:FeatureCollection
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://ogr.maptools.org/ retain_fields_three.xsd"
xmlns:ogr="http://ogr.maptools.org/"
xmlns:gml="http://www.opengis.net/gml">
<gml:boundedBy>
<gml:Box>
<gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
<gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
</gml:Box>
</gml:boundedBy>

<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.0">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>aaaaa</ogr:name>
<ogr:intval>33</ogr:intval>
<ogr:floatval>44.123456</ogr:floatval>
</ogr:retain_fields_three>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.1">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>Aaaaa</ogr:name>
<ogr:intval>-33</ogr:intval>
<ogr:floatval>0</ogr:floatval>
</ogr:retain_fields_three>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.2">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>bbaaa</ogr:name>
<ogr:intval xsi:nil="true"/>
<ogr:floatval>0.123</ogr:floatval>
</ogr:retain_fields_three>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.3">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>ASDF</ogr:name>
<ogr:intval>0</ogr:intval>
<ogr:floatval xsi:nil="true"/>
</ogr:retain_fields_three>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.4">
<ogr:name xsi:nil="true"/>
<ogr:intval>120</ogr:intval>
<ogr:floatval>-100291.43213</ogr:floatval>
</ogr:retain_fields_three>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_three fid="retain_fields_three.5">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>elim</ogr:name>
<ogr:intval>2</ogr:intval>
<ogr:floatval>3.33</ogr:floatval>
</ogr:retain_fields_three>
</gml:featureMember>
</ogr:FeatureCollection>
43 changes: 43 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_three.xsd
View file
@@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml" elementFormDefault="qualified" version="1.0">
<xs:import namespace="http://www.opengis.net/gml" schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
<xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
<xs:complexType name="FeatureCollectionType">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureCollectionType">
<xs:attribute name="lockId" type="xs:string" use="optional"/>
<xs:attribute name="scope" type="xs:string" use="optional"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:element name="retain_fields_three" type="ogr:retain_fields_three_Type" substitutionGroup="gml:_Feature"/>
<xs:complexType name="retain_fields_three_Type">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureType">
<xs:sequence>
<xs:element name="geometryProperty" type="gml:PolygonPropertyType" nillable="true" minOccurs="0" maxOccurs="1"/>
<xs:element name="name" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:maxLength value="5"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
<xs:element name="intval" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:integer">
<xs:totalDigits value="10"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
<xs:element name="floatval" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:decimal">
</xs:restriction>
</xs:simpleType>
</xs:element>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
</xs:schema>
55 changes: 55 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_two.gml
View file
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="utf-8" ?>
<ogr:FeatureCollection
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://ogr.maptools.org/ retain_fields_two.xsd"
xmlns:ogr="http://ogr.maptools.org/"
xmlns:gml="http://www.opengis.net/gml">
<gml:boundedBy>
<gml:Box>
<gml:coord><gml:X>-1</gml:X><gml:Y>-3</gml:Y></gml:coord>
<gml:coord><gml:X>10</gml:X><gml:Y>6</gml:Y></gml:coord>
</gml:Box>
</gml:boundedBy>

<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.0">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>-1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>aaaaa</ogr:name>
<ogr:floatval>44.123456</ogr:floatval>
</ogr:retain_fields_two>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.1">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>5,5 6,4 4,4 5,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>Aaaaa</ogr:name>
<ogr:floatval>0</ogr:floatval>
</ogr:retain_fields_two>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.2">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>2,5 2,6 3,6 3,5 2,5</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>bbaaa</ogr:name>
<ogr:floatval>0.123</ogr:floatval>
</ogr:retain_fields_two>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.3">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>6,1 10,1 10,-3 6,-3 6,1</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs><gml:innerBoundaryIs><gml:LinearRing><gml:coordinates>7,0 7,-2 9,-2 9,0 7,0</gml:coordinates></gml:LinearRing></gml:innerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>ASDF</ogr:name>
<ogr:floatval xsi:nil="true"/>
</ogr:retain_fields_two>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.4">
<ogr:name xsi:nil="true"/>
<ogr:floatval>-100291.43213</ogr:floatval>
</ogr:retain_fields_two>
</gml:featureMember>
<gml:featureMember>
<ogr:retain_fields_two fid="retain_fields_two.5">
<ogr:geometryProperty><gml:Polygon srsName="EPSG:4326"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>3,2 6,1 6,-3 2,-1 2,2 3,2</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon></ogr:geometryProperty>
<ogr:name>elim</ogr:name>
<ogr:floatval>3.33</ogr:floatval>
</ogr:retain_fields_two>
</gml:featureMember>
</ogr:FeatureCollection>
36 changes: 36 additions & 0 deletions python/plugins/processing/tests/testdata/expected/retain_fields_two.xsd
View file
@@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml" elementFormDefault="qualified" version="1.0">
<xs:import namespace="http://www.opengis.net/gml" schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
<xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
<xs:complexType name="FeatureCollectionType">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureCollectionType">
<xs:attribute name="lockId" type="xs:string" use="optional"/>
<xs:attribute name="scope" type="xs:string" use="optional"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:element name="retain_fields_two" type="ogr:retain_fields_two_Type" substitutionGroup="gml:_Feature"/>
<xs:complexType name="retain_fields_two_Type">
<xs:complexContent>
<xs:extension base="gml:AbstractFeatureType">
<xs:sequence>
<xs:element name="geometryProperty" type="gml:PolygonPropertyType" nillable="true" minOccurs="0" maxOccurs="1"/>
<xs:element name="name" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:maxLength value="5"/>
</xs:restriction>
</xs:simpleType>
</xs:element>
<xs:element name="floatval" nillable="true" minOccurs="0" maxOccurs="1">
<xs:simpleType>
<xs:restriction base="xs:decimal">
</xs:restriction>
</xs:simpleType>
</xs:element>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
</xs:schema>
44 changes: 44 additions & 0 deletions python/plugins/processing/tests/testdata/qgis_algorithm_tests4.yaml
View file
Expand Up @@ -2594,6 +2594,7 @@ tests:
COLUMN:
- fid
- intval
- doesntexist # should NOT cause algorithm to abort, just a warning -- otherwise we can't make flexible models
- floatval
INPUT:
name: polys.gml|layername=polys2
Expand All @@ -2603,5 +2604,48 @@ tests:
name: expected/delete_columns_three.gml
type: vector

- algorithm: native:retainfields
name: Retain one field
params:
FIELDS:
- name
INPUT:
name: polys.gml|layername=polys2
type: vector
results:
OUTPUT:
name: expected/retain_fields_one.gml
type: vector

- algorithm: native:retainfields
name: Retain two fields
params:
FIELDS:
- floatval
- name
INPUT:
name: polys.gml|layername=polys2
type: vector
results:
OUTPUT:
name: expected/retain_fields_two.gml
type: vector

- algorithm: native:retainfields
name: Retain three fields
params:
FIELDS:
- name
- floatval
- doesntexist # should NOT cause algorithm to abort, just a warning -- otherwise we can't make flexible models
- intval
INPUT:
name: polys.gml|layername=polys2
type: vector
results:
OUTPUT:
name: expected/retain_fields_three.gml
type: vector


# See ../README.md for a description of the file format

0 comments on commit b6ee3e6

Please sign in to comment.