[processing] List unique values improvements

- allow running on non-spatial tables - allow choice of more than one field
qgis · Dec 15, 2017 · 5b1da98 · 5b1da98
1 parent ebd98e3
commit 5b1da98
Show file tree

Hide file tree

Showing 7 changed files with 150 additions and 20 deletions.
diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml
@@ -267,7 +267,7 @@ qgis:linestopolygons: >
   The attribute table of the output layer is the same as the one from of the input line layer.
 
 qgis:listuniquevalues: >
-  This algorithm generates a report with information about the categories found in a given attribute of a vector layer.
+  This algorithm generates a report with information about the unique values found in a given attribute (or attributes) of a vector layer.
 
 qgis:meanandstandarddeviationplot:
 

diff --git a/python/plugins/processing/algs/qgis/UniqueValues.py b/python/plugins/processing/algs/qgis/UniqueValues.py
@@ -34,7 +34,9 @@
                        QgsWkbTypes,
                        QgsFeature,
                        QgsFeatureSink,
+                       QgsFeatureRequest,
                        QgsFields,
+                       QgsProcessing,
                        QgsProcessingParameterField,
                        QgsProcessingParameterFeatureSource,
                        QgsProcessingParameterFeatureSink,
@@ -51,7 +53,7 @@
 class UniqueValues(QgisAlgorithm):
 
     INPUT = 'INPUT'
-    FIELD_NAME = 'FIELD_NAME'
+    FIELDS = 'FIELDS'
     TOTAL_VALUES = 'TOTAL_VALUES'
     UNIQUE_VALUES = 'UNIQUE_VALUES'
     OUTPUT = 'OUTPUT'
@@ -71,10 +73,10 @@ def __init__(self):
 
     def initAlgorithm(self, config=None):
         self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT,
-                                                              self.tr('Input layer')))
-        self.addParameter(QgsProcessingParameterField(self.FIELD_NAME,
-                                                      self.tr('Target field'),
-                                                      parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any))
+                                                              self.tr('Input layer'), types=[QgsProcessing.TypeVector]))
+        self.addParameter(QgsProcessingParameterField(self.FIELDS,
+                                                      self.tr('Target field(s)'),
+                                                      parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any, allowMultiple=True))
 
         self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Unique values'), optional=True, defaultValue=''))
 
@@ -91,23 +93,48 @@ def displayName(self):
 
     def processAlgorithm(self, parameters, context, feedback):
         source = self.parameterAsSource(parameters, self.INPUT, context)
-        field_name = self.parameterAsString(parameters, self.FIELD_NAME, context)
-        values = source.uniqueValues(source.fields().lookupField(field_name))
+        field_names = self.parameterAsFields(parameters, self.FIELDS, context)
 
         fields = QgsFields()
-        field = source.fields()[source.fields().lookupField(field_name)]
-        field.setName('VALUES')
-        fields.append(field)
+        field_indices = []
+        for field_name in field_names:
+            field_index = source.fields().lookupField(field_name)
+            if field_index < 0:
+                feedback.reportError(self.tr('Invalid field name {}').format(field_name))
+                continue
+            field = source.fields()[field_index]
+            fields.append(field)
+            field_indices.append(field_index)
         (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context,
                                                fields, QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem())
+
         results = {}
+        values = set()
+        if len(field_indices) == 1:
+            # one field, can use provider optimised method
+            values = tuple([v] for v in source.uniqueValues(field_indices[0]))
+        else:
+            # have to scan whole table
+            # TODO - add this support to QgsVectorDataProvider so we can run it on
+            # the backend
+            request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)
+            request.setSubsetOfAttributes(field_indices)
+            total = 100.0 / source.featureCount() if source.featureCount() else 0
+            for current, f in enumerate(source.getFeatures(request)):
+                if feedback.isCanceled():
+                    break
+
+                value = tuple(f.attribute(i) for i in field_indices)
+                values.add(value)
+                feedback.setProgress(int(current * total))
+
         if sink:
             for value in values:
                 if feedback.isCanceled():
                     break
 
                 f = QgsFeature()
-                f.setAttributes([value])
+                f.setAttributes([attr for attr in value])
                 sink.addFeature(f, QgsFeatureSink.FastInsert)
             results[self.OUTPUT] = dest_id
 
@@ -117,7 +144,7 @@ def processAlgorithm(self, parameters, context, feedback):
             results[self.OUTPUT_HTML_FILE] = output_file
 
         results[self.TOTAL_VALUES] = len(values)
-        results[self.UNIQUE_VALUES] = ';'.join([str(v) for v in
+        results[self.UNIQUE_VALUES] = ';'.join([','.join([str(attr) for attr in v]) for v in
                                                 values])
         return results
 
@@ -130,5 +157,5 @@ def createHTML(self, outputFile, algData):
             f.write(self.tr('<p>Unique values:</p>'))
             f.write('<ul>')
             for s in algData:
-                f.write('<li>' + str(s) + '</li>')
+                f.write('<li>' + ','.join([str(attr) for attr in s]) + '</li>')
             f.write('</ul></body></html>')
diff --git a/python/plugins/processing/tests/testdata/expected/unique_values.gfs b/python/plugins/processing/tests/testdata/expected/unique_values.gfs
@@ -7,8 +7,8 @@
       <FeatureCount>3</FeatureCount>
     </DatasetSpecificInfo>
     <PropertyDefn>
-      <Name>VALUES</Name>
-      <ElementPath>VALUES</ElementPath>
+      <Name>id2</Name>
+      <ElementPath>id2</ElementPath>
       <Type>Integer</Type>
     </PropertyDefn>
   </GMLFeatureClass>

diff --git a/python/plugins/processing/tests/testdata/expected/unique_values.gml b/python/plugins/processing/tests/testdata/expected/unique_values.gml
@@ -8,17 +8,17 @@
 
   <gml:featureMember>
     <ogr:unique_values fid="unique_values.0">
-      <ogr:VALUES>0</ogr:VALUES>
+      <ogr:id2>0</ogr:id2>
     </ogr:unique_values>
   </gml:featureMember>
   <gml:featureMember>
     <ogr:unique_values fid="unique_values.1">
-      <ogr:VALUES>1</ogr:VALUES>
+      <ogr:id2>1</ogr:id2>
     </ogr:unique_values>
   </gml:featureMember>
   <gml:featureMember>
     <ogr:unique_values fid="unique_values.2">
-      <ogr:VALUES>2</ogr:VALUES>
+      <ogr:id2>2</ogr:id2>
     </ogr:unique_values>
   </gml:featureMember>
 </ogr:FeatureCollection>
diff --git a/python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<ogr:FeatureCollection
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://ogr.maptools.org/ unique_values_multiple.xsd"
+     xmlns:ogr="http://ogr.maptools.org/"
+     xmlns:gml="http://www.opengis.net/gml">
+  <gml:boundedBy><gml:null>missing</gml:null></gml:boundedBy>
+
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.0">
+      <ogr:name>aa</ogr:name>
+      <ogr:intval>1</ogr:intval>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.1">
+      <ogr:name>bb</ogr:name>
+      <ogr:intval>2</ogr:intval>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.2">
+      <ogr:name>cc</ogr:name>
+      <ogr:intval xsi:nil="true"/>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.3">
+      <ogr:name xsi:nil="true"/>
+      <ogr:intval>120</ogr:intval>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.4">
+      <ogr:name>bb</ogr:name>
+      <ogr:intval>1</ogr:intval>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:unique_values_multiple fid="unique_values_multiple.5">
+      <ogr:name>dd</ogr:name>
+      <ogr:intval xsi:nil="true"/>
+    </ogr:unique_values_multiple>
+  </gml:featureMember>
+</ogr:FeatureCollection>
diff --git a/python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema targetNamespace="http://ogr.maptools.org/" xmlns:ogr="http://ogr.maptools.org/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:gml="http://www.opengis.net/gml" elementFormDefault="qualified" version="1.0">
+<xs:import namespace="http://www.opengis.net/gml" schemaLocation="http://schemas.opengis.net/gml/2.1.2/feature.xsd"/>
+<xs:element name="FeatureCollection" type="ogr:FeatureCollectionType" substitutionGroup="gml:_FeatureCollection"/>
+<xs:complexType name="FeatureCollectionType">
+  <xs:complexContent>
+    <xs:extension base="gml:AbstractFeatureCollectionType">
+      <xs:attribute name="lockId" type="xs:string" use="optional"/>
+      <xs:attribute name="scope" type="xs:string" use="optional"/>
+    </xs:extension>
+  </xs:complexContent>
+</xs:complexType>
+<xs:element name="unique_values_multiple" type="ogr:unique_values_multiple_Type" substitutionGroup="gml:_Feature"/>
+<xs:complexType name="unique_values_multiple_Type">
+  <xs:complexContent>
+    <xs:extension base="gml:AbstractFeatureType">
+      <xs:sequence>
+        <xs:element name="name" nillable="true" minOccurs="0" maxOccurs="1">
+          <xs:simpleType>
+            <xs:restriction base="xs:string">
+              <xs:maxLength value="2"/>
+            </xs:restriction>
+          </xs:simpleType>
+        </xs:element>
+        <xs:element name="intval" nillable="true" minOccurs="0" maxOccurs="1">
+          <xs:simpleType>
+            <xs:restriction base="xs:integer">
+              <xs:totalDigits value="10"/>
+            </xs:restriction>
+          </xs:simpleType>
+        </xs:element>
+      </xs:sequence>
+    </xs:extension>
+  </xs:complexContent>
+</xs:complexType>
+</xs:schema>
diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml
@@ -408,11 +408,33 @@ tests:
       INPUT:
         name: points.gml
         type: vector
-      FIELD_NAME: id2
+      FIELDS: id2
     results:
       OUTPUT:
         name: expected/unique_values.gml
         type: vector
+        pk:
+        - id2
+
+  - algorithm: qgis:listuniquevalues
+    name: Unique values (multiple fields)
+    params:
+      FIELDS:
+      - name
+      - intval
+      INPUT:
+        name: dissolve_polys.gml
+        type: vector
+    results:
+      OUTPUT:
+        name: expected/unique_values_multiple.gml
+        type: vector
+        compare:
+          fields:
+            fid: skip
+        pk:
+        - name
+        - intval
 
   - algorithm: native:addautoincrementalfield
     name: Add autoincremental field