Skip to content

Commit

Permalink
[memory] Add support for list field types
Browse files Browse the repository at this point in the history
  • Loading branch information
@nyalldawson
nyalldawson committed Mar 25, 2021
1 parent b03bc6c commit 2f8adf7
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 41 deletions.
42 changes: 39 additions & 3 deletions src/core/providers/memory/qgsmemoryprovider.cpp
View file
Expand Up @@ -108,6 +108,12 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
// blob
<< QgsVectorDataProvider::NativeType( tr( "Binary object (BLOB)" ), QStringLiteral( "binary" ), QVariant::ByteArray )

// list types
<< QgsVectorDataProvider::NativeType( tr( "String list" ), QStringLiteral( "stringlist" ), QVariant::List, 0, 0, 0, 0, QVariant::String )
<< QgsVectorDataProvider::NativeType( tr( "Integer list" ), QStringLiteral( "integerlist" ), QVariant::List, 0, 0, 0, 0, QVariant::Int )
<< QgsVectorDataProvider::NativeType( tr( "Decimal (real) list" ), QStringLiteral( "doublelist" ), QVariant::List, 0, 0, 0, 0, QVariant::Double )
<< QgsVectorDataProvider::NativeType( tr( "Integer (64bit) list" ), QStringLiteral( "doublelist" ), QVariant::List, 0, 0, 0, 0, QVariant::Double )

);

if ( query.hasQueryItem( QStringLiteral( "field" ) ) )
Expand Down Expand Up @@ -196,7 +202,8 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
{
//array
subType = type;
type = ( subType == QVariant::String ? QVariant::StringList : QVariant::List );
type = QVariant::List;
typeName += QStringLiteral( "list" );
}
}
if ( !name.isEmpty() )
Expand Down Expand Up @@ -270,9 +277,38 @@ QString QgsMemoryProvider::dataSourceUri( bool expandAuthConfig ) const
QgsAttributeList attrs = const_cast<QgsMemoryProvider *>( this )->attributeIndexes();
for ( int i = 0; i < attrs.size(); i++ )
{
QgsField field = mFields.at( attrs[i] );
const QgsField field = mFields.at( attrs[i] );
QString fieldDef = field.name();
fieldDef.append( QStringLiteral( ":%2(%3,%4)" ).arg( field.typeName() ).arg( field.length() ).arg( field.precision() ) );

QString typeName = field.typeName();
bool isList = false;
if ( field.type() == QVariant::List || field.type() == QVariant::StringList )
{
switch ( field.subType() )
{
case QVariant::Int:
typeName = QStringLiteral( "integer" );
break;

case QVariant::LongLong:
typeName = QStringLiteral( "long" );
break;

case QVariant::Double:
typeName = QStringLiteral( "double" );
break;

case QVariant::String:
typeName = QStringLiteral( "string" );
break;

default:
break;
}
isList = true;
}

fieldDef.append( QStringLiteral( ":%2(%3,%4)%5" ).arg( typeName ).arg( field.length() ).arg( field.precision() ).arg( isList ? QStringLiteral( "[]" ) : QString() ) );
query.addQueryItem( QStringLiteral( "field" ), fieldDef );
}
uri.setQuery( query );
Expand Down
5 changes: 4 additions & 1 deletion src/core/providers/memory/qgsmemoryproviderutils.cpp
View file
Expand Up @@ -74,7 +74,10 @@ QgsVectorLayer *QgsMemoryProviderUtils::createMemoryLayer( const QString &name,
for ( const auto &field : fields )
{
const QString lengthPrecision = QStringLiteral( "(%1,%2)" ).arg( field.length() ).arg( field.precision() );
parts << QStringLiteral( "field=%1:%2%3" ).arg( QString( QUrl::toPercentEncoding( field.name() ) ), memoryLayerFieldType( field.type() ), lengthPrecision );
parts << QStringLiteral( "field=%1:%2%3%4" ).arg( QString( QUrl::toPercentEncoding( field.name() ) ),
memoryLayerFieldType( field.type() == QVariant::List ? field.subType() : field.type() ),
lengthPrecision,
field.type() == QVariant::List || field.type() == QVariant::StringList ? QStringLiteral( "[]" ) : QString() );
}

QString uri = geomType + '?' + parts.join( '&' );
Expand Down
3 changes: 1 addition & 2 deletions src/core/vector/qgsvectordataprovider.cpp
View file
Expand Up @@ -376,8 +376,7 @@ bool QgsVectorDataProvider::supportedType( const QgsField &field ) const
.arg( field.length() )
.arg( field.precision() ), 2 );

const auto constMNativeTypes = mNativeTypes;
for ( const NativeType &nativeType : constMNativeTypes )
for ( const NativeType &nativeType : mNativeTypes )
{
QgsDebugMsgLevel( QStringLiteral( "native field type = %1 min length = %2 max length = %3 min precision = %4 max precision = %5" )
.arg( QVariant::typeToName( nativeType.mType ) )
Expand Down
157 changes: 122 additions & 35 deletions tests/src/python/test_provider_memory.py
View file
Expand Up @@ -12,6 +12,7 @@

from urllib.parse import parse_qs

from qgis.PyQt.QtCore import QVariant, QByteArray, QDate, QDateTime, QTime
from qgis.core import (
QgsField,
QgsFields,
Expand All @@ -29,23 +30,19 @@
QgsRectangle,
QgsTestUtils,
QgsFeatureSource,
QgsProjUtils,
QgsFeatureSink,
)

from qgis.testing import (
start_app,
unittest
)

from providertestbase import ProviderTestCase
from utilities import (
unitTestDataPath,
compareWkt
)

from providertestbase import ProviderTestCase
from qgis.PyQt.QtCore import QVariant, QByteArray, QDate, QDateTime, QTime

start_app()
TEST_DATA_DIR = unitTestDataPath()

Expand All @@ -60,22 +57,30 @@ def createLayer(cls):
assert (vl.isValid())

f1 = QgsFeature()
f1.setAttributes([5, -200, NULL, 'NuLl', '5', QDateTime(QDate(2020, 5, 4), QTime(12, 13, 14)), QDate(2020, 5, 2), QTime(12, 13, 1)])
f1.setAttributes(
[5, -200, NULL, 'NuLl', '5', QDateTime(QDate(2020, 5, 4), QTime(12, 13, 14)), QDate(2020, 5, 2),
QTime(12, 13, 1)])
f1.setGeometry(QgsGeometry.fromWkt('Point (-71.123 78.23)'))

f2 = QgsFeature()
f2.setAttributes([3, 300, 'Pear', 'PEaR', '3', NULL, NULL, NULL])

f3 = QgsFeature()
f3.setAttributes([1, 100, 'Orange', 'oranGe', '1', QDateTime(QDate(2020, 5, 3), QTime(12, 13, 14)), QDate(2020, 5, 3), QTime(12, 13, 14)])
f3.setAttributes(
[1, 100, 'Orange', 'oranGe', '1', QDateTime(QDate(2020, 5, 3), QTime(12, 13, 14)), QDate(2020, 5, 3),
QTime(12, 13, 14)])
f3.setGeometry(QgsGeometry.fromWkt('Point (-70.332 66.33)'))

f4 = QgsFeature()
f4.setAttributes([2, 200, 'Apple', 'Apple', '2', QDateTime(QDate(2020, 5, 4), QTime(12, 14, 14)), QDate(2020, 5, 4), QTime(12, 14, 14)])
f4.setAttributes(
[2, 200, 'Apple', 'Apple', '2', QDateTime(QDate(2020, 5, 4), QTime(12, 14, 14)), QDate(2020, 5, 4),
QTime(12, 14, 14)])
f4.setGeometry(QgsGeometry.fromWkt('Point (-68.2 70.8)'))

f5 = QgsFeature()
f5.setAttributes([4, 400, 'Honey', 'Honey', '4', QDateTime(QDate(2021, 5, 4), QTime(13, 13, 14)), QDate(2021, 5, 4), QTime(13, 13, 14)])
f5.setAttributes(
[4, 400, 'Honey', 'Honey', '4', QDateTime(QDate(2021, 5, 4), QTime(13, 13, 14)), QDate(2021, 5, 4),
QTime(13, 13, 14)])
f5.setGeometry(QgsGeometry.fromWkt('Point (-65.32 78.3)'))

vl.dataProvider().addFeatures([f1, f2, f3, f4, f5])
Expand Down Expand Up @@ -264,24 +269,47 @@ def testGetFields(self):

provider.addAttributes([QgsField("name", QVariant.String),
QgsField("age", QVariant.Int),
QgsField("size", QVariant.Double)])
myMessage = ('Expected: %s\nGot: %s\n' %
(3, len(provider.fields())))

assert len(provider.fields()) == 3, myMessage
QgsField("size", QVariant.Double),
QgsField("vallist", QVariant.List, subType=QVariant.Int),
QgsField("stringlist", QVariant.List, subType=QVariant.String),
QgsField("reallist", QVariant.List, subType=QVariant.Double),
QgsField("longlist", QVariant.List, subType=QVariant.LongLong)])
self.assertEqual(len(provider.fields()), 7)
self.assertEqual(provider.fields()[0].name(), "name")
self.assertEqual(provider.fields()[0].type(), QVariant.String)
self.assertEqual(provider.fields()[0].subType(), QVariant.Invalid)
self.assertEqual(provider.fields()[1].name(), "age")
self.assertEqual(provider.fields()[1].type(), QVariant.Int)
self.assertEqual(provider.fields()[1].subType(), QVariant.Invalid)
self.assertEqual(provider.fields()[2].name(), "size")
self.assertEqual(provider.fields()[2].type(), QVariant.Double)
self.assertEqual(provider.fields()[2].subType(), QVariant.Invalid)
self.assertEqual(provider.fields()[3].name(), "vallist")
self.assertEqual(provider.fields()[3].type(), QVariant.List)
self.assertEqual(provider.fields()[3].subType(), QVariant.Int)
self.assertEqual(provider.fields()[4].name(), "stringlist")
self.assertEqual(provider.fields()[4].type(), QVariant.List)
self.assertEqual(provider.fields()[4].subType(), QVariant.String)
self.assertEqual(provider.fields()[5].name(), "reallist")
self.assertEqual(provider.fields()[5].type(), QVariant.List)
self.assertEqual(provider.fields()[5].subType(), QVariant.Double)
self.assertEqual(provider.fields()[6].name(), "longlist")
self.assertEqual(provider.fields()[6].type(), QVariant.List)
self.assertEqual(provider.fields()[6].subType(), QVariant.LongLong)

ft = QgsFeature()
ft.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(10, 10)))
ft.setAttributes(["Johny",
20,
0.3])
0.3,
[1, 2, 3],
['a', 'b', 'c'],
[1.1, 2.2, 3.3],
[1, 2, 3]])
provider.addFeatures([ft])

for f in provider.getFeatures(QgsFeatureRequest()):
myMessage = ('Expected: %s\nGot: %s\n' %
("Johny", f['name']))

self.assertEqual(f["name"], "Johny", myMessage)
self.assertEqual(f.attributes(), ['Johny', 20, 0.3, [1, 2, 3], ['a', 'b', 'c'], [1.1, 2.2, 3.3], [1, 2, 3]])

def testFromUri(self):
"""Test we can construct the mem provider from a uri"""
Expand All @@ -291,9 +319,9 @@ def testFromUri(self):
'test',
'memory')

assert myMemoryLayer is not None, 'Provider not initialized'
self.assertIsNotNone(myMemoryLayer)
myProvider = myMemoryLayer.dataProvider()
assert myProvider is not None
self.assertIsNotNone(myProvider)

def testLengthPrecisionFromUri(self):
"""Test we can assign length and precision from a uri"""
Expand All @@ -320,6 +348,40 @@ def testLengthPrecisionFromUri(self):

self.assertEqual(myMemoryLayer.fields().field('size').length(), -1)

def testListFromUri(self):
"""Test we can create list type fields from a uri"""
myMemoryLayer = QgsVectorLayer(
('Point?crs=epsg:4326&field=a:string(-1)[]&index=yes'),
'test',
'memory')

self.assertEqual(myMemoryLayer.fields().field('a').type(), QVariant.List)
self.assertEqual(myMemoryLayer.fields().field('a').subType(), QVariant.String)

myMemoryLayer = QgsVectorLayer(
('Point?crs=epsg:4326&field=a:double(-1,-1)[]&index=yes'),
'test',
'memory')

self.assertEqual(myMemoryLayer.fields().field('a').type(), QVariant.List)
self.assertEqual(myMemoryLayer.fields().field('a').subType(), QVariant.Double)

myMemoryLayer = QgsVectorLayer(
('Point?crs=epsg:4326&field=a:long(-1,-1)[]&index=yes'),
'test',
'memory')

self.assertEqual(myMemoryLayer.fields().field('a').type(), QVariant.List)
self.assertEqual(myMemoryLayer.fields().field('a').subType(), QVariant.LongLong)

myMemoryLayer = QgsVectorLayer(
('Point?crs=epsg:4326&field=a:int(-1,-1)[]&index=yes'),
'test',
'memory')

self.assertEqual(myMemoryLayer.fields().field('a').type(), QVariant.List)
self.assertEqual(myMemoryLayer.fields().field('a').subType(), QVariant.Int)

def testFromUriWithEncodedField(self):
"""Test we can construct the mem provider from a uri when a field name is encoded"""
layer = QgsVectorLayer(
Expand All @@ -344,28 +406,38 @@ def testSaveFields(self):
QgsField('TestString', QVariant.String, 'string', 50, 0),
QgsField('TestDate', QVariant.Date, 'date'),
QgsField('TestTime', QVariant.Time, 'time'),
QgsField('TestDateTime', QVariant.DateTime, 'datetime')]
assert myMemoryLayer.startEditing()
QgsField('TestDateTime', QVariant.DateTime, 'datetime'),
QgsField("vallist", QVariant.List, subType=QVariant.Int),
QgsField("stringlist", QVariant.List, subType=QVariant.String),
QgsField("reallist", QVariant.List, subType=QVariant.Double),
QgsField("longlist", QVariant.List, subType=QVariant.LongLong)]
self.assertTrue(myMemoryLayer.startEditing())
for f in myFields:
assert myMemoryLayer.addAttribute(f)
assert myMemoryLayer.commitChanges()
self.assertTrue(myMemoryLayer.commitChanges())
myMemoryLayer.updateFields()

for f in myFields:
self.assertEqual(f, myMemoryLayer.fields().field(f.name()))

# Export the layer to a layer-definition-XML
qlr = QgsLayerDefinition.exportLayerDefinitionLayers([myMemoryLayer], QgsReadWriteContext())
assert qlr is not None
self.assertIsNotNone(qlr)

# Import the layer from the layer-definition-XML
layers = QgsLayerDefinition.loadLayerDefinitionLayers(qlr, QgsReadWriteContext())
assert layers is not None
self.assertTrue(layers)
myImportedLayer = layers[0]
assert myImportedLayer is not None
self.assertIsNotNone(myImportedLayer)

# Check for the presence of the fields
importedFields = myImportedLayer.fields()
assert importedFields is not None
for f in myFields:
assert f == importedFields.field(f.name())
self.assertEqual(f.name(), importedFields.field(f.name()).name())
self.assertEqual(f.type(), importedFields.field(f.name()).type())
self.assertEqual(f.subType(), importedFields.field(f.name()).subType())
self.assertEqual(f.precision(), importedFields.field(f.name()).precision())
self.assertEqual(f.length(), importedFields.field(f.name()).length())

def testRenameAttributes(self):
layer = QgsVectorLayer("Point", "test", "memory")
Expand Down Expand Up @@ -446,11 +518,13 @@ def testCreateMemoryLayer(self):
self.assertEqual(layer.crs().authid(), 'EPSG:3111')

# custom CRS
crs = QgsCoordinateReferenceSystem.fromProj('+proj=qsc +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs')
crs = QgsCoordinateReferenceSystem.fromProj(
'+proj=qsc +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs')
layer = QgsMemoryProviderUtils.createMemoryLayer('my name', QgsFields(), QgsWkbTypes.PolygonZM, crs)
self.assertTrue(layer.isValid())
self.assertTrue(layer.crs().isValid())
self.assertEqual(layer.crs().toProj(), '+proj=qsc +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs +type=crs')
self.assertEqual(layer.crs().toProj(),
'+proj=qsc +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs +type=crs')

# clone it, just to check
layer2 = layer.clone()
Expand All @@ -470,6 +544,11 @@ def testCreateMemoryLayer(self):
fields.append(QgsField("complex/name", QVariant.String))
fields.append(QgsField("binaryfield", QVariant.ByteArray))
fields.append(QgsField("boolfield", QVariant.Bool))
fields.append(QgsField("vallist", QVariant.List, subType=QVariant.Int))
fields.append(QgsField("stringlist", QVariant.List, subType=QVariant.String))
fields.append(QgsField("reallist", QVariant.List, subType=QVariant.Double))
fields.append(QgsField("longlist", QVariant.List, subType=QVariant.LongLong))

layer = QgsMemoryProviderUtils.createMemoryLayer('my name', fields)
self.assertTrue(layer.isValid())
self.assertFalse(layer.fields().isEmpty())
Expand Down Expand Up @@ -840,22 +919,30 @@ def setUpClass(cls):
cls.source = cls.vl.dataProvider()

f1 = QgsFeature()
f1.setAttributes([5, -200, NULL, 'NuLl', '5', QDateTime(QDate(2020, 5, 4), QTime(12, 13, 14)), QDate(2020, 5, 2), QTime(12, 13, 1)])
f1.setAttributes(
[5, -200, NULL, 'NuLl', '5', QDateTime(QDate(2020, 5, 4), QTime(12, 13, 14)), QDate(2020, 5, 2),
QTime(12, 13, 1)])
f1.setGeometry(QgsGeometry.fromWkt('Point (-71.123 78.23)'))

f2 = QgsFeature()
f2.setAttributes([3, 300, 'Pear', 'PEaR', '3', NULL, NULL, NULL])

f3 = QgsFeature()
f3.setAttributes([1, 100, 'Orange', 'oranGe', '1', QDateTime(QDate(2020, 5, 3), QTime(12, 13, 14)), QDate(2020, 5, 3), QTime(12, 13, 14)])
f3.setAttributes(
[1, 100, 'Orange', 'oranGe', '1', QDateTime(QDate(2020, 5, 3), QTime(12, 13, 14)), QDate(2020, 5, 3),
QTime(12, 13, 14)])
f3.setGeometry(QgsGeometry.fromWkt('Point (-70.332 66.33)'))

f4 = QgsFeature()
f4.setAttributes([2, 200, 'Apple', 'Apple', '2', QDateTime(QDate(2020, 5, 4), QTime(12, 14, 14)), QDate(2020, 5, 4), QTime(12, 14, 14)])
f4.setAttributes(
[2, 200, 'Apple', 'Apple', '2', QDateTime(QDate(2020, 5, 4), QTime(12, 14, 14)), QDate(2020, 5, 4),
QTime(12, 14, 14)])
f4.setGeometry(QgsGeometry.fromWkt('Point (-68.2 70.8)'))

f5 = QgsFeature()
f5.setAttributes([4, 400, 'Honey', 'Honey', '4', QDateTime(QDate(2021, 5, 4), QTime(13, 13, 14)), QDate(2021, 5, 4), QTime(13, 13, 14)])
f5.setAttributes(
[4, 400, 'Honey', 'Honey', '4', QDateTime(QDate(2021, 5, 4), QTime(13, 13, 14)), QDate(2021, 5, 4),
QTime(13, 13, 14)])
f5.setGeometry(QgsGeometry.fromWkt('Point (-65.32 78.3)'))

cls.source.addFeatures([f1, f2, f3, f4, f5])
Expand Down

0 comments on commit 2f8adf7

Please sign in to comment.