Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #44348 from domi4484/backport-44099-to-release-3_20_a
Backport "Added missing type names to uri parsing regex for memory provider" 3.20
  • Loading branch information
m-kuhn committed Aug 10, 2021
2 parents e80964c + fad0375 commit c4b2ca9
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 88 deletions.
186 changes: 102 additions & 84 deletions src/core/providers/memory/qgsmemoryprovider.cpp
Expand Up @@ -25,7 +25,7 @@

#include <QUrl>
#include <QUrlQuery>
#include <QRegExp>
#include <QRegularExpression>

///@cond PRIVATE

Expand Down Expand Up @@ -103,7 +103,7 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "text" ), QVariant::String, -1, -1, -1, -1 )

// boolean
<< QgsVectorDataProvider::NativeType( tr( "Boolean" ), QStringLiteral( "bool" ), QVariant::Bool )
<< QgsVectorDataProvider::NativeType( tr( "Boolean" ), QStringLiteral( "boolean" ), QVariant::Bool )

// blob
<< QgsVectorDataProvider::NativeType( tr( "Binary object (BLOB)" ), QStringLiteral( "binary" ), QVariant::ByteArray )
Expand All @@ -119,95 +119,101 @@ QgsMemoryProvider::QgsMemoryProvider( const QString &uri, const ProviderOptions
if ( query.hasQueryItem( QStringLiteral( "field" ) ) )
{
QList<QgsField> attributes;
QRegExp reFieldDef( "\\:"
"(int|integer|long|int8|real|double|string|date|time|datetime|binary|bool|boolean)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$", Qt::CaseInsensitive );
const thread_local QRegularExpression reFieldDef( "\\:"
"([\\w\\s]+)" // type
"(?:\\((\\-?\\d+)" // length
"(?:\\,(\\-?\\d+))?" // precision
"\\))?(\\[\\])?" // array
"$",
QRegularExpression::CaseInsensitiveOption );
QStringList fields = query.allQueryItemValues( QStringLiteral( "field" ) );
for ( int i = 0; i < fields.size(); i++ )
{
QString name = QUrl::fromPercentEncoding( fields.at( i ).toUtf8() );
QRegularExpressionMatch regularExpressionMatch = reFieldDef.match( name );

// If no match -> use string as type
QVariant::Type type = QVariant::String;
QVariant::Type subType = QVariant::Invalid;
QString typeName( QStringLiteral( "string" ) );
int length = 255;
int precision = 0;

int pos = reFieldDef.indexIn( name );
if ( pos >= 0 )
if ( regularExpressionMatch.hasMatch() )
{
name = name.mid( 0, pos );
typeName = reFieldDef.cap( 1 ).toLower();
if ( typeName == QLatin1String( "int" ) || typeName == QLatin1String( "integer" ) )
name = name.mid( 0, regularExpressionMatch.capturedStart() );
typeName = regularExpressionMatch.captured( 1 ).toLower();

// Search typeName correspondence in native types
bool isNativeType = false;
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
{
type = QVariant::Int;
typeName = QStringLiteral( "integer" );
length = -1;
if ( nativeType.mTypeName.toLower() == typeName )
{
isNativeType = true;
type = nativeType.mType;
subType = nativeType.mSubType;
typeName = nativeType.mTypeName;
break;
}
}
else if ( typeName == QLatin1String( "int8" ) || typeName == QLatin1String( "long" ) )

// Not a native type -> check other supported types:
if ( isNativeType == false )
{
type = QVariant::LongLong;
typeName = QStringLiteral( "int8" );
length = -1;
if ( typeName == QLatin1String( "int" ) )
{
type = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( typeName == QLatin1String( "long" ) )
{
type = QVariant::LongLong;
typeName = QStringLiteral( "int8" );
}
else if ( typeName == QLatin1String( "bool" ) )
{
type = QVariant::Bool;
typeName = QStringLiteral( "boolean" );
}
else
{
QgsLogger::warning( tr( "Unsupported typeName '%1'. Will be handled as string." ).arg( typeName ) );
type = QVariant::String;
typeName = QStringLiteral( "string" );
}
}
else if ( typeName == QLatin1String( "real" ) || typeName == QLatin1String( "double" ) )

// Set default length/precision for double/real
if ( typeName == QLatin1String( "real" ) || typeName == QLatin1String( "double" ) )
{
type = QVariant::Double;
typeName = QStringLiteral( "double" );
length = 20;
precision = 5;
}
else if ( typeName == QLatin1String( "date" ) )
{
type = QVariant::Date;
typeName = QStringLiteral( "date" );
length = -1;
}
else if ( typeName == QLatin1String( "time" ) )
{
type = QVariant::Time;
typeName = QStringLiteral( "time" );
length = -1;
}
else if ( typeName == QLatin1String( "datetime" ) )
{
type = QVariant::DateTime;
typeName = QStringLiteral( "datetime" );
length = -1;
}
else if ( typeName == QLatin1String( "bool" ) || typeName == QLatin1String( "boolean" ) )
{
type = QVariant::Bool;
typeName = QStringLiteral( "boolean" );
length = -1;
}
else if ( typeName == QLatin1String( "binary" ) )
{
type = QVariant::ByteArray;
typeName = QStringLiteral( "binary" );
length = -1;
}

if ( !reFieldDef.cap( 2 ).isEmpty() )
{
length = reFieldDef.cap( 2 ).toInt();
}
if ( !reFieldDef.cap( 3 ).isEmpty() )
{
precision = reFieldDef.cap( 3 ).toInt();
}
if ( !reFieldDef.cap( 4 ).isEmpty() )
if ( !regularExpressionMatch.captured( 2 ).isEmpty() )
length = regularExpressionMatch.captured( 2 ).toInt();

if ( !regularExpressionMatch.captured( 3 ).isEmpty() )
precision = regularExpressionMatch.captured( 3 ).toInt();

// Array
if ( !regularExpressionMatch.captured( 4 ).isEmpty() )
{
//array
subType = type;
type = type == QVariant::String ? QVariant::StringList : QVariant::List;
typeName += QLatin1String( "list" );
if ( subType == QVariant::Invalid )
subType = type;

if ( type != QVariant::List && type != QVariant::StringList )
type = type == QVariant::String ? QVariant::StringList : QVariant::List;

const QLatin1String listSuffix( "list" );
if ( !typeName.endsWith( listSuffix ) )
typeName += QLatin1String( "list" );
}
}
if ( !name.isEmpty() )
attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );

attributes.append( QgsField( name, type, typeName, length, precision, QString(), subType ) );
}
addAttributes( attributes );
}
Expand Down Expand Up @@ -556,28 +562,40 @@ bool QgsMemoryProvider::deleteFeatures( const QgsFeatureIds &id )

bool QgsMemoryProvider::addAttributes( const QList<QgsField> &attributes )
{
for ( QList<QgsField>::const_iterator it = attributes.begin(); it != attributes.end(); ++it )
for ( QgsField field : attributes )
{
switch ( it->type() )
if ( !supportedType( field ) )
continue;

// Make sure added attributes typeName correspond to a native type name
bool isNativeTypeName = false;
NativeType nativeTypeCandidate( QString(), QString(), QVariant::Invalid );
const QList<QgsVectorDataProvider::NativeType> nativeTypesList( nativeTypes() );
for ( const NativeType &nativeType : nativeTypesList )
{
case QVariant::Int:
case QVariant::Double:
case QVariant::String:
case QVariant::Date:
case QVariant::Time:
case QVariant::DateTime:
case QVariant::LongLong:
case QVariant::StringList:
case QVariant::List:
case QVariant::Bool:
case QVariant::ByteArray:
if ( nativeType.mTypeName.toLower() == field.typeName().toLower() )
{
isNativeTypeName = true;
break;
default:
QgsDebugMsg( "Field type not supported: " + it->typeName() );
}

if ( nativeType.mType == field.type()
&& nativeTypeCandidate.mType == QVariant::Invalid )
nativeTypeCandidate = nativeType;
}
if ( !isNativeTypeName )
{
if ( nativeTypeCandidate.mType == QVariant::Invalid )
{
QgsLogger::warning( "Field type not supported: " + field.typeName() );
continue;
}

field.setTypeName( nativeTypeCandidate.mTypeName );
}

// add new field as a last one
mFields.append( *it );
mFields.append( field );

for ( QgsFeatureMap::iterator fit = mFeatures.begin(); fit != mFeatures.end(); ++fit )
{
Expand Down
2 changes: 1 addition & 1 deletion src/core/qgsfield.cpp
Expand Up @@ -312,7 +312,7 @@ QString QgsField::displayString( const QVariant &v ) const
}
}
// Default for doubles with precision
else if ( d->type == QVariant::Double && d->precision > 0 )
else if ( d->precision > 0 )
{
if ( -1 < v.toDouble() && v.toDouble() < 1 )
{
Expand Down
65 changes: 63 additions & 2 deletions tests/src/python/test_provider_memory.py
Expand Up @@ -784,7 +784,7 @@ def testSpatialIndex(self):
vl.dataProvider().createSpatialIndex()
self.assertEqual(vl.hasSpatialIndex(), QgsFeatureSource.SpatialIndexPresent)

def testClone(self):
def testCloneId(self):
"""Test that a cloned layer has a single new id and
the same fields as the source layer"""

Expand All @@ -798,7 +798,7 @@ def testClone(self):
QgsField("size", QVariant.Double)]))
vl2 = vl.clone()
self.assertTrue(
'memory?geometry=Point&crs=EPSG:4326&field=name:(0,0)&field=age:(0,0)&field=size:(0,0)' in vl2.publicSource())
'memory?geometry=Point&crs=EPSG:4326&field=name:string(0,0)&field=age:integer(0,0)&field=size:double(0,0)' in vl2.publicSource())
self.assertEqual(len(parse_qs(vl.publicSource())['uid']), 1)
self.assertEqual(len(parse_qs(vl2.publicSource())['uid']), 1)
self.assertNotEqual(parse_qs(vl2.publicSource())['uid'][0], parse_qs(vl.publicSource())['uid'][0])
Expand Down Expand Up @@ -915,6 +915,67 @@ def testTypeValidation(self):
f = vl.getFeature(1)
self.assertEqual(f.attribute('int'), 123)

def testAddAttributes(self):
"""Test that fields with empty/invalid typenames are updated to native type names"""

vl = QgsVectorLayer("Point", "temporary_points", "memory")
pr = vl.dataProvider()

# add fields
pr.addAttributes([QgsField("name", QVariant.String),
QgsField("age", QVariant.Int, "invalidInteger"),
QgsField("size", QVariant.Double),
QgsField("mytext", QVariant.String, "text"),
QgsField("size2", QVariant.Double, "double precision"),
QgsField("short", QVariant.Int, "int2"),
QgsField("lessshort", QVariant.Int, "int4"),
QgsField("numericfield", QVariant.Double, "numeric"),
QgsField("decimalfield", QVariant.Double, "decimal"),
QgsField("stringlistfield", QVariant.StringList, "stringlist"),
QgsField("integerlistfield", QVariant.List, "integerlist"),
QgsField("doublelistfield", QVariant.List, "doublelist")])

self.assertEqual(pr.fields()[0].typeName(), "string")
self.assertEqual(pr.fields()[1].typeName(), "integer")
self.assertEqual(pr.fields()[2].typeName(), "double")
self.assertEqual(pr.fields()[3].typeName(), "text")
self.assertEqual(pr.fields()[4].typeName(), "double precision")
self.assertEqual(pr.fields()[5].typeName(), "int2")
self.assertEqual(pr.fields()[6].typeName(), "int4")
self.assertEqual(pr.fields()[7].typeName(), "numeric")
self.assertEqual(pr.fields()[8].typeName(), "decimal")
self.assertEqual(pr.fields()[9].typeName(), "stringlist")
self.assertEqual(pr.fields()[10].typeName(), "integerlist")
self.assertEqual(pr.fields()[11].typeName(), "doublelist")

vl2 = vl.clone()

self.assertEqual(pr.fields()[0].name(), vl2.fields()[0].name())
self.assertEqual(pr.fields()[1].name(), vl2.fields()[1].name())
self.assertEqual(pr.fields()[2].name(), vl2.fields()[2].name())
self.assertEqual(pr.fields()[3].name(), vl2.fields()[3].name())
self.assertEqual(pr.fields()[4].name(), vl2.fields()[4].name())
self.assertEqual(pr.fields()[5].name(), vl2.fields()[5].name())
self.assertEqual(pr.fields()[6].name(), vl2.fields()[6].name())
self.assertEqual(pr.fields()[7].name(), vl2.fields()[7].name())
self.assertEqual(pr.fields()[8].name(), vl2.fields()[8].name())
self.assertEqual(pr.fields()[9].name(), vl2.fields()[9].name())
self.assertEqual(pr.fields()[10].name(), vl2.fields()[10].name())
self.assertEqual(pr.fields()[11].name(), vl2.fields()[11].name())

self.assertEqual(pr.fields()[0].typeName(), vl2.fields()[0].typeName())
self.assertEqual(pr.fields()[1].typeName(), vl2.fields()[1].typeName())
self.assertEqual(pr.fields()[2].typeName(), vl2.fields()[2].typeName())
self.assertEqual(pr.fields()[3].typeName(), vl2.fields()[3].typeName())
self.assertEqual(pr.fields()[4].typeName(), vl2.fields()[4].typeName())
self.assertEqual(pr.fields()[5].typeName(), vl2.fields()[5].typeName())
self.assertEqual(pr.fields()[6].typeName(), vl2.fields()[6].typeName())
self.assertEqual(pr.fields()[7].typeName(), vl2.fields()[7].typeName())
self.assertEqual(pr.fields()[8].typeName(), vl2.fields()[8].typeName())
self.assertEqual(pr.fields()[9].typeName(), vl2.fields()[9].typeName())
self.assertEqual(pr.fields()[10].typeName(), vl2.fields()[10].typeName())
self.assertEqual(pr.fields()[11].typeName(), vl2.fields()[11].typeName())


class TestPyQgsMemoryProviderIndexed(unittest.TestCase, ProviderTestCase):
"""Runs the provider test suite against an indexed memory layer"""
Expand Down
2 changes: 1 addition & 1 deletion tests/src/python/test_qgsauxiliarystorage.py
Expand Up @@ -550,7 +550,7 @@ def testInvalidPrimaryKey(self):

# create auxiliary storage based on the invalid field
s = QgsAuxiliaryStorage()
pkf = vl.fields().field(vl.fields().indexOf('invalid_pk'))
pkf = field
al = s.createAuxiliaryLayer(pkf, vl)

self.assertEqual(al, None)
Expand Down

0 comments on commit c4b2ca9

Please sign in to comment.