Skip to content

Commit

Permalink
JSON Support for GeoPackage [FEATURE]
Browse files Browse the repository at this point in the history 
Merge pull request #8707 from signedav/json-for-gpkg
Integration of JSON type of GeoPackage
  • Loading branch information
@m-kuhn
m-kuhn committed Jan 7, 2019
2 parents 69f6ea5 + a6f8db1 commit 0c71e72
Show file tree
Hide file tree
Showing 9 changed files with 162 additions and 11 deletions.
5 changes: 4 additions & 1 deletion src/app/qgsaddattrdialog.cpp
View file
Expand Up @@ -111,6 +111,7 @@ void QgsAddAttrDialog::accept()

QgsField QgsAddAttrDialog::field() const
{

QgsDebugMsg( QStringLiteral( "idx:%1 name:%2 type:%3 typeName:%4 length:%5 prec:%6 comment:%7" )
.arg( mTypeBox->currentIndex() )
.arg( mNameEdit->text() )
Expand All @@ -126,5 +127,7 @@ QgsField QgsAddAttrDialog::field() const
mTypeBox->currentData( Qt::UserRole + 1 ).toString(),
mLength->value(),
mPrec->value(),
mCommentEdit->text() );
mCommentEdit->text(),
static_cast<QVariant::Type>( mTypeBox->currentData( Qt::UserRole ).toInt() ) == QVariant::Map ? QVariant::String : QVariant::Invalid
);
}
2 changes: 1 addition & 1 deletion src/core/qgsfield.cpp
View file
Expand Up @@ -254,7 +254,7 @@ QString QgsField::displayString( const QVariant &v ) const
if ( ok )
return QLocale().toString( converted );
}
else if ( d->typeName == QLatin1String( "json" ) || d->typeName == QLatin1String( "jsonb" ) )
else if ( d->typeName.compare( QLatin1String( "json" ), Qt::CaseInsensitive ) == 0 || d->typeName == QLatin1String( "jsonb" ) )
{
QJsonDocument doc = QJsonDocument::fromVariant( v );
return QString::fromUtf8( doc.toJson().data() );
Expand Down
18 changes: 18 additions & 0 deletions src/core/qgsogrutils.cpp
View file
Expand Up @@ -22,6 +22,7 @@
#include <QTextCodec>
#include <QUuid>
#include <cpl_error.h>
#include <QJsonDocument>

// Starting with GDAL 2.2, there are 2 concepts: unset fields and null fields
// whereas previously there was only unset fields. For QGIS purposes, both
Expand Down Expand Up @@ -151,6 +152,13 @@ QgsFields QgsOgrUtils::readOgrFields( OGRFeatureH ogrFet, QTextCodec *encoding )
varType = QVariant::DateTime;
break;
case OFTString:
#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
if ( OGR_Fld_GetSubType( fldDef ) == OFSTJSON )
varType = QVariant::Map;
else
varType = QVariant::String;
break;
#endif
default:
varType = QVariant::String; // other unsupported, leave it as a string
}
Expand Down Expand Up @@ -238,6 +246,16 @@ QVariant QgsOgrUtils::getOgrFeatureAttribute( OGRFeatureH ogrFet, const QgsField
break;
}

case QVariant::Map:
{
//it has to be JSON
//it's null if no json format
if ( encoding )
value = QJsonDocument::fromJson( encoding->toUnicode( OGR_F_GetFieldAsString( ogrFet, attIndex ) ).toUtf8() ).toVariant();
else
value = QJsonDocument::fromJson( QString::fromUtf8( OGR_F_GetFieldAsString( ogrFet, attIndex ) ).toUtf8() ).toVariant();
break;
}
default:
Q_ASSERT_X( false, "QgsOgrUtils::getOgrFeatureAttribute", "unsupported field type" );
if ( ok )
Expand Down
2 changes: 1 addition & 1 deletion src/gui/editorwidgets/qgslistwidgetfactory.cpp
View file
Expand Up @@ -44,5 +44,5 @@ QgsEditorConfigWidget *QgsListWidgetFactory::configWidget( QgsVectorLayer *vl, i
unsigned int QgsListWidgetFactory::fieldScore( const QgsVectorLayer *vl, int fieldIdx ) const
{
const QgsField field = vl->fields().field( fieldIdx );
return ( field.type() == QVariant::List || field.type() == QVariant::StringList || field.typeName() == QLatin1String( "json" ) || field.typeName() == QLatin1String( "jsonb" ) ) && field.subType() != QVariant::Invalid ? 20 : 0;
return ( field.type() == QVariant::List || field.type() == QVariant::StringList || field.type() == QVariant::Map ) && field.subType() != QVariant::Invalid ? 20 : 0;
}
76 changes: 70 additions & 6 deletions src/providers/ogr/qgsogrprovider.cpp
View file
Expand Up @@ -155,6 +155,12 @@ bool QgsOgrProvider::convertField( QgsField &field, const QTextCodec &encoding )
ogrType = OFTDateTime;
break;
#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
case QVariant::Map:
ogrType = OFTString;
ogrSubType = OFSTJSON;
break;
#endif
default:
return false;
}
Expand Down Expand Up @@ -472,6 +478,9 @@ QgsOgrProvider::QgsOgrProvider( QString const &uri, const ProviderOptions &optio
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (real)" ), QStringLiteral( "double" ), QVariant::Double, 0, nMaxDoubleLen, 0, nMaxDoublePrec )
<< QgsVectorDataProvider::NativeType( tr( "Text (string)" ), QStringLiteral( "string" ), QVariant::String, 0, 65535 );

if ( mGDALDriverName == QLatin1String( "GPKG" ) )
nativeTypes << QgsVectorDataProvider::NativeType( tr( "JSON (string)" ), QStringLiteral( "JSON" ), QVariant::Map, 0, 0, 0, 0, QVariant::String );

bool supportsDate = true;
bool supportsTime = mGDALDriverName != QLatin1String( "ESRI Shapefile" ) && mGDALDriverName != QLatin1String( "GPKG" );
bool supportsDateTime = mGDALDriverName != QLatin1String( "ESRI Shapefile" );
Expand Down Expand Up @@ -1009,6 +1018,7 @@ void QgsOgrProvider::loadFields()
OGRFieldSubType ogrSubType = OFSTNone;

QVariant::Type varType;
QVariant::Type varSubType = QVariant::Invalid;
switch ( ogrType )
{
case OFTInteger:
Expand Down Expand Up @@ -1041,6 +1051,19 @@ void QgsOgrProvider::loadFields()
break;

case OFTString:
#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
if ( OGR_Fld_GetSubType( fldDef ) == OFSTJSON )
{
ogrSubType = OFSTJSON;
varType = QVariant::Map;
varSubType = QVariant::String;
}
else
{
varType = QVariant::String;
}
break;
#endif
default:
varType = QVariant::String; // other unsupported, leave it as a string
}
Expand Down Expand Up @@ -1081,7 +1104,7 @@ void QgsOgrProvider::loadFields()
#else
textEncoding()->toUnicode( typeName.toStdString().c_str() ),
#endif
width, prec
width, prec, QString(), varSubType
);

// check if field is nullable
Expand Down Expand Up @@ -1385,6 +1408,17 @@ OGRGeometryH QgsOgrProvider::ConvertGeometryIfNecessary( OGRGeometryH hGeom )
return OGR_G_ForceTo( hGeom, layerGeomType, nullptr );
}

QString QgsOgrProvider::jsonStringValue( const QVariant &value ) const
{
QString stringValue = QString::fromUtf8( QJsonDocument::fromVariant( value ).toJson().constData() );
if ( stringValue.isEmpty() )
{
//store as string, because it's no valid QJson value
stringValue = value.toString();
}
return stringValue;
}

bool QgsOgrProvider::addFeaturePrivate( QgsFeature &f, Flags flags )
{
bool returnValue = true;
Expand Down Expand Up @@ -1509,13 +1543,26 @@ bool QgsOgrProvider::addFeaturePrivate( QgsFeature &f, Flags flags )
break;

case OFTString:
{
QString stringValue;

#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
if ( OGR_Fld_GetSubType( fldDef ) == OFSTJSON )
stringValue = jsonStringValue( attrVal );
else
{
stringValue = attrVal.toString();
}
#else
stringValue = attrVal.toString();
#endif
QgsDebugMsgLevel( QStringLiteral( "Writing string attribute %1 with %2, encoding %3" )
.arg( qgisAttId )
.arg( attrVal.toString(),
textEncoding()->name().data() ), 3 );
OGR_F_SetFieldString( feature.get(), ogrAttId, textEncoding()->fromUnicode( attrVal.toString() ).constData() );
OGR_F_SetFieldString( feature.get(), ogrAttId, textEncoding()->fromUnicode( stringValue ).constData() );
break;

}
case OFTBinary:
{
const QByteArray ba = attrVal.toByteArray();
Expand Down Expand Up @@ -1633,7 +1680,9 @@ bool QgsOgrProvider::addAttributeOGRLevel( const QgsField &field, bool &ignoreEr
case QVariant::ByteArray:
type = OFTBinary;
break;

case QVariant::Map:
type = OFTString;
break;
default:
pushError( tr( "type %1 for field %2 not found" ).arg( field.typeName(), field.name() ) );
ignoreErrorOut = true;
Expand All @@ -1652,7 +1701,11 @@ bool QgsOgrProvider::addAttributeOGRLevel( const QgsField &field, bool &ignoreEr
case QVariant::Bool:
OGR_Fld_SetSubType( fielddefn.get(), OFSTBoolean );
break;

#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
case QVariant::Map:
OGR_Fld_SetSubType( fielddefn.get(), OFSTJSON );
break;
#endif
default:
break;
}
Expand Down Expand Up @@ -2080,8 +2133,19 @@ bool QgsOgrProvider::changeAttributeValues( const QgsChangedAttributesMap &attr_
0 );
break;
case OFTString:
OGR_F_SetFieldString( of.get(), f, textEncoding()->fromUnicode( it2->toString() ).constData() );
{
QString stringValue;
#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(2,4,0)
if ( OGR_Fld_GetSubType( fd ) == OFSTJSON )
stringValue = jsonStringValue( it2.value() );
else
stringValue = it2->toString();
#else
stringValue = it2->toString();
#endif
OGR_F_SetFieldString( of.get(), f, textEncoding()->fromUnicode( stringValue ).constData() );
break;
}

case OFTBinary:
{
Expand Down
3 changes: 3 additions & 0 deletions src/providers/ogr/qgsogrprovider.h
View file
Expand Up @@ -284,6 +284,9 @@ class QgsOgrProvider : public QgsVectorDataProvider

mutable QStringList mSubLayerList;

//! converts \a value from json QVariant to QString
QString jsonStringValue( const QVariant &value ) const;

bool addFeaturePrivate( QgsFeature &f, QgsFeatureSink::Flags flags );
//! Deletes one feature
bool deleteFeature( QgsFeatureId id );
Expand Down
4 changes: 2 additions & 2 deletions src/providers/postgres/qgspostgresprovider.cpp
View file
Expand Up @@ -243,11 +243,11 @@ QgsPostgresProvider::QgsPostgresProvider( QString const &uri, const ProviderOpti

if ( connectionRO()->pgVersion() >= 90200 )
{
nativeTypes << QgsVectorDataProvider::NativeType( tr( "Map (json)" ), QStringLiteral( "json" ), QVariant::Map, -1, -1, -1, -1, QVariant::String );
nativeTypes << QgsVectorDataProvider::NativeType( tr( "JSON (json)" ), QStringLiteral( "json" ), QVariant::Map, -1, -1, -1, -1, QVariant::String );

if ( connectionRO()->pgVersion() >= 90400 )
{
nativeTypes << QgsVectorDataProvider::NativeType( tr( "Map (jsonb)" ), QStringLiteral( "jsonb" ), QVariant::Map, -1, -1, -1, -1, QVariant::String );
nativeTypes << QgsVectorDataProvider::NativeType( tr( "JSON (jsonb)" ), QStringLiteral( "jsonb" ), QVariant::Map, -1, -1, -1, -1, QVariant::String );
}
}
setNativeTypes( nativeTypes );
Expand Down
63 changes: 63 additions & 0 deletions tests/src/python/test_provider_ogr_gpkg.py
View file
Expand Up @@ -1230,6 +1230,69 @@ def testTransaction(self):
self.assertEqual(len([f for f in vl2_external.getFeatures(QgsFeatureRequest())]), 1)
del vl2_external

def testJson(self):
if int(gdal.VersionInfo('VERSION_NUM')) < GDAL_COMPUTE_VERSION(2, 4, 0):
return

tmpfile = os.path.join(self.basetestpath, 'test_json.gpkg')
testdata_path = unitTestDataPath('provider')
shutil.copy(os.path.join(unitTestDataPath('provider'), 'test_json.gpkg'), tmpfile)

vl = QgsVectorLayer('{}|layerid=0'.format(tmpfile, 'foo', 'ogr'))
self.assertTrue(vl.isValid())

fields = vl.dataProvider().fields()
self.assertEqual(fields.at(fields.indexFromName('json_content')).type(), QVariant.Map)

fi = vl.getFeatures(QgsFeatureRequest())
f = QgsFeature()

#test reading dict value from attribute
while fi.nextFeature(f):
if f['fid'] == 1:
self.assertIsInstance(f['json_content'], dict)
self.assertEqual(f['json_content'], {'foo': 'bar'})
#test changing dict value in attribute
f['json_content'] = {'foo': 'baz'}
self.assertEqual(f['json_content'], {'foo': 'baz'})
#test changint dict to list
f['json_content'] = ['eins', 'zwei', 'drei']
self.assertEqual(f['json_content'], ['eins', 'zwei', 'drei'])
#test changing list value in attribute
f['json_content'] = ['eins', 'zwei', 'drei', 4]
self.assertEqual(f['json_content'], ['eins', 'zwei', 'drei', 4])
#test changing to complex json structure
f['json_content'] = {'name': 'Lily', 'age': '0', 'cars': {'car1': ['fiat tipo', 'fiat punto', 'davoser schlitten'], 'car2': 'bobbycar', 'car3': 'tesla'}}
self.assertEqual(f['json_content'], {'name': 'Lily', 'age': '0', 'cars': {'car1': ['fiat tipo', 'fiat punto', 'davoser schlitten'], 'car2': 'bobbycar', 'car3': 'tesla'}})

#test adding attribute
vl.startEditing()
self.assertTrue(vl.addAttribute(QgsField('json_content2', QVariant.Map, "JSON", 60, 0, 'no comment', QVariant.String)))
self.assertTrue(vl.commitChanges())

vl.startEditing()
self.assertTrue(vl.addAttribute(QgsField('json_content3', QVariant.Map, "JSON", 60, 0, 'no comment', QVariant.String)))
self.assertTrue(vl.commitChanges())

#test setting values to new attributes
while fi.nextFeature(f):
if f['fid'] == 2:
f['json_content'] = {'uno': 'foo'}
f['json_content2'] = ['uno', 'due', 'tre']
f['json_content3'] = {'uno': ['uno', 'due', 'tre']}
self.assertEqual(f['json_content'], {'foo': 'baz'})
self.assertEqual(f['json_content2'], ['uno', 'due', 'tre'])
self.assertEqual(f['json_content3'], {'uno': ['uno', 'due', 'tre']})

#test deleting attribute
vl.startEditing()
self.assertTrue(vl.deleteAttribute(vl.fields().indexFromName('json_content3')))
self.assertTrue(vl.commitChanges())

#test if index of existent field is not -1 and the one of the deleted is -1
self.assertNotEqual(vl.fields().indexFromName('json_content2'), -1)
self.assertEqual(vl.fields().indexFromName('json_content3'), -1)


if __name__ == '__main__':
unittest.main()
Binary file added tests/testdata/provider/test_json.gpkg
View file
Binary file not shown.

0 comments on commit 0c71e72

Please sign in to comment.