Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[ogr][gdal] Proper vsi{zip,tar,gzip} uri decoding/encoding
  • Loading branch information
nirvn committed Mar 16, 2021
1 parent eb185d2 commit 48d8442
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 16 deletions.
35 changes: 26 additions & 9 deletions src/core/providers/gdal/qgsgdalproviderbase.cpp
Expand Up @@ -260,7 +260,6 @@ QgsRectangle QgsGdalProviderBase::extent( GDALDatasetH gdalDataset )const
GDALDatasetH QgsGdalProviderBase::gdalOpen( const QString &uri, unsigned int nOpenFlags )
{
QVariantMap parts = decodeGdalUri( uri );
QString filePath = parts.value( QStringLiteral( "path" ) ).toString();
const QStringList openOptions = parts.value( QStringLiteral( "openOptions" ) ).toStringList();
parts.remove( QStringLiteral( "openOptions" ) );

Expand Down Expand Up @@ -329,8 +328,25 @@ QVariantMap QgsGdalProviderBase::decodeGdalUri( const QString &uri )
QStringList openOptions;

QString vsiPrefix = qgsVsiPrefix( path );
if ( !path.isEmpty() )
QString vsiSuffix;
if ( path.startsWith( vsiPrefix, Qt::CaseInsensitive ) )
{
path = path.mid( vsiPrefix.count() );
if ( vsiPrefix == QLatin1String( "/vsizip/" ) )
{
const QRegularExpression vsiRegex( QStringLiteral( "(?:\\.zip|\\.tar|\\.gz|\\.tar\\.gz|\\.tgz)([^|]*)" ) );
QRegularExpressionMatch match = vsiRegex.match( path );
if ( match.hasMatch() )
{
vsiSuffix = match.captured( 1 );
path = path.remove( match.capturedStart( 1 ), match.capturedLength( 1 ) );
}
}
}
else
{
vsiPrefix.clear();
}

if ( path.indexOf( ':' ) != -1 )
{
Expand Down Expand Up @@ -373,22 +389,23 @@ QVariantMap QgsGdalProviderBase::decodeGdalUri( const QString &uri )
uriComponents.insert( QStringLiteral( "openOptions" ), openOptions );
if ( !vsiPrefix.isEmpty() )
uriComponents.insert( QStringLiteral( "vsiPrefix" ), vsiPrefix );
if ( !vsiSuffix.isEmpty() )
uriComponents.insert( QStringLiteral( "vsiSuffix" ), vsiSuffix );
return uriComponents;
}

QString QgsGdalProviderBase::encodeGdalUri( const QVariantMap &parts )
{
const QString vsiPrefix = parts.value( QStringLiteral( "vsiPrefix" ) ).toString();
const QString vsiSuffix = parts.value( QStringLiteral( "vsiSuffix" ) ).toString();
const QString path = parts.value( QStringLiteral( "path" ) ).toString();
const QString layerName = parts.value( QStringLiteral( "layerName" ) ).toString();

QString uri;
if ( !layerName.isEmpty() && path.endsWith( QLatin1String( "gpkg" ) ) )
uri = QStringLiteral( "GPKG:%1:%2" ).arg( path, layerName );
QString uri = vsiPrefix + path + vsiSuffix;
if ( !layerName.isEmpty() && uri.endsWith( QLatin1String( "gpkg" ) ) )
uri = QStringLiteral( "GPKG:%1:%2" ).arg( uri, layerName );
else if ( !layerName.isEmpty() )
uri = path + QStringLiteral( "|%1" ).arg( layerName );
else
uri = path;
uri = uri + QStringLiteral( "|%1" ).arg( layerName );

const QStringList openOptions = parts.value( QStringLiteral( "openOptions" ) ).toStringList();

Expand All @@ -398,7 +415,7 @@ QString QgsGdalProviderBase::encodeGdalUri( const QVariantMap &parts )
uri += openOption;
}

return !vsiPrefix.isEmpty() ? vsiPrefix + uri : uri;
return uri;
}

///@endcond
8 changes: 5 additions & 3 deletions src/core/providers/ogr/qgsogrconnpool.h
Expand Up @@ -40,16 +40,18 @@ inline void qgsConnectionPool_ConnectionCreate( const QString &connInfo, QgsOgrC
{
c = new QgsOgrConn;

QVariantMap parts = QgsOgrProviderMetadata().decodeUri( connInfo );
QString filePath = parts.value( QStringLiteral( "path" ) ).toString();
const QVariantMap parts = QgsOgrProviderMetadata().decodeUri( connInfo );
const QString fullPath = parts.value( QStringLiteral( "vsiPrefix" ) ).toString()
+ parts.value( QStringLiteral( "path" ) ).toString()
+ parts.value( QStringLiteral( "vsiSuffix" ) ).toString();
const QStringList openOptions = parts.value( QStringLiteral( "openOptions" ) ).toStringList();
char **papszOpenOptions = nullptr;
for ( const QString &option : openOptions )
{
papszOpenOptions = CSLAddString( papszOpenOptions,
option.toUtf8().constData() );
}
c->ds = QgsOgrProviderUtils::GDALOpenWrapper( filePath.toUtf8().constData(), false, papszOpenOptions, nullptr );
c->ds = QgsOgrProviderUtils::GDALOpenWrapper( fullPath.toUtf8().constData(), false, papszOpenOptions, nullptr );
CSLDestroy( papszOpenOptions );
c->path = connInfo;
c->valid = true;
Expand Down
34 changes: 32 additions & 2 deletions src/core/providers/ogr/qgsogrprovider.cpp
Expand Up @@ -319,7 +319,10 @@ static QString AnalyzeURI( QString const &uri,
openOptions = parts.value( QStringLiteral( "openOptions" ) ).toStringList();
}

return parts.value( QStringLiteral( "path" ) ).toString();
const QString fullPath = parts.value( QStringLiteral( "vsiPrefix" ) ).toString()
+ parts.value( QStringLiteral( "path" ) ).toString()
+ parts.value( QStringLiteral( "vsiSuffix" ) ).toString();
return fullPath;
}

QgsVectorLayerExporter::ExportError QgsOgrProvider::createEmptyLayer( const QString &uri,
Expand Down Expand Up @@ -3572,6 +3575,27 @@ QVariantMap QgsOgrProviderMetadata::decodeUri( const QString &uri ) const

int layerId = -1;

QString vsiPrefix = qgsVsiPrefix( path );
QString vsiSuffix;
if ( path.startsWith( vsiPrefix, Qt::CaseInsensitive ) )
{
path = path.mid( vsiPrefix.count() );
if ( vsiPrefix == QLatin1String( "/vsizip/" ) )
{
const QRegularExpression vsiRegex( QStringLiteral( "(?:\\.zip|\\.tar|\\.gz|\\.tar\\.gz|\\.tgz)([^|]*)" ) );
QRegularExpressionMatch match = vsiRegex.match( path );
if ( match.hasMatch() )
{
vsiSuffix = match.captured( 1 );
path = path.remove( match.capturedStart( 1 ), match.capturedLength( 1 ) );
}
}
}
else
{
vsiPrefix.clear();
}

if ( path.contains( '|' ) )
{
const QRegularExpression geometryTypeRegex( QStringLiteral( "\\|geometrytype=([a-zA-Z0-9]*)" ), QRegularExpression::PatternOption::CaseInsensitiveOption );
Expand Down Expand Up @@ -3667,18 +3691,24 @@ QVariantMap QgsOgrProviderMetadata::decodeUri( const QString &uri ) const
uriComponents.insert( QStringLiteral( "databaseName" ), databaseName );
if ( !openOptions.isEmpty() )
uriComponents.insert( QStringLiteral( "openOptions" ), openOptions );
if ( !vsiPrefix.isEmpty() )
uriComponents.insert( QStringLiteral( "vsiPrefix" ), vsiPrefix );
if ( !vsiSuffix.isEmpty() )
uriComponents.insert( QStringLiteral( "vsiSuffix" ), vsiSuffix );
return uriComponents;
}

QString QgsOgrProviderMetadata::encodeUri( const QVariantMap &parts ) const
{
const QString vsiPrefix = parts.value( QStringLiteral( "vsiPrefix" ) ).toString();
const QString vsiSuffix = parts.value( QStringLiteral( "vsiSuffix" ) ).toString();
const QString path = parts.value( QStringLiteral( "path" ) ).toString();
const QString layerName = parts.value( QStringLiteral( "layerName" ) ).toString();
const QString layerId = parts.value( QStringLiteral( "layerId" ) ).toString();
const QString subset = parts.value( QStringLiteral( "subset" ) ).toString();
const QString geometryType = parts.value( QStringLiteral( "geometryType" ) ).toString();
const QStringList openOptions = parts.value( QStringLiteral( "openOptions" ) ).toStringList();
QString uri = path
QString uri = vsiPrefix + path + vsiSuffix
+ ( !layerName.isEmpty() ? QStringLiteral( "|layername=%1" ).arg( layerName ) : !layerId.isEmpty() ? QStringLiteral( "|layerid=%1" ).arg( layerId ) : QString() )
+ ( !geometryType.isEmpty() ? QStringLiteral( "|geometrytype=%1" ).arg( geometryType ) : QString() );
for ( const QString &openOption : openOptions )
Expand Down
6 changes: 4 additions & 2 deletions tests/src/core/testqgsgdalprovider.cpp
Expand Up @@ -105,8 +105,9 @@ void TestQgsGdalProvider::decodeUri()

uri = QStringLiteral( "/vsizip//home/to/path/file.zip/my.tif" );
components = QgsProviderRegistry::instance()->decodeUri( QStringLiteral( "gdal" ), uri );
QCOMPARE( components[QStringLiteral( "path" )].toString(), QStringLiteral( "/home/to/path/file.zip/my.tif" ) );
QCOMPARE( components[QStringLiteral( "path" )].toString(), QStringLiteral( "/home/to/path/file.zip" ) );
QCOMPARE( components[QStringLiteral( "vsiPrefix" )].toString(), QStringLiteral( "/vsizip/" ) );
QCOMPARE( components[QStringLiteral( "vsiSuffix" )].toString(), QStringLiteral( "/my.tif" ) );

//test windows path
uri = QStringLiteral( "gpkg:c:/home/to/path/my_file.gpkg:layer_name" );
Expand All @@ -125,8 +126,9 @@ void TestQgsGdalProvider::encodeUri()
QCOMPARE( QgsProviderRegistry::instance()->encodeUri( QStringLiteral( "gdal" ), parts ), QStringLiteral( "GPKG:/home/user/test.gpkg:layername" ) );

parts.clear();
parts.insert( QStringLiteral( "path" ), QStringLiteral( "/home/user/test.zip/my.tif" ) );
parts.insert( QStringLiteral( "path" ), QStringLiteral( "/home/user/test.zip" ) );
parts.insert( QStringLiteral( "vsiPrefix" ), QStringLiteral( "/vsizip/" ) );
parts.insert( QStringLiteral( "vsiSuffix" ), QStringLiteral( "/my.tif" ) );
QCOMPARE( QgsProviderRegistry::instance()->encodeUri( QStringLiteral( "gdal" ), parts ), QStringLiteral( "/vsizip//home/user/test.zip/my.tif" ) );
}

Expand Down
9 changes: 9 additions & 0 deletions tests/src/python/test_provider_gdal.py
Expand Up @@ -111,6 +111,15 @@ def testDecodeEncodeUriOptions(self):
encodedUri = QgsProviderRegistry.instance().encodeUri('gdal', parts)
self.assertEqual(encodedUri, uri)

def testDecodeEncodeUriVsizip(self):
"""Test decodeUri/encodeUri for /vsizip/ prefixed URIs"""

uri = '/vsizip//my/file.zip/image.tif'
parts = QgsProviderRegistry.instance().decodeUri('gdal', uri)
self.assertEqual(parts, {'path': '/my/file.zip', 'layerName': None, 'vsiPrefix': '/vsizip/', 'vsiSuffix': '/image.tif'})
encodedUri = QgsProviderRegistry.instance().encodeUri('gdal', parts)
self.assertEqual(encodedUri, uri)


if __name__ == '__main__':
unittest.main()
27 changes: 27 additions & 0 deletions tests/src/python/test_provider_ogr.py
Expand Up @@ -928,6 +928,33 @@ def testEmbeddedSymbolsKml(self):
self.assertCountEqual([s.color().alpha() for s in symbols if s is not None], [127, 135, 255, 127])
self.assertEqual(len([s for s in symbols if s is None]), 2)

def testDecodeEncodeUriVsizip(self):
"""Test decodeUri/encodeUri for /vsizip/ prefixed URIs"""

uri = '/vsizip//my/file.zip/shapefile.shp'
parts = QgsProviderRegistry.instance().decodeUri('ogr', uri)
self.assertEqual(parts, {'path': '/my/file.zip', 'layerName': None, 'layerId': None, 'vsiPrefix': '/vsizip/', 'vsiSuffix': '/shapefile.shp'})
encodedUri = QgsProviderRegistry.instance().encodeUri('ogr', parts)
self.assertEqual(encodedUri, uri)

uri = '/my/file.zip'
parts = QgsProviderRegistry.instance().decodeUri('ogr', uri)
self.assertEqual(parts, {'path': '/my/file.zip', 'layerName': None, 'layerId': None})
encodedUri = QgsProviderRegistry.instance().encodeUri('ogr', parts)
self.assertEqual(encodedUri, uri)

uri = '/vsizip//my/file.zip|layername=shapefile'
parts = QgsProviderRegistry.instance().decodeUri('ogr', uri)
self.assertEqual(parts, {'path': '/my/file.zip', 'layerName': 'shapefile', 'layerId': None, 'vsiPrefix': '/vsizip/'})
encodedUri = QgsProviderRegistry.instance().encodeUri('ogr', parts)
self.assertEqual(encodedUri, uri)

uri = '/vsizip//my/file.zip|layername=shapefile|subset="field"=\'value\''
parts = QgsProviderRegistry.instance().decodeUri('ogr', uri)
self.assertEqual(parts, {'path': '/my/file.zip', 'layerName': 'shapefile', 'layerId': None, 'subset': '"field"=\'value\'', 'vsiPrefix': '/vsizip/'})
encodedUri = QgsProviderRegistry.instance().encodeUri('ogr', parts)
self.assertEqual(encodedUri, uri)


if __name__ == '__main__':
unittest.main()

0 comments on commit 48d8442

Please sign in to comment.