Skip to content

Commit

Permalink
[gdal] Correctly implement FastScan for querySublayers
Browse files Browse the repository at this point in the history
  • Loading branch information
@nyalldawson
nyalldawson committed Jul 23, 2021
1 parent 8349457 commit c7c9710
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 77 deletions.
88 changes: 55 additions & 33 deletions src/core/providers/gdal/qgsgdalprovider.cpp
View file
Expand Up @@ -3562,48 +3562,70 @@ QList<QgsProviderSublayerDetails> QgsGdalProviderMetadata::querySublayers( const
}
}

if ( flags & Qgis::SublayerQueryFlag::FastScan )
const QString path = uriParts.value( QStringLiteral( "path" ) ).toString();
const QFileInfo pathInfo( path );
if ( flags & Qgis::SublayerQueryFlag::FastScan && ( pathInfo.isFile() || pathInfo.isDir() ) )
{
// filter based on extension
const QVariantMap uriParts = decodeUri( gdalUri );
const QString path = uriParts.value( QStringLiteral( "path" ) ).toString();
QFileInfo info( path );
if ( info.isFile() )
{
const QString suffix = info.suffix().toLower();
// fast scan, so we don't actually try to open the dataset and instead just check the extension alone
static QString sFilterString;
static QStringList sExtensions;
static QStringList sWildcards;

static QString sFilterString;
static QStringList sExtensions;
static QStringList sWildcards;
// get supported extensions
static std::once_flag initialized;
std::call_once( initialized, [ = ]
{
buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards );
QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 );
QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 );
} );

// get supported extensions
static std::once_flag initialized;
std::call_once( initialized, [ = ]
{
buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards );
QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 );
QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 );
} );
const QString suffix = pathInfo.suffix().toLower();

if ( !sExtensions.contains( suffix ) )
if ( !sExtensions.contains( suffix ) )
{
bool matches = false;
for ( const QString &wildcard : std::as_const( sWildcards ) )
{
bool matches = false;
for ( const QString &wildcard : std::as_const( sWildcards ) )
const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern(
QRegularExpression::wildcardToRegularExpression( wildcard )
), QRegularExpression::CaseInsensitiveOption );
const QRegularExpressionMatch match = rx.match( pathInfo.fileName() );
if ( match.hasMatch() )
{
const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern(
QRegularExpression::wildcardToRegularExpression( wildcard )
), QRegularExpression::CaseInsensitiveOption );
const QRegularExpressionMatch match = rx.match( info.fileName() );
if ( match.hasMatch() )
{
matches = true;
break;
}
matches = true;
break;
}
if ( !matches )
return {};
}
if ( !matches )
return {};
}

// if this is a VRT file make sure it is raster VRT
if ( suffix == QLatin1String( "vrt" ) )
{
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
GDALDriverH hDriver = GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_RASTER, nullptr, nullptr );
CPLPopErrorHandler();
if ( !hDriver )
{
// vrt is not a raster vrt, skip it
return {};
}
}

QgsProviderSublayerDetails details;
details.setType( QgsMapLayerType::RasterLayer );
details.setProviderKey( QStringLiteral( "gdal" ) );
details.setUri( uri );
details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) );
if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) )
{
// uri may contain sublayers, but query flags prevent us from examining them
details.setSkippedContainerScan( true );
}
return {details};
}

if ( !uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString().isEmpty()
Expand Down
27 changes: 3 additions & 24 deletions src/core/providers/ogr/qgsogrprovidermetadata.cpp
View file
Expand Up @@ -30,6 +30,7 @@ email : nyall dot dawson at gmail dot com
#include "qgsprovidersublayerdetails.h"
#include "qgszipitem.h"
#include "qgsproviderutils.h"
#include "qgsgdalutils.h"

#include <QFileInfo>
#include <QFile>
Expand Down Expand Up @@ -1134,30 +1135,8 @@ QList<QgsProviderSublayerDetails> QgsOgrProviderMetadata::querySublayers( const
// these extensions are trivial to read, so there's no need to rely on
// the extension only scan here -- avoiding it always gives us the correct data type
// and sublayer visibility
static QStringList sSkipFastTrackExtensions { QStringLiteral( "xlsx" ),
QStringLiteral( "ods" ),
QStringLiteral( "csv" ),
QStringLiteral( "nc" ),
QStringLiteral( "shp.zip" ) };

if ( !sSkipFastTrackExtensions.contains( suffix ) )
if ( !QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS.contains( suffix ) )
{
// Filters out the OGR/GDAL supported formats that can contain multiple layers
// and should be treated as a potential layer container
static QStringList sOgrSupportedDbLayersExtensions { QStringLiteral( "gpkg" ),
QStringLiteral( "sqlite" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ),
QStringLiteral( "kml" ),
QStringLiteral( "osm" ),
QStringLiteral( "mdb" ),
QStringLiteral( "accdb" ),
QStringLiteral( "xls" ),
QStringLiteral( "xlsx" ),
QStringLiteral( "gpx" ),
QStringLiteral( "pdf" ),
QStringLiteral( "pbf" ) };

// if this is a VRT file make sure it is vector VRT
if ( suffix == QLatin1String( "vrt" ) )
{
Expand All @@ -1177,7 +1156,7 @@ QList<QgsProviderSublayerDetails> QgsOgrProviderMetadata::querySublayers( const
details.setProviderKey( QStringLiteral( "ogr" ) );
details.setUri( uri );
details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) );
if ( sOgrSupportedDbLayersExtensions.contains( suffix ) )
if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) )
{
// uri may contain sublayers, but query flags prevent us from examining them
details.setSkippedContainerScan( true );
Expand Down
27 changes: 27 additions & 0 deletions src/core/qgsgdalutils.cpp
View file
Expand Up @@ -28,6 +28,33 @@
#include <QImage>
#include <QFileInfo>

// File extensions for formats supported by GDAL which may contain multiple layers
// and should be treated as a potential layer container
const QStringList QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS
{
QStringLiteral( "gpkg" ),
QStringLiteral( "sqlite" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ),
QStringLiteral( "kml" ),
QStringLiteral( "osm" ),
QStringLiteral( "mdb" ),
QStringLiteral( "accdb" ),
QStringLiteral( "xls" ),
QStringLiteral( "xlsx" ),
QStringLiteral( "gpx" ),
QStringLiteral( "pdf" ),
QStringLiteral( "pbf" ),
QStringLiteral( "nc" ) };

const QStringList QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS
{
QStringLiteral( "xlsx" ),
QStringLiteral( "ods" ),
QStringLiteral( "csv" ),
QStringLiteral( "nc" ),
QStringLiteral( "shp.zip" ) };

bool QgsGdalUtils::supportsRasterCreate( GDALDriverH driver )
{
QString driverShortName = GDALGetDriverShortName( driver );
Expand Down
8 changes: 8 additions & 0 deletions src/core/qgsgdalutils.h
View file
Expand Up @@ -148,6 +148,14 @@ class CORE_EXPORT QgsGdalUtils
*/
static bool pathIsCheapToOpen( const QString &path, int smallFileSizeLimit = 50000 );


/**
* File extensions for formats supported by GDAL which may contain multiple layers
* and should be treated as a potential layer container.
* \since QGIS 3.22
*/
static const QStringList SUPPORTED_DB_LAYERS_EXTENSIONS;

friend class TestQgsGdalUtils;
};

Expand Down
80 changes: 60 additions & 20 deletions tests/src/core/testqgsgdalprovider.cpp
View file
Expand Up @@ -64,6 +64,7 @@ class TestQgsGdalProvider : public QObject
void scale0(); //test when data has scale 0 (#20493)
void transformCoordinates();
void testGdalProviderQuerySublayers();
void testGdalProviderQuerySublayersFastScan();

private:
QString mTestDataDir;
Expand Down Expand Up @@ -428,8 +429,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
// not a raster
res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp" );
QVERIFY( res.empty() );
res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );

// single layer raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" );
Expand Down Expand Up @@ -468,8 +467,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
QCOMPARE( res.at( 1 ).driverName(), QStringLiteral( "GPKG" ) );
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 2 );

// netcdf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" );
Expand All @@ -493,9 +490,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );

res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 8 );

// netcdf with open options
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" );
QCOMPARE( res.count(), 8 );
Expand Down Expand Up @@ -544,19 +538,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );

// aigrid, with fast scan
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).layerNumber(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) );
QCOMPARE( res.at( 0 ).description(), QString() );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid" ).arg( QStringLiteral( TEST_DATA_DIR ) ) );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QCOMPARE( res.at( 0 ).driverName(), QStringLiteral( "AIG" ) );
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );

// zip archive, only 1 file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/zip/landsat_b1.zip" );
QCOMPARE( res.count(), 1 );
Expand Down Expand Up @@ -626,5 +607,64 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
QVERIFY( rl->isValid() );
}

void TestQgsGdalProvider::testGdalProviderQuerySublayersFastScan()
{
// test querying sub layers for a mesh layer
QgsProviderMetadata *gdalMetadata = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "gdal" ) );

// invalid uri
QList< QgsProviderSublayerDetails >res = gdalMetadata->querySublayers( QString(), Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );

// not a raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );

// single layer raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "landsat" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( !res.at( 0 ).skippedContainerScan() );

// geopackage with two raster layers
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "mixed_layers" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );

// netcdf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );

// netcdf with open options
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );

// aigrid, pointing to .adf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid/hdr.adf", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid/hdr.adf" ).arg( QStringLiteral( TEST_DATA_DIR ) ) );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( !res.at( 0 ).skippedContainerScan() );
}

QGSTEST_MAIN( TestQgsGdalProvider )
#include "testqgsgdalprovider.moc"

0 comments on commit c7c9710

Please sign in to comment.