[ogr] Correctly handle FastScan flag in querySublayers by performing

an extension only scan for files/folders
qgis · Jul 23, 2021 · 816bb43 · 816bb43
1 parent f654019
commit 816bb43
Show file tree

Hide file tree

Showing 3 changed files with 191 additions and 0 deletions.
diff --git a/src/core/providers/ogr/qgsogrprovidermetadata.cpp b/src/core/providers/ogr/qgsogrprovidermetadata.cpp
@@ -29,6 +29,7 @@ email                : nyall dot dawson at gmail dot com
 #include "qgsogrdbconnection.h"
 #include "qgsprovidersublayerdetails.h"
 #include "qgszipitem.h"
+#include "qgsproviderutils.h"
 
 #include <QFileInfo>
 #include <QFile>
@@ -1098,6 +1099,93 @@ QList<QgsProviderSublayerDetails> QgsOgrProviderMetadata::querySublayers( const
     }
   }
 
+  const QString path = uriParts.value( QStringLiteral( "path" ) ).toString();
+  const QFileInfo pathInfo( path );
+  if ( flags & Qgis::SublayerQueryFlag::FastScan && ( pathInfo.isFile() || pathInfo.isDir() ) )
+  {
+    // fast scan, so we don't actually try to open the dataset and instead just check the extension alone
+    const QStringList fileExtensions = QgsOgrProviderUtils::fileExtensions();
+    const QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions();
+
+    const QString suffix = pathInfo.suffix().toLower();
+
+    // allow only normal files or supported directories to continue
+    const bool isOgrSupportedDirectory = pathInfo.isDir() && dirExtensions.contains( suffix );
+    if ( !isOgrSupportedDirectory && !pathInfo.isFile() )
+      return {};
+
+    if ( !fileExtensions.contains( suffix ) && !dirExtensions.contains( suffix ) )
+    {
+      bool matches = false;
+      const QStringList wildcards = QgsOgrProviderUtils::wildcards();
+      for ( const QString &wildcard : wildcards )
+      {
+        const QRegularExpression rx( QRegularExpression::wildcardToRegularExpression( wildcard ), QRegularExpression::CaseInsensitiveOption );
+        if ( rx.match( pathInfo.fileName() ).hasMatch() )
+        {
+          matches = true;
+          break;
+        }
+      }
+      if ( !matches )
+        return {};
+    }
+
+    // these extensions are trivial to read, so there's no need to rely on
+    // the extension only scan here -- avoiding it always gives us the correct data type
+    // and sublayer visibility
+    static QStringList sSkipFastTrackExtensions { QStringLiteral( "xlsx" ),
+        QStringLiteral( "ods" ),
+        QStringLiteral( "csv" ),
+        QStringLiteral( "nc" ),
+        QStringLiteral( "shp.zip" ) };
+
+    if ( !sSkipFastTrackExtensions.contains( suffix ) )
+    {
+      // Filters out the OGR/GDAL supported formats that can contain multiple layers
+      // and should be treated as a potential layer container
+      static QStringList sOgrSupportedDbLayersExtensions { QStringLiteral( "gpkg" ),
+          QStringLiteral( "sqlite" ),
+          QStringLiteral( "db" ),
+          QStringLiteral( "gdb" ),
+          QStringLiteral( "kml" ),
+          QStringLiteral( "osm" ),
+          QStringLiteral( "mdb" ),
+          QStringLiteral( "accdb" ),
+          QStringLiteral( "xls" ),
+          QStringLiteral( "xlsx" ),
+          QStringLiteral( "gpx" ),
+          QStringLiteral( "pdf" ),
+          QStringLiteral( "pbf" ) };
+
+      // if this is a VRT file make sure it is vector VRT
+      if ( suffix == QLatin1String( "vrt" ) )
+      {
+        CPLPushErrorHandler( CPLQuietErrorHandler );
+        CPLErrorReset();
+        GDALDriverH hDriver = GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr );
+        CPLPopErrorHandler();
+        if ( !hDriver )
+        {
+          // vrt is not a vector vrt, skip it
+          return {};
+        }
+      }
+
+      QgsProviderSublayerDetails details;
+      details.setType( QgsMapLayerType::VectorLayer );
+      details.setProviderKey( QStringLiteral( "ogr" ) );
+      details.setUri( uri );
+      details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) );
+      if ( sOgrSupportedDbLayersExtensions.contains( suffix ) )
+      {
+        // uri may contain sublayers, but query flags prevent us from examining them
+        details.setSkippedContainerScan( true );
+      }
+      return {details};
+    }
+  }
+
   const QString originalUriLayerName = uriParts.value( QStringLiteral( "layerName" ) ).toString();
   int layerId = 0;
   bool originalUriLayerIdWasSpecified = false;

diff --git a/tests/src/python/test_provider_ogr.py b/tests/src/python/test_provider_ogr.py
@@ -1909,6 +1909,109 @@ def test_provider_sublayer_details(self):
                                 'driverName': 'SQLite',
                                 'geomColName': ''}])
 
+    def test_provider_sublayer_details_fast_scan(self):
+        """
+        Test retrieving sublayer details from data provider metadata, using fast scan
+        """
+        metadata = QgsProviderRegistry.instance().providerMetadata('ogr')
+
+        # invalid uri
+        res = metadata.querySublayers('', Qgis.SublayerQueryFlag.FastScan)
+        self.assertFalse(res)
+
+        # not a vector
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, 'landsat.tif'), Qgis.SublayerQueryFlag.FastScan)
+        self.assertFalse(res)
+
+        # single layer vector
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, 'lines.shp'), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "lines")
+        self.assertEqual(res[0].description(), '')
+        self.assertEqual(res[0].uri(), TEST_DATA_DIR + "/lines.shp")
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertFalse(res[0].skippedContainerScan())
+
+        # geometry collection sublayers -- requires a scan to resolve geometry type
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, 'multipatch.shp'), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "multipatch")
+        self.assertEqual(res[0].description(), '')
+        self.assertEqual(res[0].uri(), TEST_DATA_DIR + "/multipatch.shp")
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertEqual(res[0].wkbType(), QgsWkbTypes.Unknown)
+        self.assertEqual(res[0].geometryColumnName(), '')
+        self.assertFalse(res[0].skippedContainerScan())
+
+        # single layer geopackage -- sublayers MUST have the layerName set on the uri,
+        # in case more layers are added in future to the gpkg
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, 'curved_polys.gpkg'), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "curved_polys")
+        self.assertEqual(res[0].description(), '')
+        self.assertEqual(res[0].uri(), TEST_DATA_DIR + "/curved_polys.gpkg")
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertTrue(res[0].skippedContainerScan())
+
+        # geopackage with two vector layers
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, "mixed_layers.gpkg"), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "mixed_layers")
+        self.assertEqual(res[0].description(), "")
+        self.assertEqual(res[0].uri(), "{}/mixed_layers.gpkg".format(TEST_DATA_DIR))
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertTrue(res[0].skippedContainerScan())
+
+        # layer with mixed geometry types - without resolving geometry types
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, "mixed_types.TAB"), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "mixed_types")
+        self.assertEqual(res[0].description(), "")
+        self.assertEqual(res[0].uri(), "{}/mixed_types.TAB".format(TEST_DATA_DIR))
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertFalse(res[0].skippedContainerScan())
+
+        # spatialite
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, "provider/spatialite.db"), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "spatialite")
+        self.assertEqual(res[0].description(), "")
+        self.assertEqual(res[0].uri(), "{}/provider/spatialite.db".format(TEST_DATA_DIR))
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertTrue(res[0].skippedContainerScan())
+
+        # fast scan, but for trivial type -- fast scan flag will be ignored
+        res = metadata.querySublayers(os.path.join(TEST_DATA_DIR, "spreadsheet.ods"), Qgis.SublayerQueryFlag.FastScan)
+        self.assertEqual(len(res), 2)
+        self.assertEqual(res[0].layerNumber(), 0)
+        self.assertEqual(res[0].name(), "Sheet1")
+        self.assertEqual(res[0].description(), "")
+        self.assertEqual(res[0].uri(), "{}/spreadsheet.ods|layername=Sheet1".format(TEST_DATA_DIR))
+        self.assertEqual(res[0].providerKey(), "ogr")
+        self.assertEqual(res[0].driverName(), "ODS")
+        self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer)
+        self.assertFalse(res[0].skippedContainerScan())
+        self.assertEqual(res[1].layerNumber(), 1)
+        self.assertEqual(res[1].name(), "Sheet2")
+        self.assertEqual(res[1].description(), "")
+        self.assertEqual(res[1].uri(), "{}/spreadsheet.ods|layername=Sheet2".format(TEST_DATA_DIR))
+        self.assertEqual(res[1].providerKey(), "ogr")
+        self.assertEqual(res[1].driverName(), "ODS")
+        self.assertEqual(res[1].type(), QgsMapLayerType.VectorLayer)
+        self.assertFalse(res[1].skippedContainerScan())
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/testdata/spreadsheet.ods b/tests/testdata/spreadsheet.ods