[FEATURE][ogr/gdal] Add UI to expose HTTP(S) and cloud datasets

qgis · Apr 26, 2018 · e875e5e · e875e5e
1 parent 4e175fd
commit e875e5e
Show file tree

Hide file tree

Showing 12 changed files with 608 additions and 57 deletions.
diff --git a/src/auth/basic/qgsauthbasicmethod.cpp b/src/auth/basic/qgsauthbasicmethod.cpp
@@ -43,6 +43,7 @@ QgsAuthBasicMethod::QgsAuthBasicMethod()
                     << QStringLiteral( "wcs" )
                     << QStringLiteral( "wms" )
                     << QStringLiteral( "ogr" )
+                    << QStringLiteral( "gdal" )
                     << QStringLiteral( "proxy" ) );
 
 }
@@ -120,7 +121,7 @@ bool QgsAuthBasicMethod::updateDataSourceUriItems( QStringList &connectionItems,
   }
 
   // Branch for OGR
-  if ( dataprovider == QStringLiteral( "ogr" ) )
+  if ( dataprovider == QStringLiteral( "ogr" ) || dataprovider == QStringLiteral( "gdal" ) )
   {
     if ( ! password.isEmpty() )
     {
@@ -215,8 +216,11 @@ bool QgsAuthBasicMethod::updateDataSourceUriItems( QStringList &connectionItems,
         else if ( uri.startsWith( QStringLiteral( "couchdb" ) )
                   || uri.startsWith( QStringLiteral( "DODS" ) )
                   || uri.startsWith( "http://" )
+                  || uri.startsWith( "/vsicurl/http://" )
                   || uri.startsWith( "https://" )
-                  || uri.startsWith( "ftp://" ) // not really sure that this is supported ...
+                  || uri.startsWith( "/vsicurl/https://" )
+                  || uri.startsWith( "ftp://" )
+                  || uri.startsWith( "/vsicurl/ftp://" )
                 )
         {
           uri = uri.replace( QStringLiteral( "://" ), QStringLiteral( "://%1:%2@" ).arg( username, password ) );

diff --git a/src/gui/ogr/qgsogrhelperfunctions.cpp b/src/gui/ogr/qgsogrhelperfunctions.cpp
@@ -221,7 +221,42 @@ QString createDatabaseURI( const QString &connectionType, const QString &host, c
 QString createProtocolURI( const QString &type, const QString &url,  const QString &configId, const QString &username, const QString &password, bool expandAuthConfig )
 {
   QString uri;
-  if ( type == QLatin1String( "GeoJSON" ) )
+  if ( type == QLatin1String( "HTTP/HTTPS/FTP" ) )
+  {
+    uri = url;
+    // If no protocol is provided in the URL, default to HTTP
+    if ( !uri.startsWith( "http://" ) && !uri.startsWith( "https://" ) && !uri.startsWith( "ftp://" ) )
+    {
+      uri.prepend( QStringLiteral( "http://" ) );
+    }
+    uri.prepend( QStringLiteral( "/vsicurl/" ) );
+  }
+  else if ( type == QLatin1String( "AWS S3" ) )
+  {
+    uri = url;
+    uri.prepend( QStringLiteral( "/vsis3/" ) );
+  }
+  else if ( type == QLatin1String( "Google Cloud Storage" ) )
+  {
+    uri = url;
+    uri.prepend( QStringLiteral( "/vsigs/" ) );
+  }
+  else if ( type == QLatin1String( "Microsoft Azure Blob" ) )
+  {
+    uri = url;
+    uri.prepend( QStringLiteral( "/vsiaz/" ) );
+  }
+  else if ( type == QLatin1String( "Alibaba Cloud OSS" ) )
+  {
+    uri = url;
+    uri.prepend( QStringLiteral( "/vsioss/" ) );
+  }
+  else if ( type == QLatin1String( "OpenStack Swift Object Storage" ) )
+  {
+    uri = url;
+    uri.prepend( QStringLiteral( "/vsiswift/" ) );
+  }
+  else if ( type == QLatin1String( "GeoJSON" ) )
   {
     uri = url;
   }

diff --git a/src/providers/gdal/CMakeLists.txt b/src/providers/gdal/CMakeLists.txt
@@ -21,11 +21,13 @@ ENDIF ()
 
 INCLUDE_DIRECTORIES (
   ${CMAKE_SOURCE_DIR}/src/core
+  ${CMAKE_SOURCE_DIR}/src/core/auth
   ${CMAKE_SOURCE_DIR}/src/core/expression
   ${CMAKE_SOURCE_DIR}/src/core/raster
   ${CMAKE_SOURCE_DIR}/src/core/geometry
   ${CMAKE_SOURCE_DIR}/src/core/metadata
   ${CMAKE_SOURCE_DIR}/src/gui
+  ${CMAKE_SOURCE_DIR}/src/gui/auth
 
   ${CMAKE_BINARY_DIR}/src/core
   ${CMAKE_BINARY_DIR}/src/gui
@@ -34,6 +36,8 @@ INCLUDE_DIRECTORIES (
 INCLUDE_DIRECTORIES (SYSTEM
   ${GDAL_INCLUDE_DIR}
   ${GEOS_INCLUDE_DIR}
+  ${QTKEYCHAIN_INCLUDE_DIR}
+  ${QCA_INCLUDE_DIR}
 )
 
 QT5_WRAP_CPP(GDAL_MOC_SRCS ${GDAL_MOC_HDRS})

diff --git a/src/providers/gdal/qgsgdalprovider.cpp b/src/providers/gdal/qgsgdalprovider.cpp
@@ -22,6 +22,7 @@
 #include "qgsconfig.h"
 
 #include "qgsapplication.h"
+#include "qgsauthmanager.h"
 #include "qgscoordinatetransform.h"
 #include "qgsdataitem.h"
 #include "qgsdatasourceuri.h"
@@ -255,6 +256,33 @@ QgsGdalProvider::QgsGdalProvider( const QgsGdalProvider &other )
   copyBaseSettings( other );
 }
 
+QString QgsGdalProvider::dataSourceUri( bool expandAuthConfig ) const
+{
+  if ( expandAuthConfig && QgsDataProvider::dataSourceUri( ).contains( QLatin1String( "authcfg" ) ) )
+  {
+    QString uri( QgsDataProvider::dataSourceUri() );
+    // Check for authcfg
+    QRegularExpression authcfgRe( " authcfg='([^']+)'" );
+    QRegularExpressionMatch match;
+    if ( uri.contains( authcfgRe, &match ) )
+    {
+      uri = uri.replace( match.captured( 0 ), QString() );
+      QString configId( match.captured( 1 ) );
+      QStringList connectionItems;
+      connectionItems << uri;
+      if ( QgsApplication::authManager()->updateDataSourceUriItems( connectionItems, configId, QStringLiteral( "gdal" ) ) )
+      {
+        uri = connectionItems.first( );
+      }
+    }
+    return uri;
+  }
+  else
+  {
+    return QgsDataProvider::dataSourceUri();
+  }
+}
+
 QgsGdalProvider *QgsGdalProvider::clone() const
 {
   return new QgsGdalProvider( *this );
@@ -1723,7 +1751,7 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
 
     // test if the file is writable
     //QFileInfo myQFile( mDataSource );
-    QFileInfo myQFile( dataSourceUri() );
+    QFileInfo myQFile( dataSourceUri( true ) );
 
     if ( !myQFile.isWritable() )
     {
@@ -1753,12 +1781,12 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
       GDALClose( mGdalDataset );
       //mGdalBaseDataset = GDALOpen( QFile::encodeName( dataSourceUri() ).constData(), GA_Update );
 
-      mGdalBaseDataset = gdalOpen( dataSourceUri().toUtf8().constData(), GA_Update );
+      mGdalBaseDataset = gdalOpen( dataSourceUri( true ).toUtf8().constData(), GA_Update );
 
       // if the dataset couldn't be opened in read / write mode, tell the user
       if ( !mGdalBaseDataset )
       {
-        mGdalBaseDataset = gdalOpen( dataSourceUri().toUtf8().constData(), GA_ReadOnly );
+        mGdalBaseDataset = gdalOpen( dataSourceUri( true ).toUtf8().constData(), GA_ReadOnly );
         //Since we are not a virtual warped dataset, mGdalDataSet and mGdalBaseDataset are supposed to be the same
         mGdalDataset = mGdalBaseDataset;
         return QStringLiteral( "ERROR_WRITE_FORMAT" );
@@ -1859,7 +1887,7 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
       //something bad happenend
       //QString myString = QString (CPLGetLastError());
       GDALClose( mGdalBaseDataset );
-      mGdalBaseDataset = gdalOpen( dataSourceUri().toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
+      mGdalBaseDataset = gdalOpen( dataSourceUri( true ).toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
       //Since we are not a virtual warped dataset, mGdalDataSet and mGdalBaseDataset are supposed to be the same
       mGdalDataset = mGdalBaseDataset;
 
@@ -1911,7 +1939,7 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> &rasterPyr
     QgsDebugMsg( "Reopening dataset ..." );
     //close the gdal dataset and reopen it in read only mode
     GDALClose( mGdalBaseDataset );
-    mGdalBaseDataset = gdalOpen( dataSourceUri().toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
+    mGdalBaseDataset = gdalOpen( dataSourceUri( true ).toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
     //Since we are not a virtual warped dataset, mGdalDataSet and mGdalBaseDataset are supposed to be the same
     mGdalDataset = mGdalBaseDataset;
   }
@@ -2625,7 +2653,7 @@ bool QgsGdalProvider::initIfNeeded()
 
   mHasInit = true;
 
-  QString gdalUri = dataSourceUri();
+  QString gdalUri = dataSourceUri( true );
 
   // Try to open using VSIFileHandler (see qgsogrprovider.cpp)
   QString vsiPrefix = QgsZipItem::vsiPrefix( gdalUri );
@@ -2636,7 +2664,7 @@ bool QgsGdalProvider::initIfNeeded()
     QgsDebugMsg( QString( "Trying %1 syntax, uri= %2" ).arg( vsiPrefix, dataSourceUri() ) );
   }
 
-  gdalUri = dataSourceUri();
+  gdalUri = dataSourceUri( true );
 
   CPLErrorReset();
   mGdalBaseDataset = gdalOpen( gdalUri.toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
@@ -3041,7 +3069,7 @@ bool QgsGdalProvider::remove()
     mGdalDataset = nullptr;
 
     CPLErrorReset();
-    CPLErr err = GDALDeleteDataset( driver, dataSourceUri().toUtf8().constData() );
+    CPLErr err = GDALDeleteDataset( driver, dataSourceUri( true ).toUtf8().constData() );
     if ( err != CPLE_None )
     {
       QgsLogger::warning( "RasterIO error: " + QString::fromUtf8( CPLGetLastErrorMsg() ) );
@@ -3240,7 +3268,7 @@ bool QgsGdalProvider::setEditable( bool enabled )
   mUpdate = enabled;
 
   // reopen the dataset
-  mGdalBaseDataset = gdalOpen( dataSourceUri().toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
+  mGdalBaseDataset = gdalOpen( dataSourceUri( true ).toUtf8().constData(), mUpdate ? GA_Update : GA_ReadOnly );
   if ( !mGdalBaseDataset )
   {
     QString msg = QStringLiteral( "Cannot reopen GDAL dataset %1:\n%2" ).arg( dataSourceUri(), QString::fromUtf8( CPLGetLastErrorMsg() ) );

diff --git a/src/providers/gdal/qgsgdalprovider.h b/src/providers/gdal/qgsgdalprovider.h
@@ -78,6 +78,17 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
 
     ~QgsGdalProvider() override;
 
+    /**
+     * Get the data source specification. This may be a path or a protocol
+     * connection string
+     * \param expandAuthConfig Whether to expand any assigned authentication configuration
+     * \returns data source specification
+     * \note The default authentication configuration expansion is FALSE. This keeps credentials
+     * out of layer data source URIs and project files. Expansion should be specifically done
+     * only when needed within a provider
+     */
+    QString dataSourceUri( bool expandAuthConfig = false ) const override;
+
     /**
      * Clone the provider.
      *