Skip to content

Commit

Permalink
add support for .tar/.tgz files and relevant tests ; delay scan of .t…
Browse files Browse the repository at this point in the history 
…gz files and large .zip files until requested
  • Loading branch information
@etiennesky @jef-n
etiennesky authored and jef-n committed Jun 16, 2012
1 parent 4aab209 commit a4a85cb
Show file tree
Hide file tree
Showing 11 changed files with 302 additions and 175 deletions.
141 changes: 100 additions & 41 deletions src/core/qgsdataitem.cpp
View file
Expand Up @@ -477,7 +477,6 @@ QVector<QgsDataItem*> QgsDirectoryItem::createChildren( )
QVector<QgsDataItem*> children;
QDir dir( mPath );
QSettings settings;
bool scanZip = ( settings.value( "/qgis/scanZipInBrowser", QVariant( "basic" ) ).toString() != "no" );

QStringList entries = dir.entryList( QDir::AllDirs | QDir::NoDotAndDotDot, QDir::Name | QDir::IgnoreCase );
foreach( QString subdir, entries )
Expand All @@ -497,8 +496,10 @@ QVector<QgsDataItem*> QgsDirectoryItem::createChildren( )
QString path = dir.absoluteFilePath( name );
QFileInfo fileInfo( path );

QString vsiPrefix = QgsZipItem::vsiPrefix( path );
// vsizip support was added to GDAL/OGR 1.6 but GDAL_VERSION_NUM not available here
if ( fileInfo.suffix() == "zip" && scanZip )
if (( settings.value( "/qgis/scanZipInBrowser", QVariant( "basic" ) ).toString() != "no" ) &&
( vsiPrefix == "/vsizip/" || vsiPrefix == "/vsitar/" ) )
{
QgsDataItem * item = QgsZipItem::itemFromPath( this, path, name );
if ( item )
Expand Down Expand Up @@ -747,6 +748,7 @@ QgsZipItem::QgsZipItem( QgsDataItem* parent, QString name, QString path )
{
mType = Collection; //Zip??
mIcon = iconZip();
mVsiPrefix = vsiPrefix( path );

if ( mProviderNames.size() == 0 )
{
Expand Down Expand Up @@ -796,7 +798,6 @@ QgsZipItem::QgsZipItem( QgsDataItem* parent, QString name, QString path )
}
}
}

}

QgsZipItem::~QgsZipItem()
Expand Down Expand Up @@ -879,13 +880,12 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
QVector<QgsDataItem*> children;
QString tmpPath;
QString childPath;

QSettings settings;
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();

mZipFileList.clear();

QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3" ).arg( path() ).arg( name() ).arg( scanZipSetting ) );
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path() ).arg( name() ).arg( scanZipSetting ).arg( mVsiPrefix ) );

// if scanZipBrowser == no: skip to the next file
if ( scanZipSetting == "no" )
Expand All @@ -901,31 +901,14 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
// return children;
// }

// get list of files inside zip file
QgsDebugMsg( QString( "Open file %1 with gdal vsi" ).arg( path() ) );
char **papszSiblingFiles = VSIReadDirRecursive1( QString( "/vsizip/" + path() ).toLocal8Bit().constData() );
if ( papszSiblingFiles )
{
for ( int i = 0; i < CSLCount( papszSiblingFiles ); i++ )
{
tmpPath = papszSiblingFiles[i];
QgsDebugMsg( QString( "Read file %1" ).arg( tmpPath ) );
// skip directories (files ending with /)
if ( tmpPath.right( 1 ) != "/" )
mZipFileList << tmpPath;
}
CSLDestroy( papszSiblingFiles );
}
else
{
QgsDebugMsg( QString( "Error reading %1" ).arg( path() ) );
}
// first get list of files
getZipFileList();

// loop over files inside zip
foreach( QString fileName, mZipFileList )
{
QFileInfo info( fileName );
tmpPath = "/vsizip/" + path() + "/" + fileName;
tmpPath = mVsiPrefix + path() + "/" + fileName;
QgsDebugMsg( "tmpPath = " + tmpPath );

// foreach( dataItem_t *dataItem, mDataItemPtr )
Expand All @@ -934,7 +917,7 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
// ugly hack to remove .dbf file if there is a .shp file
if ( mProviderNames[i] == "ogr" )
{
if ( info.suffix() == "dbf" )
if ( info.suffix().toLower() == "dbf" )
{
if ( mZipFileList.indexOf( fileName.left( fileName.count() - 4 ) + ".shp" ) != -1 )
continue;
Expand Down Expand Up @@ -976,20 +959,38 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
return children;
}


QString QgsZipItem::vsiPrefix( QString path )
{
if ( path.startsWith( "/vsizip/", Qt::CaseInsensitive ) ||
path.endsWith( ".zip", Qt::CaseInsensitive ) )
return "/vsizip/";
else if ( path.startsWith( "/vsitar/", Qt::CaseInsensitive ) ||
path.endsWith( ".tar", Qt::CaseInsensitive ) ||
path.endsWith( ".tar.gz", Qt::CaseInsensitive ) ||
path.endsWith( ".tgz", Qt::CaseInsensitive ) )
return "/vsitar/";
else if ( path.startsWith( "/vsigzip/", Qt::CaseInsensitive ) ||
path.endsWith( ".gz", Qt::CaseInsensitive ) )
return "/vsigzip/";
else
return "";
}

QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QString name )
{
QSettings settings;
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
QString vsizipPath = path;
QString vsiPath = path;
int zipFileCount = 0;
QStringList zipFileList;
QFileInfo fileInfo( path );
QString vsiPrefix = QgsZipItem::vsiPrefix( path );
QgsZipItem * zipItem = 0;
bool populated = false;

QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3" ).arg( path ).arg( name ).arg( scanZipSetting ) );
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path ).arg( name ).arg( scanZipSetting ).arg( vsiPrefix ) );

// if scanZipBrowser == no: skip to the next file
// if scanZipBrowser == no: don't read the zip file
if ( scanZipSetting == "no" )
{
return 0;
Expand All @@ -1007,29 +1008,47 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin

if ( zipItem )
{
// force populate zipItem
zipItem->populate();
QgsDebugMsg( QString( "Got zipItem with %1 children, path=%2, name=%3" ).arg( zipItem->rowCount() ).arg( zipItem->path() ).arg( zipItem->name() ) );
// force populate zipItem if it has less than 10 items and is not a .tgz or .tar.gz file (slow loading)
// for other items populating will be delayed until item is opened
// this might be polluting the tree with empty items but is necessary for performance reasons
// could also accept all files smaller than a certain size and add options for file count and/or size

// first get list of files inside .zip or .tar files
if ( path.endsWith( ".zip", Qt::CaseInsensitive ) ||
path.endsWith( ".tar", Qt::CaseInsensitive ) )
{
zipFileList = zipItem->getZipFileList();
}
// force populate if less than 10 items
if ( zipFileList.count() > 0 && zipFileList.count() <= 10 )
{
zipItem->populate();
populated = true; // there is no QgsDataItem::isPopulated() function
QgsDebugMsg( QString( "Got zipItem with %1 children, path=%2, name=%3" ).arg( zipItem->rowCount() ).arg( zipItem->path() ).arg( zipItem->name() ) );
}
else
{
QgsDebugMsg( QString( "Delaying populating zipItem with path=%1, name=%2" ).arg( zipItem->path() ).arg( zipItem->name() ) );
}
}

// only display if has children
// other option would be to delay until item is opened, but we would be polluting the tree with empty items
if ( zipItem && zipItem->rowCount() > 1 )
// only display if has children or if is not populated
if ( zipItem && ( !populated || zipItem->rowCount() > 1 ) )
{
QgsDebugMsg( "returning zipItem" );
return zipItem;
}
// if 1 or 0 child found, create a single data item using the normal path or the full path given by QgsZipItem
// if 1 or 0 child found, create a single data item using the normal path or the full path given by QgsZipItem
else
{
if ( zipItem )
{
vsizipPath = zipItem->path();
zipFileCount = zipItem->getZipFileList().count();
vsiPath = zipItem->path();
zipFileCount = zipFileList.count();
delete zipItem;
}

QgsDebugMsg( QString( "will try to create a normal dataItem from path= %2 or %3" ).arg( path ).arg( vsizipPath ) );
QgsDebugMsg( QString( "will try to create a normal dataItem from path= %2 or %3" ).arg( path ).arg( vsiPath ) );

// try to open using registered providers (gdal and ogr)
for ( int i = 0; i < mProviderNames.size(); i++ )
Expand All @@ -1047,7 +1066,7 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin
item = dataItem( path, parent );
// try with /vsizip/
if ( ! item )
item = dataItem( vsizipPath, parent );
item = dataItem( vsiPath, parent );
if ( item )
return item;
}
Expand All @@ -1056,3 +1075,43 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin

return 0;
}

const QStringList & QgsZipItem::getZipFileList()
{
if ( ! mZipFileList.isEmpty() )
return mZipFileList;

QString tmpPath;
QSettings settings;
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();

QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path() ).arg( name() ).arg( scanZipSetting ).arg( mVsiPrefix ) );

// if scanZipBrowser == no: skip to the next file
if ( scanZipSetting == "no" )
{
return mZipFileList;
}

// get list of files inside zip file
QgsDebugMsg( QString( "Open file %1 with gdal vsi" ).arg( mVsiPrefix + path() ) );
char **papszSiblingFiles = VSIReadDirRecursive1( QString( mVsiPrefix + path() ).toLocal8Bit().constData() );
if ( papszSiblingFiles )
{
for ( int i = 0; i < CSLCount( papszSiblingFiles ); i++ )
{
tmpPath = papszSiblingFiles[i];
QgsDebugMsg( QString( "Read file %1" ).arg( tmpPath ) );
// skip directories (files ending with /)
if ( tmpPath.right( 1 ) != "/" )
mZipFileList << tmpPath;
}
CSLDestroy( papszSiblingFiles );
}
else
{
QgsDebugMsg( QString( "Error reading %1" ).arg( path() ) );
}

return mZipFileList;
}
7 changes: 4 additions & 3 deletions src/core/qgsdataitem.h
View file
Expand Up @@ -298,24 +298,25 @@ class CORE_EXPORT QgsZipItem : public QgsDataCollectionItem
Q_OBJECT

protected:
QString mVsiPrefix;
QStringList mZipFileList;

public:
QgsZipItem( QgsDataItem* parent, QString name, QString path );
~QgsZipItem();

QVector<QgsDataItem*> createChildren();
QStringList getFiles();
const QStringList & getZipFileList();

static QVector<dataItem_t *> mDataItemPtr;
static QStringList mProviderNames;

static QString vsiPrefix( QString uri );

static QgsDataItem* itemFromPath( QgsDataItem* parent, QString path, QString name );

static const QIcon &iconZip();

const QStringList & getZipFileList() const { return mZipFileList; }

};

#endif // QGSDATAITEM_H
Expand Down
23 changes: 17 additions & 6 deletions src/core/qgsmaplayer.cpp
View file
Expand Up @@ -642,6 +642,14 @@ QString QgsMapLayer::styleURI( )
// ideally we should look for .qml file inside zip file
myURI.remove( 0, 8 );
}
else if ( myURI.startsWith( "/vsitar/", Qt::CaseInsensitive ) &&
( myURI.endsWith( ".tar", Qt::CaseInsensitive ) ||
myURI.endsWith( ".tar.gz", Qt::CaseInsensitive ) ||
myURI.endsWith( ".tgz", Qt::CaseInsensitive ) ) )
{
// ideally we should look for .qml file inside tar file
myURI.remove( 0, 8 );
}

QFileInfo myFileInfo( myURI );
QString key;
Expand All @@ -650,15 +658,18 @@ QString QgsMapLayer::styleURI( )
{
// if file is using the /vsizip/ or /vsigzip/ mechanism, cleanup the name
if ( myURI.endsWith( ".gz", Qt::CaseInsensitive ) )
{
myURI.chop( 3 );
myFileInfo.setFile( myURI );
}
else if ( myURI.endsWith( ".zip", Qt::CaseInsensitive ) )
{
myURI.chop( 4 );
myFileInfo.setFile( myURI );
}
else if ( myURI.endsWith( ".tar", Qt::CaseInsensitive ) )
myURI.chop( 4 );
else if ( myURI.endsWith( ".tar.gz", Qt::CaseInsensitive ) )
myURI.chop( 7 );
else if ( myURI.endsWith( ".tgz", Qt::CaseInsensitive ) )
myURI.chop( 4 );
else if ( myURI.endsWith( ".gz", Qt::CaseInsensitive ) )
myURI.chop( 3 );
myFileInfo.setFile( myURI );
// get the file name for our .qml style file
key = myFileInfo.path() + QDir::separator() + myFileInfo.completeBaseName() + ".qml";
}
Expand Down
42 changes: 18 additions & 24 deletions src/providers/gdal/qgsgdaldataitems.cpp
View file
Expand Up @@ -399,10 +399,10 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
// zip settings + info
QSettings settings;
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
bool is_vsizip = ( thePath.startsWith( "/vsizip/" ) ||
thePath.endsWith( ".zip", Qt::CaseInsensitive ) );
bool is_vsigzip = ( thePath.startsWith( "/vsigzip/" ) ||
thePath.endsWith( ".gz", Qt::CaseInsensitive ) );
QString vsiPrefix = QgsZipItem::vsiPrefix( thePath );
bool is_vsizip = ( vsiPrefix == "/vsizip/" );
bool is_vsigzip = ( vsiPrefix == "/vsigzip/" );
bool is_vsitar = ( vsiPrefix == "/vsitar/" );

// get suffix, removing .gz if present
QString tmpPath = thePath; //path used for testing, not for layer creation
Expand All @@ -414,10 +414,11 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
info.setFile( thePath );
QString name = info.fileName();

QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name + " suffix= " + suffix );
QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name
+ " suffix= " + suffix + " vsiPrefix= " + vsiPrefix );

// allow only normal files or VSIFILE items to continue
if ( !info.isFile() && !is_vsizip && !is_vsigzip )
if ( !info.isFile() && vsiPrefix == "" )
return 0;

// get supported extensions
Expand All @@ -434,10 +435,6 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
!extensions.contains( "aux.xml" ) )
return 0;

// skip .tar.gz files
if ( thePath.endsWith( ".tar.gz", Qt::CaseInsensitive ) )
return 0;

// Filter files by extension
if ( !extensions.contains( suffix ) )
{
Expand All @@ -455,30 +452,27 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
return 0;
}

// add /vsizip/ or /vsigzip/ to path if file extension is .zip or .gz
if ( is_vsigzip )
{
if ( !thePath.startsWith( "/vsigzip/" ) )
thePath = "/vsigzip/" + thePath;
}
else if ( is_vsizip )
// fix vsifile path and name
if ( vsiPrefix != "" )
{
if ( !thePath.startsWith( "/vsizip/" ) )
thePath = "/vsizip/" + thePath;
// add vsiPrefix to path if needed
if ( !thePath.startsWith( vsiPrefix ) )
thePath = vsiPrefix + thePath;
// if this is a /vsigzip/path_to_zip.zip/file_inside_zip remove the full path from the name
if ( thePath != "/vsizip/" + parentItem->path() )
if (( is_vsizip || is_vsitar ) && ( thePath != vsiPrefix + parentItem->path() ) )
{
name = thePath;
name = name.replace( "/vsizip/" + parentItem->path() + "/", "" );
name = name.replace( vsiPrefix + parentItem->path() + "/", "" );
}
}

// return a /vsizip/ item without testing if:
// zipfile and scan zip == "Basic scan"
// not zipfile and scan items == "Check extension"
if (( is_vsizip && scanZipSetting == "basic" ) ||
( !is_vsizip && ( settings.value( "/qgis/scanItemsInBrowser",
"extension" ).toString() == "extension" ) ) )
if ((( is_vsizip || is_vsitar ) && scanZipSetting == "basic" ) ||
( !is_vsizip && !is_vsitar &&
( settings.value( "/qgis/scanItemsInBrowser",
"extension" ).toString() == "extension" ) ) )
{
// if this is a VRT file make sure it is raster VRT to avoid duplicates
if ( suffix == "vrt" )
Expand Down

0 comments on commit a4a85cb

Please sign in to comment.