Skip to content

Commit a4a85cb

Browse files
etienneskyjef-n
etiennesky
authored and
jef-n
committedJun 16, 2012
add support for .tar/.tgz files and relevant tests ; delay scan of .tgz files and large .zip files until requested
1 parent 4aab209 commit a4a85cb

File tree

11 files changed

+302
-175
lines changed

11 files changed

+302
-175
lines changed
 

‎src/core/qgsdataitem.cpp

Lines changed: 100 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,6 @@ QVector<QgsDataItem*> QgsDirectoryItem::createChildren( )
477477
QVector<QgsDataItem*> children;
478478
QDir dir( mPath );
479479
QSettings settings;
480-
bool scanZip = ( settings.value( "/qgis/scanZipInBrowser", QVariant( "basic" ) ).toString() != "no" );
481480

482481
QStringList entries = dir.entryList( QDir::AllDirs | QDir::NoDotAndDotDot, QDir::Name | QDir::IgnoreCase );
483482
foreach( QString subdir, entries )
@@ -497,8 +496,10 @@ QVector<QgsDataItem*> QgsDirectoryItem::createChildren( )
497496
QString path = dir.absoluteFilePath( name );
498497
QFileInfo fileInfo( path );
499498

499+
QString vsiPrefix = QgsZipItem::vsiPrefix( path );
500500
// vsizip support was added to GDAL/OGR 1.6 but GDAL_VERSION_NUM not available here
501-
if ( fileInfo.suffix() == "zip" && scanZip )
501+
if (( settings.value( "/qgis/scanZipInBrowser", QVariant( "basic" ) ).toString() != "no" ) &&
502+
( vsiPrefix == "/vsizip/" || vsiPrefix == "/vsitar/" ) )
502503
{
503504
QgsDataItem * item = QgsZipItem::itemFromPath( this, path, name );
504505
if ( item )
@@ -747,6 +748,7 @@ QgsZipItem::QgsZipItem( QgsDataItem* parent, QString name, QString path )
747748
{
748749
mType = Collection; //Zip??
749750
mIcon = iconZip();
751+
mVsiPrefix = vsiPrefix( path );
750752

751753
if ( mProviderNames.size() == 0 )
752754
{
@@ -796,7 +798,6 @@ QgsZipItem::QgsZipItem( QgsDataItem* parent, QString name, QString path )
796798
}
797799
}
798800
}
799-
800801
}
801802

802803
QgsZipItem::~QgsZipItem()
@@ -879,13 +880,12 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
879880
QVector<QgsDataItem*> children;
880881
QString tmpPath;
881882
QString childPath;
882-
883883
QSettings settings;
884884
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
885885

886886
mZipFileList.clear();
887887

888-
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3" ).arg( path() ).arg( name() ).arg( scanZipSetting ) );
888+
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path() ).arg( name() ).arg( scanZipSetting ).arg( mVsiPrefix ) );
889889

890890
// if scanZipBrowser == no: skip to the next file
891891
if ( scanZipSetting == "no" )
@@ -901,31 +901,14 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
901901
// return children;
902902
// }
903903

904-
// get list of files inside zip file
905-
QgsDebugMsg( QString( "Open file %1 with gdal vsi" ).arg( path() ) );
906-
char **papszSiblingFiles = VSIReadDirRecursive1( QString( "/vsizip/" + path() ).toLocal8Bit().constData() );
907-
if ( papszSiblingFiles )
908-
{
909-
for ( int i = 0; i < CSLCount( papszSiblingFiles ); i++ )
910-
{
911-
tmpPath = papszSiblingFiles[i];
912-
QgsDebugMsg( QString( "Read file %1" ).arg( tmpPath ) );
913-
// skip directories (files ending with /)
914-
if ( tmpPath.right( 1 ) != "/" )
915-
mZipFileList << tmpPath;
916-
}
917-
CSLDestroy( papszSiblingFiles );
918-
}
919-
else
920-
{
921-
QgsDebugMsg( QString( "Error reading %1" ).arg( path() ) );
922-
}
904+
// first get list of files
905+
getZipFileList();
923906

924907
// loop over files inside zip
925908
foreach( QString fileName, mZipFileList )
926909
{
927910
QFileInfo info( fileName );
928-
tmpPath = "/vsizip/" + path() + "/" + fileName;
911+
tmpPath = mVsiPrefix + path() + "/" + fileName;
929912
QgsDebugMsg( "tmpPath = " + tmpPath );
930913

931914
// foreach( dataItem_t *dataItem, mDataItemPtr )
@@ -934,7 +917,7 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
934917
// ugly hack to remove .dbf file if there is a .shp file
935918
if ( mProviderNames[i] == "ogr" )
936919
{
937-
if ( info.suffix() == "dbf" )
920+
if ( info.suffix().toLower() == "dbf" )
938921
{
939922
if ( mZipFileList.indexOf( fileName.left( fileName.count() - 4 ) + ".shp" ) != -1 )
940923
continue;
@@ -976,20 +959,38 @@ QVector<QgsDataItem*> QgsZipItem::createChildren( )
976959
return children;
977960
}
978961

979-
962+
QString QgsZipItem::vsiPrefix( QString path )
963+
{
964+
if ( path.startsWith( "/vsizip/", Qt::CaseInsensitive ) ||
965+
path.endsWith( ".zip", Qt::CaseInsensitive ) )
966+
return "/vsizip/";
967+
else if ( path.startsWith( "/vsitar/", Qt::CaseInsensitive ) ||
968+
path.endsWith( ".tar", Qt::CaseInsensitive ) ||
969+
path.endsWith( ".tar.gz", Qt::CaseInsensitive ) ||
970+
path.endsWith( ".tgz", Qt::CaseInsensitive ) )
971+
return "/vsitar/";
972+
else if ( path.startsWith( "/vsigzip/", Qt::CaseInsensitive ) ||
973+
path.endsWith( ".gz", Qt::CaseInsensitive ) )
974+
return "/vsigzip/";
975+
else
976+
return "";
977+
}
980978

981979
QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QString name )
982980
{
983981
QSettings settings;
984982
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
985-
QString vsizipPath = path;
983+
QString vsiPath = path;
986984
int zipFileCount = 0;
985+
QStringList zipFileList;
987986
QFileInfo fileInfo( path );
987+
QString vsiPrefix = QgsZipItem::vsiPrefix( path );
988988
QgsZipItem * zipItem = 0;
989+
bool populated = false;
989990

990-
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3" ).arg( path ).arg( name ).arg( scanZipSetting ) );
991+
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path ).arg( name ).arg( scanZipSetting ).arg( vsiPrefix ) );
991992

992-
// if scanZipBrowser == no: skip to the next file
993+
// if scanZipBrowser == no: don't read the zip file
993994
if ( scanZipSetting == "no" )
994995
{
995996
return 0;
@@ -1007,29 +1008,47 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin
10071008

10081009
if ( zipItem )
10091010
{
1010-
// force populate zipItem
1011-
zipItem->populate();
1012-
QgsDebugMsg( QString( "Got zipItem with %1 children, path=%2, name=%3" ).arg( zipItem->rowCount() ).arg( zipItem->path() ).arg( zipItem->name() ) );
1011+
// force populate zipItem if it has less than 10 items and is not a .tgz or .tar.gz file (slow loading)
1012+
// for other items populating will be delayed until item is opened
1013+
// this might be polluting the tree with empty items but is necessary for performance reasons
1014+
// could also accept all files smaller than a certain size and add options for file count and/or size
1015+
1016+
// first get list of files inside .zip or .tar files
1017+
if ( path.endsWith( ".zip", Qt::CaseInsensitive ) ||
1018+
path.endsWith( ".tar", Qt::CaseInsensitive ) )
1019+
{
1020+
zipFileList = zipItem->getZipFileList();
1021+
}
1022+
// force populate if less than 10 items
1023+
if ( zipFileList.count() > 0 && zipFileList.count() <= 10 )
1024+
{
1025+
zipItem->populate();
1026+
populated = true; // there is no QgsDataItem::isPopulated() function
1027+
QgsDebugMsg( QString( "Got zipItem with %1 children, path=%2, name=%3" ).arg( zipItem->rowCount() ).arg( zipItem->path() ).arg( zipItem->name() ) );
1028+
}
1029+
else
1030+
{
1031+
QgsDebugMsg( QString( "Delaying populating zipItem with path=%1, name=%2" ).arg( zipItem->path() ).arg( zipItem->name() ) );
1032+
}
10131033
}
10141034

1015-
// only display if has children
1016-
// other option would be to delay until item is opened, but we would be polluting the tree with empty items
1017-
if ( zipItem && zipItem->rowCount() > 1 )
1035+
// only display if has children or if is not populated
1036+
if ( zipItem && ( !populated || zipItem->rowCount() > 1 ) )
10181037
{
10191038
QgsDebugMsg( "returning zipItem" );
10201039
return zipItem;
10211040
}
1022-
// if 1 or 0 child found, create a single data item using the normal path or the full path given by QgsZipItem
1041+
// if 1 or 0 child found, create a single data item using the normal path or the full path given by QgsZipItem
10231042
else
10241043
{
10251044
if ( zipItem )
10261045
{
1027-
vsizipPath = zipItem->path();
1028-
zipFileCount = zipItem->getZipFileList().count();
1046+
vsiPath = zipItem->path();
1047+
zipFileCount = zipFileList.count();
10291048
delete zipItem;
10301049
}
10311050

1032-
QgsDebugMsg( QString( "will try to create a normal dataItem from path= %2 or %3" ).arg( path ).arg( vsizipPath ) );
1051+
QgsDebugMsg( QString( "will try to create a normal dataItem from path= %2 or %3" ).arg( path ).arg( vsiPath ) );
10331052

10341053
// try to open using registered providers (gdal and ogr)
10351054
for ( int i = 0; i < mProviderNames.size(); i++ )
@@ -1047,7 +1066,7 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin
10471066
item = dataItem( path, parent );
10481067
// try with /vsizip/
10491068
if ( ! item )
1050-
item = dataItem( vsizipPath, parent );
1069+
item = dataItem( vsiPath, parent );
10511070
if ( item )
10521071
return item;
10531072
}
@@ -1056,3 +1075,43 @@ QgsDataItem* QgsZipItem::itemFromPath( QgsDataItem* parent, QString path, QStrin
10561075

10571076
return 0;
10581077
}
1078+
1079+
const QStringList & QgsZipItem::getZipFileList()
1080+
{
1081+
if ( ! mZipFileList.isEmpty() )
1082+
return mZipFileList;
1083+
1084+
QString tmpPath;
1085+
QSettings settings;
1086+
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
1087+
1088+
QgsDebugMsg( QString( "path = %1 name= %2 scanZipSetting= %3 vsiPrefix= %4" ).arg( path() ).arg( name() ).arg( scanZipSetting ).arg( mVsiPrefix ) );
1089+
1090+
// if scanZipBrowser == no: skip to the next file
1091+
if ( scanZipSetting == "no" )
1092+
{
1093+
return mZipFileList;
1094+
}
1095+
1096+
// get list of files inside zip file
1097+
QgsDebugMsg( QString( "Open file %1 with gdal vsi" ).arg( mVsiPrefix + path() ) );
1098+
char **papszSiblingFiles = VSIReadDirRecursive1( QString( mVsiPrefix + path() ).toLocal8Bit().constData() );
1099+
if ( papszSiblingFiles )
1100+
{
1101+
for ( int i = 0; i < CSLCount( papszSiblingFiles ); i++ )
1102+
{
1103+
tmpPath = papszSiblingFiles[i];
1104+
QgsDebugMsg( QString( "Read file %1" ).arg( tmpPath ) );
1105+
// skip directories (files ending with /)
1106+
if ( tmpPath.right( 1 ) != "/" )
1107+
mZipFileList << tmpPath;
1108+
}
1109+
CSLDestroy( papszSiblingFiles );
1110+
}
1111+
else
1112+
{
1113+
QgsDebugMsg( QString( "Error reading %1" ).arg( path() ) );
1114+
}
1115+
1116+
return mZipFileList;
1117+
}

‎src/core/qgsdataitem.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -298,24 +298,25 @@ class CORE_EXPORT QgsZipItem : public QgsDataCollectionItem
298298
Q_OBJECT
299299

300300
protected:
301+
QString mVsiPrefix;
301302
QStringList mZipFileList;
302303

303304
public:
304305
QgsZipItem( QgsDataItem* parent, QString name, QString path );
305306
~QgsZipItem();
306307

307308
QVector<QgsDataItem*> createChildren();
308-
QStringList getFiles();
309+
const QStringList & getZipFileList();
309310

310311
static QVector<dataItem_t *> mDataItemPtr;
311312
static QStringList mProviderNames;
312313

314+
static QString vsiPrefix( QString uri );
315+
313316
static QgsDataItem* itemFromPath( QgsDataItem* parent, QString path, QString name );
314317

315318
static const QIcon &iconZip();
316319

317-
const QStringList & getZipFileList() const { return mZipFileList; }
318-
319320
};
320321

321322
#endif // QGSDATAITEM_H

‎src/core/qgsmaplayer.cpp

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,14 @@ QString QgsMapLayer::styleURI( )
642642
// ideally we should look for .qml file inside zip file
643643
myURI.remove( 0, 8 );
644644
}
645+
else if ( myURI.startsWith( "/vsitar/", Qt::CaseInsensitive ) &&
646+
( myURI.endsWith( ".tar", Qt::CaseInsensitive ) ||
647+
myURI.endsWith( ".tar.gz", Qt::CaseInsensitive ) ||
648+
myURI.endsWith( ".tgz", Qt::CaseInsensitive ) ) )
649+
{
650+
// ideally we should look for .qml file inside tar file
651+
myURI.remove( 0, 8 );
652+
}
645653

646654
QFileInfo myFileInfo( myURI );
647655
QString key;
@@ -650,15 +658,18 @@ QString QgsMapLayer::styleURI( )
650658
{
651659
// if file is using the /vsizip/ or /vsigzip/ mechanism, cleanup the name
652660
if ( myURI.endsWith( ".gz", Qt::CaseInsensitive ) )
653-
{
654661
myURI.chop( 3 );
655-
myFileInfo.setFile( myURI );
656-
}
657662
else if ( myURI.endsWith( ".zip", Qt::CaseInsensitive ) )
658-
{
659663
myURI.chop( 4 );
660-
myFileInfo.setFile( myURI );
661-
}
664+
else if ( myURI.endsWith( ".tar", Qt::CaseInsensitive ) )
665+
myURI.chop( 4 );
666+
else if ( myURI.endsWith( ".tar.gz", Qt::CaseInsensitive ) )
667+
myURI.chop( 7 );
668+
else if ( myURI.endsWith( ".tgz", Qt::CaseInsensitive ) )
669+
myURI.chop( 4 );
670+
else if ( myURI.endsWith( ".gz", Qt::CaseInsensitive ) )
671+
myURI.chop( 3 );
672+
myFileInfo.setFile( myURI );
662673
// get the file name for our .qml style file
663674
key = myFileInfo.path() + QDir::separator() + myFileInfo.completeBaseName() + ".qml";
664675
}

‎src/providers/gdal/qgsgdaldataitems.cpp

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -399,10 +399,10 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
399399
// zip settings + info
400400
QSettings settings;
401401
QString scanZipSetting = settings.value( "/qgis/scanZipInBrowser", "basic" ).toString();
402-
bool is_vsizip = ( thePath.startsWith( "/vsizip/" ) ||
403-
thePath.endsWith( ".zip", Qt::CaseInsensitive ) );
404-
bool is_vsigzip = ( thePath.startsWith( "/vsigzip/" ) ||
405-
thePath.endsWith( ".gz", Qt::CaseInsensitive ) );
402+
QString vsiPrefix = QgsZipItem::vsiPrefix( thePath );
403+
bool is_vsizip = ( vsiPrefix == "/vsizip/" );
404+
bool is_vsigzip = ( vsiPrefix == "/vsigzip/" );
405+
bool is_vsitar = ( vsiPrefix == "/vsitar/" );
406406

407407
// get suffix, removing .gz if present
408408
QString tmpPath = thePath; //path used for testing, not for layer creation
@@ -414,10 +414,11 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
414414
info.setFile( thePath );
415415
QString name = info.fileName();
416416

417-
QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name + " suffix= " + suffix );
417+
QgsDebugMsg( "thePath= " + thePath + " tmpPath= " + tmpPath + " name= " + name
418+
+ " suffix= " + suffix + " vsiPrefix= " + vsiPrefix );
418419

419420
// allow only normal files or VSIFILE items to continue
420-
if ( !info.isFile() && !is_vsizip && !is_vsigzip )
421+
if ( !info.isFile() && vsiPrefix == "" )
421422
return 0;
422423

423424
// get supported extensions
@@ -434,10 +435,6 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
434435
!extensions.contains( "aux.xml" ) )
435436
return 0;
436437

437-
// skip .tar.gz files
438-
if ( thePath.endsWith( ".tar.gz", Qt::CaseInsensitive ) )
439-
return 0;
440-
441438
// Filter files by extension
442439
if ( !extensions.contains( suffix ) )
443440
{
@@ -455,30 +452,27 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
455452
return 0;
456453
}
457454

458-
// add /vsizip/ or /vsigzip/ to path if file extension is .zip or .gz
459-
if ( is_vsigzip )
460-
{
461-
if ( !thePath.startsWith( "/vsigzip/" ) )
462-
thePath = "/vsigzip/" + thePath;
463-
}
464-
else if ( is_vsizip )
455+
// fix vsifile path and name
456+
if ( vsiPrefix != "" )
465457
{
466-
if ( !thePath.startsWith( "/vsizip/" ) )
467-
thePath = "/vsizip/" + thePath;
458+
// add vsiPrefix to path if needed
459+
if ( !thePath.startsWith( vsiPrefix ) )
460+
thePath = vsiPrefix + thePath;
468461
// if this is a /vsigzip/path_to_zip.zip/file_inside_zip remove the full path from the name
469-
if ( thePath != "/vsizip/" + parentItem->path() )
462+
if (( is_vsizip || is_vsitar ) && ( thePath != vsiPrefix + parentItem->path() ) )
470463
{
471464
name = thePath;
472-
name = name.replace( "/vsizip/" + parentItem->path() + "/", "" );
465+
name = name.replace( vsiPrefix + parentItem->path() + "/", "" );
473466
}
474467
}
475468

476469
// return a /vsizip/ item without testing if:
477470
// zipfile and scan zip == "Basic scan"
478471
// not zipfile and scan items == "Check extension"
479-
if (( is_vsizip && scanZipSetting == "basic" ) ||
480-
( !is_vsizip && ( settings.value( "/qgis/scanItemsInBrowser",
481-
"extension" ).toString() == "extension" ) ) )
472+
if ((( is_vsizip || is_vsitar ) && scanZipSetting == "basic" ) ||
473+
( !is_vsizip && !is_vsitar &&
474+
( settings.value( "/qgis/scanItemsInBrowser",
475+
"extension" ).toString() == "extension" ) ) )
482476
{
483477
// if this is a VRT file make sure it is raster VRT to avoid duplicates
484478
if ( suffix == "vrt" )

0 commit comments

Comments
 (0)