Skip to content

Commit ebe0406

Browse files
ccrookccrook
committedApr 11, 2013
WIP
·
ltr-3_40archive/master_2
1 parent c6f678f commit ebe0406

12 files changed

+1125
-423
lines changed
 

‎src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ bool QgsDelimitedTextFeatureIterator::nextFeature( QgsFeature& feature )
7373
if ( !geom && P->mWkbType != QGis::WKBNoGeometry )
7474
{
7575
// Already dealt with invalid lines in provider - no need to repeat
76-
// P->mInvalidLines << line;
76+
// removed code (CC 2013-04-13) ...
77+
// P->mInvalidLines << line;
78+
// In any case it may be a valid line that is excluded because of
79+
// bounds check...
7780
continue;
7881
}
7982

@@ -153,7 +156,7 @@ QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryWkt( const QStringList
153156
// support these.
154157
if ( P->mWktHasZM )
155158
{
156-
sWkt.remove( P->mWktZMRegexp ).replace( P->mWktCrdRegexp, "\\1" );
159+
sWkt.remove( P->WktZMRegexp ).replace( P->WktCrdRegexp, "\\1" );
157160
}
158161

159162
geom = QgsGeometry::fromWkt( sWkt );
@@ -163,7 +166,7 @@ QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryWkt( const QStringList
163166
geom = 0;
164167
}
165168

166-
if ( geom && geom->wkbType() != P->mWkbType )
169+
if ( geom && geom->type() != P->mGeometryType )
167170
{
168171
delete geom;
169172
geom = 0;

‎src/providers/delimitedtext/qgsdelimitedtextfile.cpp

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
#include <QFile>
2323
#include <QDataStream>
2424
#include <QTextStream>
25+
#include <QTextCodec>
2526
#include <QStringList>
2627
#include <QRegExp>
2728
#include <QUrl>
2829

2930
QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
3031
mFileName(QString()),
32+
mEncoding("UTF-8"),
3133
mFile(0),
3234
mStream(0),
3335
mDefinitionValid(false),
@@ -74,6 +76,11 @@ bool QgsDelimitedTextFile::open()
7476
return false;
7577
}
7678
mStream = new QTextStream( mFile );
79+
if( mEncoding.isEmpty() && mEncoding != "System")
80+
{
81+
QTextCodec *codec = QTextCodec::codecForName(mEncoding.toAscii());
82+
mStream->setCodec(codec);
83+
}
7784
}
7885
return true;
7986
}
@@ -101,6 +108,12 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
101108
// Extract the file name
102109
setFileName( url.toLocalFile());
103110

111+
// Extract the encoding
112+
if( url.hasQueryItem("encoding"))
113+
{
114+
mEncoding =url.queryItemValue("encoding");
115+
}
116+
104117
// The default type is csv, to be consistent with the
105118
// previous implementation (except that quoting should be handled properly)
106119

@@ -150,19 +163,16 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
150163
}
151164

152165
QgsDebugMsg( "Delimited text file is: " + mFileName );
166+
QgsDebugMsg( "Encoding is: " + mEncoding);
153167
QgsDebugMsg( "Delimited file type is: " + type );
154-
QgsDebugMsg( "Delimiter is: " + delimiter );
155-
QgsDebugMsg( "Quote character is: " + quote);
156-
QgsDebugMsg( "Escape character is: " + escape);
168+
QgsDebugMsg( "Delimiter is: [" + delimiter + "]" );
169+
QgsDebugMsg( "Quote character is: [" + quote +"]");
170+
QgsDebugMsg( "Escape character is: [" + escape + "]");
157171
QgsDebugMsg( "Skip lines: " + QString::number(mSkipLines) );
158172
QgsDebugMsg( "Skip lines: " + QString(mUseHeader ? "Yes" : "No") );
159173

160-
if( type == "csv" )
161-
{
162-
setTypeCSV(delimiter,quote,escape);
163-
}
164174
// Support for previous version of plain characters
165-
else if( type == "plain" )
175+
if( type == "csv" || type == "plain" )
166176
{
167177
setTypeCSV(delimiter,quote,escape);
168178
}
@@ -184,6 +194,10 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
184194
QUrl QgsDelimitedTextFile::url()
185195
{
186196
QUrl url = QUrl::fromLocalFile( mFileName );
197+
if( mEncoding != "UTF-8" )
198+
{
199+
url.addQueryItem("encoding",mEncoding);
200+
}
187201
url.addQueryItem("type",type());
188202
if( mType == DelimTypeRegexp )
189203
{
@@ -212,6 +226,12 @@ void QgsDelimitedTextFile::setFileName( QString filename )
212226
mFileName = filename;
213227
}
214228

229+
void QgsDelimitedTextFile::setEncoding( QString encoding )
230+
{
231+
resetDefinition();
232+
mEncoding = encoding;
233+
}
234+
215235
QString QgsDelimitedTextFile::type()
216236
{
217237
if( mType == DelimTypeWhitespace ) return QString("whitespace");

‎src/providers/delimitedtext/qgsdelimitedtextfile.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,15 @@ class QgsDelimitedTextFile
102102
return mFileName;
103103
}
104104

105+
/** Set the file encoding (defuault is UTF-8)
106+
* @param encoding the encoding to use for the fileName()
107+
*/
108+
void setEncoding( QString encoding );
109+
/** Return the file encoding
110+
* @return encoding The file encoding
111+
*/
112+
QString encoding(){ return mEncoding; }
113+
105114
/** Decode the parser settings from a url as a string
106115
* @param url The url from which the delimiter and delimiterType items are read
107116
*/
@@ -259,6 +268,7 @@ class QgsDelimitedTextFile
259268
Status (QgsDelimitedTextFile::*mParser)( QStringList &fields );
260269

261270
QString mFileName;
271+
QString mEncoding;
262272
QFile *mFile;
263273
QTextStream *mStream;
264274

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 112 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "qgsfield.h"
3434
#include "qgsgeometry.h"
3535
#include "qgslogger.h"
36+
#include "qgsmessagelog.h"
3637
#include "qgsmessageoutput.h"
3738
#include "qgsrectangle.h"
3839
#include "qgis.h"
@@ -44,7 +45,9 @@
4445
static const QString TEXT_PROVIDER_KEY = "delimitedtext";
4546
static const QString TEXT_PROVIDER_DESCRIPTION = "Delimited text data provider";
4647

47-
48+
QRegExp QgsDelimitedTextProvider::WktPrefixRegexp("^\\s*(?:\\d+\\s+|SRID\\=\\d+\\;)" );
49+
QRegExp QgsDelimitedTextProvider::WktZMRegexp("\\s+(?:z|m|zm)(?=\\s*\\()", Qt::CaseInsensitive );
50+
QRegExp QgsDelimitedTextProvider::WktCrdRegexp("(\\-?\\d+(?:\\.\\d*)?\\s+\\-?\\d+(?:\\.\\d*)?)\\s[\\s\\d\\.\\-]+" );
4851

4952
QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
5053
: QgsVectorDataProvider( uri )
@@ -55,12 +58,11 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
5558
, mWktFieldIndex( -1 )
5659
, mWktHasZM( false )
5760
, mWktHasPrefix( false )
58-
, mWktZMRegexp( "\\s+(?:z|m|zm)(?=\\s*\\()", Qt::CaseInsensitive )
59-
, mWktCrdRegexp( "(\\-?\\d+(?:\\.\\d*)?\\s+\\-?\\d+(?:\\.\\d*)?)\\s[\\s\\d\\.\\-]+" )
60-
, mWktPrefixRegexp( "^(?:\\d+\\s+|SRID\\=\\d+\\;)" )
61-
, mShowInvalidLines( false )
61+
, mMaxInvalidLines( 50 )
62+
, mShowInvalidLines( true )
6263
, mCrs()
63-
, mWkbType( QGis::WKBUnknown )
64+
, mWkbType( QGis::WKBNoGeometry )
65+
, mGeometryType( QGis::UnknownGeometry )
6466
, mActiveIterator( 0 )
6567
{
6668

@@ -74,23 +76,46 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
7476
QString xField;
7577
QString yField;
7678

77-
if ( url.hasQueryItem( "wktField" ) )
78-
wktField = url.queryItemValue( "wktField" );
79-
if ( url.hasQueryItem( "xField" ) )
80-
xField = url.queryItemValue( "xField" );
81-
if ( url.hasQueryItem( "yField" ) )
82-
yField = url.queryItemValue( "yField" );
79+
if( url.hasQueryItem("geomType"))
80+
{
81+
QString gtype = url.queryItemValue("geomType").toLower();
82+
if( gtype == "point") mGeometryType = QGis::Point;
83+
else if( gtype == "line") mGeometryType = QGis::Line;
84+
else if( gtype == "polygon") mGeometryType = QGis::Polygon;
85+
else if( gtype == "none ") mGeometryType = QGis::NoGeometry;
86+
}
87+
88+
if( mGeometryType != QGis::NoGeometry )
89+
{
90+
if ( url.hasQueryItem( "wktField" ) )
91+
{
92+
wktField = url.queryItemValue( "wktField" );
93+
}
94+
else if ( url.hasQueryItem( "xField" ) && url.hasQueryItem( "yField" ) )
95+
{
96+
mGeometryType = QGis::Point;
97+
xField = url.queryItemValue( "xField" );
98+
yField = url.queryItemValue( "yField" );
99+
if ( url.hasQueryItem( "decimalPoint" ) )
100+
mDecimalPoint = url.queryItemValue( "decimalPoint" );
101+
}
102+
else
103+
{
104+
mGeometryType = QGis::NoGeometry;
105+
}
106+
}
83107
if ( url.hasQueryItem( "crs" ) )
84108
mCrs.createFromString( url.queryItemValue( "crs" ) );
85-
if ( url.hasQueryItem( "decimalPoint" ) )
86-
mDecimalPoint = url.queryItemValue( "decimalPoint" );
87109

88-
QgsDebugMsg( "wktField is: " + wktField );
110+
89111
QgsDebugMsg( "wktField is: " + wktField );
90112
QgsDebugMsg( "xField is: " + xField );
91113
QgsDebugMsg( "yField is: " + yField );
92114

115+
if( url.hasQueryItem("quiet")) mShowInvalidLines = false;
116+
93117
// assume the layer is invalid until proven otherwise
118+
clearInvalidLines();
94119
mValid = false;
95120
if ( ! mFile->isValid() )
96121
{
@@ -210,7 +235,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
210235
if( status == QgsDelimitedTextFile::RecordEOF ) break;
211236
if( status != QgsDelimitedTextFile::RecordOk )
212237
{
213-
mInvalidLines << ("Invalid record format at line "+QString::number(mFile->recordLineNumber()));
238+
recordInvalidLine(tr("Invalid record format at line %1"));
214239
continue;
215240
}
216241

@@ -224,6 +249,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
224249
if( ! parts[i].isEmpty() )
225250
{
226251
mFieldCount = i+1;
252+
break;
227253
}
228254
}
229255
}
@@ -243,48 +269,57 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
243269
QgsGeometry *geom = 0;
244270
try
245271
{
246-
if ( !mWktHasPrefix && sWkt.indexOf( mWktPrefixRegexp ) >= 0 )
272+
if ( !mWktHasPrefix && sWkt.indexOf( WktPrefixRegexp ) >= 0 )
247273
mWktHasPrefix = true;
248274
if ( mWktHasPrefix )
249275
{
250-
sWkt.remove( mWktPrefixRegexp );
276+
sWkt.remove( WktPrefixRegexp );
251277
}
252-
if ( !mWktHasZM && sWkt.indexOf( mWktZMRegexp ) >= 0 )
278+
if ( !mWktHasZM && sWkt.indexOf( WktZMRegexp ) >= 0 )
253279
mWktHasZM = true;
254280
if ( mWktHasZM )
255281
{
256-
sWkt.remove( mWktZMRegexp ).replace( mWktCrdRegexp, "\\1" );
282+
sWkt.remove( WktZMRegexp ).replace( WktCrdRegexp, "\\1" );
257283
}
258284
geom = QgsGeometry::fromWkt( sWkt );
259285
}
260286
catch ( ... )
261287
{
262-
mInvalidLines << ("Invalid WKT at line "+QString::number(mFile->recordLineNumber()));
263-
geom = 0;
288+
// If something went wrong, just note invalid geometry
289+
// as geom will be null
264290
}
265291

266292
if ( geom )
267293
{
268294
QGis::WkbType type = geom->wkbType();
269295
if ( type != QGis::WKBNoGeometry )
270296
{
271-
if ( mNumberFeatures == 0 )
272-
{
273-
mNumberFeatures++;
274-
mWkbType = type;
275-
mExtent = geom->boundingBox();
276-
}
277-
else if ( type == mWkbType )
297+
if( mGeometryType==QGis::UnknownGeometry || geom->type()==mGeometryType)
278298
{
279-
mNumberFeatures++;
280-
QgsRectangle bbox( geom->boundingBox() );
281-
mExtent.combineExtentWith( &bbox );
299+
mGeometryType = geom->type();
300+
if ( mNumberFeatures == 0 )
301+
{
302+
mNumberFeatures++;
303+
mWkbType = type;
304+
mExtent = geom->boundingBox();
305+
}
306+
else
307+
{
308+
mNumberFeatures++;
309+
if( geom->isMultipart()) mWkbType=type;
310+
QgsRectangle bbox( geom->boundingBox() );
311+
mExtent.combineExtentWith( &bbox );
312+
}
282313
}
283314
}
284315
delete geom;
285316
}
317+
else
318+
{
319+
recordInvalidLine(tr("Invalid WKT at line %1"));
320+
}
286321
}
287-
else if ( mWktFieldIndex == -1 && mXFieldIndex >= 0 && mYFieldIndex >= 0 )
322+
else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
288323
{
289324
// Get the x and y values, first checking to make sure they
290325
// aren't null.
@@ -315,12 +350,13 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
315350
// Extent for the first point is just the first point
316351
mExtent.set( x, y, x, y );
317352
mWkbType = QGis::WKBPoint;
353+
mGeometryType = QGis::Point;
318354
}
319355
mNumberFeatures++;
320356
}
321357
else
322358
{
323-
mInvalidLines << ("Invalid X or Y fields at line "+QString::number(mFile->recordLineNumber()));
359+
recordInvalidLine(tr("Invalid X or Y fields at line %1"));
324360
}
325361
}
326362
else
@@ -333,7 +369,6 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
333369
for ( int i = 0; i < mFieldCount; i++ )
334370
{
335371
if( i == mWktFieldIndex ) continue;
336-
fieldPos++;
337372
if( fieldPos >= attributeColumns.count() )
338373
{
339374
QString field = "Col"+QString("%1").arg(i+1,2,10,QChar('0'));
@@ -357,6 +392,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
357392
{
358393
value.toDouble( &couldBeDouble[fieldPos] );
359394
}
395+
fieldPos++;
360396
}
361397
}
362398

@@ -381,7 +417,8 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
381417
}
382418
}
383419

384-
mValid = mWkbType != QGis::WKBUnknown;
420+
handleInvalidLines();
421+
mValid = mGeometryType != QGis::UnknownGeometry;
385422
}
386423

387424
QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
@@ -411,27 +448,53 @@ QgsFeatureIterator QgsDelimitedTextProvider::getFeatures( const QgsFeatureReques
411448
return QgsFeatureIterator( new QgsDelimitedTextFeatureIterator( this, request ) );
412449
}
413450

451+
void QgsDelimitedTextProvider::clearInvalidLines()
452+
{
453+
mInvalidLines.clear();
454+
mNExtraInvalidLines = 0;
455+
}
456+
457+
void QgsDelimitedTextProvider::recordInvalidLine( QString message )
458+
{
459+
if( mInvalidLines.size() < mMaxInvalidLines )
460+
{
461+
mInvalidLines.append( message.arg(mFile->recordLineNumber()) );
462+
}
463+
else
464+
{
465+
mNExtraInvalidLines++;
466+
}
467+
}
414468

415469
void QgsDelimitedTextProvider::handleInvalidLines()
416470
{
417-
if ( mShowInvalidLines && !mInvalidLines.isEmpty() )
471+
if ( !mInvalidLines.isEmpty() )
418472
{
419-
mShowInvalidLines = false;
420-
QgsMessageOutput* output = QgsMessageOutput::createMessageOutput();
421-
output->setTitle( tr( "Error" ) );
422-
output->setMessage( tr( "Note: the following lines were not loaded because QGIS was "
423-
"unable to determine values for the x and y coordinates:\n" ),
424-
QgsMessageOutput::MessageText );
425-
426-
output->appendMessage( "Start of invalid lines." );
473+
QString tag("DelimitedText");
474+
QgsMessageLog::logMessage( tr( "Errors in %1").arg( mFile->fileName() ), tag );
475+
QgsMessageLog::logMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg(mFile->fileName()),
476+
tag );
427477
for ( int i = 0; i < mInvalidLines.size(); ++i )
428-
output->appendMessage( mInvalidLines.at( i ) );
429-
output->appendMessage( "End of invalid lines." );
478+
QgsMessageLog::logMessage( mInvalidLines.at( i ), tag );
479+
if( mNExtraInvalidLines > 0 )
480+
QgsMessageLog::logMessage(tr("There are %1 additional errors in the file").arg(mNExtraInvalidLines), tag);
430481

431-
output->showMessage();
482+
// Display errors in a dialog...
483+
if( mShowInvalidLines )
484+
{
485+
QgsMessageOutput* output = QgsMessageOutput::createMessageOutput();
486+
output->setTitle( tr( "Delimited text file errors" ) );
487+
output->setMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg(mFile->fileName()),
488+
QgsMessageOutput::MessageText );
489+
for ( int i = 0; i < mInvalidLines.size(); ++i )
490+
output->appendMessage( mInvalidLines.at( i ) );
491+
if( mNExtraInvalidLines > 0 )
492+
output->appendMessage(tr("There are %1 additional errors in the file").arg(mNExtraInvalidLines));
493+
output->showMessage();
494+
}
432495

433496
// We no longer need these lines.
434-
mInvalidLines.clear();
497+
clearInvalidLines();
435498
}
436499
}
437500

‎src/providers/delimitedtext/qgsdelimitedtextprovider.h

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
4949

5050
public:
5151

52+
/**
53+
* Regular expression defining possible prefixes to WKT string,
54+
* (EWKT srid, Informix SRID)
55+
*/
56+
static QRegExp WktPrefixRegexp;
57+
5258
QgsDelimitedTextProvider( QString uri = QString() );
5359

5460
virtual ~QgsDelimitedTextProvider();
@@ -151,6 +157,12 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
151157
bool boundsCheck( QgsGeometry *geom );
152158

153159
private:
160+
161+
static QRegExp WktZMRegexp;
162+
static QRegExp WktCrdRegexp;
163+
164+
void clearInvalidLines();
165+
void recordInvalidLine( QString message );
154166
void handleInvalidLines();
155167
void resetStream();
156168

@@ -174,9 +186,6 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
174186

175187
bool mWktHasZM;
176188
bool mWktHasPrefix;
177-
QRegExp mWktZMRegexp;
178-
QRegExp mWktCrdRegexp;
179-
QRegExp mWktPrefixRegexp;
180189

181190
//! Layer extent
182191
QgsRectangle mExtent;
@@ -190,6 +199,8 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
190199
QString mDecimalPoint;
191200

192201
//! Storage for any lines in the file that couldn't be loaded
202+
int mMaxInvalidLines;
203+
int mNExtraInvalidLines;
193204
QStringList mInvalidLines;
194205
//! Only want to show the invalid lines once to the user
195206
bool mShowInvalidLines;
@@ -207,6 +218,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
207218
QgsCoordinateReferenceSystem mCrs;
208219

209220
QGis::WkbType mWkbType;
221+
QGis::GeometryType mGeometryType;
210222

211223
friend class QgsDelimitedTextFeatureIterator;
212224
QgsDelimitedTextFeatureIterator* mActiveIterator;

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp

Lines changed: 346 additions & 157 deletions
Large diffs are not rendered by default.

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
4040
void updateFieldLists();
4141
void getOpenFileName();
4242
QString selectedChars();
43+
void setSelectedChars( QString delimiters );
44+
void loadSettings( QString subkey=QString(), bool loadGeomSettings=true );
45+
void saveSettings( QString subkey=QString(), bool saveGeomSettings=true );
46+
void loadSettingsForFile( QString filename );
47+
void saveSettingsForFile( QString filename );
48+
bool trySetXYField(QStringList &fields, QList<bool> &isValidNumber, QString xname, QString yname );
4349

4450
private:
4551
QgsDelimitedTextFile *mFile;

‎src/ui/qgsdelimitedtextsourceselectbase.ui

Lines changed: 414 additions & 203 deletions
Large diffs are not rendered by default.

‎tests/src/python/test_qgsdelimitedtextprovider.py

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# -*- coding: utf-8 -*-
2+
"""QGIS Unit tests for QgsDelimitedTextProvider.
3+
4+
.. note:: This program is free software; you can redistribute it and/or modify
5+
it under the terms of the GNU General Public License as published by
6+
the Free Software Foundation; either version 2 of the License, or
7+
(at your option) any later version.
8+
"""
9+
__author__ = 'Alexander Bruy'
10+
__date__ = '20/08/2012'
11+
__copyright__ = 'Copyright 2012, The Quantum GIS Project'
12+
# This will get replaced with a git SHA1 when you do a git archive
13+
__revision__ = '$Format:%H$'
14+
15+
import os.path;
16+
17+
from PyQt4.QtCore import (QVariant,
18+
QUrl
19+
)
20+
21+
from qgis.core import (QGis,
22+
QgsProviderRegistry,
23+
QgsVectorLayer,
24+
QgsFeature,
25+
QgsFeatureRequest,
26+
QgsField,
27+
QgsGeometry,
28+
QgsPoint)
29+
30+
from utilities import (getQgisTestApp,
31+
TestCase,
32+
unitTestDataPath,
33+
unittest
34+
#expectedFailure
35+
)
36+
QGISAPP, CANVAS, IFACE, PARENT = getQgisTestApp()
37+
38+
39+
class TestQgsDelimitedTextProvider(TestCase):
40+
41+
def test_001_ProviderDefined( self ):
42+
registry=QgsProviderRegistry.instance()
43+
metadata = registry.providerMetadata('delimitedtext')
44+
assert metadata != None and metadata.description == "Delimited text data provider", "Provider metadata is installed"
45+
46+
def test_002_LoadCSVFile( self ):
47+
file = os.path.join(unitTestDataPath("delimitedtext"),"test.csv");
48+
url = QUrl.fromLocalFile(file);
49+
urlstr = url.toString()+"?type=csv&xField=x&yField=y"
50+
layer = QgsVectorLayer(urlstr, "test", "delimitedtext")
51+
assert layer.isValid(), "Failed to create delimited text layer " + urlstr
52+
53+
# def testLayerGeometry(self):
54+
# layer = QgsVectorLayer("Point", "test", "memory")
55+
#
56+
# myMessage = ('Expected: %s\nGot: %s\n' %
57+
# (QGis.Point, layer.geometryType()))
58+
# assert layer.geometryType() == QGis.Point, myMessage
59+
#
60+
# myMessage = ('Expected: %s\nGot: %s\n' %
61+
# (QGis.WKBPoint, layer.wkbType()))
62+
# assert layer.wkbType() == QGis.WKBPoint, myMessage
63+
#
64+
# def testAddFeatures(self):
65+
# layer = QgsVectorLayer("Point", "test", "memory")
66+
# provider = layer.dataProvider()
67+
#
68+
# res = provider.addAttributes([QgsField("name", QVariant.String,),
69+
# QgsField("age", QVariant.Int),
70+
# QgsField("size", QVariant.Double)])
71+
# assert res, "Failed to add attributes"
72+
#
73+
# myMessage = ('Expected: %s\nGot: %s\n' %
74+
# (3, len(provider.fields())))
75+
#
76+
# assert len(provider.fields()) == 3, myMessage
77+
#
78+
# ft = QgsFeature()
79+
# ft.setGeometry(QgsGeometry.fromPoint(QgsPoint(10,10)))
80+
# ft.setAttributes([ QVariant("Johny"),
81+
# QVariant(20),
82+
# QVariant(0.3) ])
83+
# res, t = provider.addFeatures([ft])
84+
#
85+
# assert res, "Failed to add feature"
86+
#
87+
# myMessage = ('Expected: %s\nGot: %s\n' %
88+
# (1, provider.featureCount()))
89+
# assert provider.featureCount() == 1, myMessage
90+
#
91+
# for f in provider.getFeatures(QgsFeatureRequest()):
92+
# attrs = f.attributes()
93+
# myMessage = ('Expected: %s\nGot: %s\n' %
94+
# ("Johny", str(attrs[0].toString())))
95+
#
96+
# assert str(attrs[0].toString()) == "Johny", myMessage
97+
#
98+
# myMessage = ('Expected: %s\nGot: %s\n' %
99+
# (20, attrs[1].toInt()[0]))
100+
#
101+
# assert attrs[1].toInt()[0] == 20, myMessage
102+
#
103+
# myMessage = ('Expected: %s\nGot: %s\n' %
104+
# (0.3, attrs[2].toFloat()[0]))
105+
#
106+
# assert (attrs[0].toFloat()[0] - 0.3) < 0.0000001, myMessage
107+
#
108+
# geom = f.geometry()
109+
#
110+
# myMessage = ('Expected: %s\nGot: %s\n' %
111+
# ("POINT(10.0 10.0)", str(geom.exportToWkt())))
112+
#
113+
# assert str(geom.exportToWkt()) == "POINT(10.0 10.0)", myMessage
114+
#
115+
# def testGetFields(self):
116+
# layer = QgsVectorLayer("Point", "test", "memory")
117+
# provider = layer.dataProvider()
118+
#
119+
# provider.addAttributes([QgsField("name", QVariant.String,),
120+
# QgsField("age", QVariant.Int),
121+
# QgsField("size", QVariant.Double)])
122+
# myMessage = ('Expected: %s\nGot: %s\n' %
123+
# (3, len(provider.fields())))
124+
#
125+
# assert len(provider.fields()) == 3, myMessage
126+
#
127+
# ft = QgsFeature()
128+
# ft.setGeometry(QgsGeometry.fromPoint(QgsPoint(10,10)))
129+
# ft.setAttributes([QVariant("Johny"),
130+
# QVariant(20),
131+
# QVariant(0.3)])
132+
# provider.addFeatures([ft])
133+
#
134+
# for f in provider.getFeatures(QgsFeatureRequest()):
135+
# myMessage = ('Expected: %s\nGot: %s\n' %
136+
# ("Johny", str(f['name'].toString())))
137+
#
138+
# self.assertEqual(str(f["name"].toString()), "Johny", myMessage)
139+
#
140+
#
141+
# def testFromUri(self):
142+
# """Test we can construct the mem provider from a uri"""
143+
# myDelimitedTextLayer = QgsVectorLayer(
144+
# ('Point?crs=epsg:4326&field=name:string(20)&'
145+
# 'field=age:integer&field=size:double&index=yes'),
146+
# 'test',
147+
# 'memory')
148+
#
149+
# assert myDelimitedTextLayer is not None, 'Provider not initialised'
150+
# myProvider = myDelimitedTextLayer.dataProvider()
151+
# assert myProvider is not None
152+
153+
if __name__ == '__main__':
154+
unittest.main()

‎tests/testdata/delimitedtext/test.csv

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
id,"description",x,y,wkt
2+
1,"Basic point/WKT",10.0,20.0,"POLYGON((10.0 20.0,25.0 20.0,25.0 30.0,10.0 30.0))"
3+
2,"Invalid point coordinate",1O.O,20.0,"POLYGON((10.0 20.0,25.0 20.0,25.0 30.0,10.0 30.0))"
4+
3,"Invalid WKT definition",15.0,20.0,"POLYGON((10.0 20.0 25.0 20.0,25.0 30.0,10.0 30.0))"
5+
4,"Alternative WKT definition",25.0,20.0,"POLYGON M((10.0 20.0 2,25.0 20.0 3.5,25.0 30.0 -4,10.0 30.0 -3.5))"
6+
6,"Alternative WKT definition",30.0,20.0,"POLYGON M((10.0 20.0 2,25.0 20.0 3.5,25.0 30.0 -4,10.0 30.0 -3.5))"
7+
7,"Alternative WKT definition",35.0,20.0,"POLYGON M((10.0 20.0 2,25.0 20.0 3.5,25.0 30.0 -4,10.0 30.0 -3.5))"
8+
7,"WKT prefix (EWKT)",40.0,20.0,"SRID=1234;POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
9+
8,"WKT prefix (Informix)",40.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
10+
20,"Quoting test - embedded quotes"" ''",45.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
11+
21,"Quoting test 2 - new line
12+
in record",50.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
13+
21,"Quoting test 2 - multiple new lines
14+
15+
16+
17+
in record",50.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
18+
22, "Quoting test 3 - spacing before and after" ,55.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
19+
23,"Quoting test 3 invalid quoting" ",55.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
20+
30,"UTF8 test with àccénts",55.0,20.0,"2 POLYGON((10.0 20.0,25.0 20.0,25.0,10.0 30.0))"
21+

‎tests/testdata/delimitedtext/test.pipe

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
id|name|x
2+
1|john|23.5|
3+
2|piper\|two
4+
3|multiline \
5+
name|433
6+
4|"quotes | test"|42.0
7+
5|"quotes |
8+
test2"|43.0
9+
6|with~another~delimiter|
10+

‎tests/testdata/delimitedtext/test.space

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
field1 field2 field3
2+
field21 field22 field23
3+

0 commit comments

Comments
 (0)
Please sign in to comment.