Skip to content

Commit 5e4f4f7

Browse files
ccrookccrook
committedApr 18, 2013
More testing and bug fixes
Regular expression delimiter: fixed non-intuitive behaviour of not including capture groups in the returned fields, and improved behaviour for anchored regular expressions. Added test cases for handling of regular expressions. File encoding: Selected file encoding was being ignored. Fixed and added test cases for file encoding. Bad formatted quoting: Fixed handling of badly formatted quotes (eg unclosed quotes). Added test case. Field names: Where a file had duplicate or blank field names it was not possible to reliably set the names of X,Y,WKT fields. Relocated field name handling to simpler common code in QgsDelimitedTextFile, with automatic renaming of fields with ambiguous or missing names. Add test case for field renaming User interface fixes: Fixed tooltip and other messages throughout. Changed encoding combo box to be alphabetically sorted. Fixed data sample to be refreshed when encoding changed.
·
ltr-3_40archive/master_2
1 parent d882c4c commit 5e4f4f7

15 files changed

+722
-308
lines changed
 

‎src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ bool QgsDelimitedTextFeatureIterator::nextFeature( QgsFeature& feature )
5656
int fid = P->mFile->recordLineNumber();
5757
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
5858
if ( status != QgsDelimitedTextFile::RecordOk ) continue;
59-
if( P->recordIsEmpty(tokens)) continue;
59+
if ( P->recordIsEmpty( tokens ) ) continue;
6060

6161
while ( tokens.size() < P->mFieldCount )
6262
tokens.append( QString::null );

‎src/providers/delimitedtext/qgsdelimitedtextfile.cpp

Lines changed: 172 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@
2727
#include <QRegExp>
2828
#include <QUrl>
2929

30+
static QString DefaultFieldName( "field_%1" );
31+
static QRegExp InvalidFieldRegexp( "^\\d*(\\.\\d*)?$" );
32+
// field_ is optional in following regexp to simplify QgsDelimitedTextFile::fieldNumber()
33+
static QRegExp DefaultFieldRegexp( "^(?:field_)?(\\d+)$", Qt::CaseInsensitive );
34+
3035
QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
3136
mFileName( QString() ),
3237
mEncoding( "UTF-8" ),
@@ -39,7 +44,8 @@ QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
3944
mSkipLines( 0 ),
4045
mMaxFields( 0 ),
4146
mLineNumber( 0 ),
42-
mRecordLineNumber( 0 )
47+
mRecordLineNumber( 0 ),
48+
mMaxFieldCount( 0 )
4349
{
4450
// The default type is CSV
4551
setTypeCSV();
@@ -80,7 +86,7 @@ bool QgsDelimitedTextFile::open()
8086
return false;
8187
}
8288
mStream = new QTextStream( mFile );
83-
if ( mEncoding.isEmpty() && mEncoding != "System" )
89+
if ( ! mEncoding.isEmpty() )
8490
{
8591
QTextCodec *codec = QTextCodec::codecForName( mEncoding.toAscii() );
8692
mStream->setCodec( codec );
@@ -93,8 +99,8 @@ bool QgsDelimitedTextFile::open()
9399
void QgsDelimitedTextFile::resetDefinition()
94100
{
95101
close();
96-
mColumnNames.clear();
97-
mMaxFields = 0;
102+
mFieldNames.clear();
103+
mMaxFieldCount = 0;
98104
}
99105

100106
// Extract the provider definition from the url
@@ -275,12 +281,18 @@ void QgsDelimitedTextFile::setTypeRegexp( QString regexp )
275281
resetDefinition();
276282
mType = DelimTypeRegexp;
277283
mDelimRegexp.setPattern( regexp );
284+
mAnchoredRegexp = regexp.startsWith( "^" );
278285
mParser = &QgsDelimitedTextFile::parseRegexp;
279286
mDefinitionValid = regexp.size() > 0 && mDelimRegexp.isValid();
280287
if ( ! mDefinitionValid )
281288
{
282289
QgsDebugMsg( "Invalid regular expression in delimited text file delimiter: " + regexp );
283290
}
291+
else if ( mAnchoredRegexp && mDelimRegexp.captureCount() == 0 )
292+
{
293+
mDefinitionValid = false;
294+
QgsDebugMsg( "Invalid anchored regular expression - must have capture groups: " + regexp );
295+
}
284296
}
285297

286298
QString QgsDelimitedTextFile::decodeChars( QString chars )
@@ -335,12 +347,89 @@ void QgsDelimitedTextFile::setDiscardEmptyFields( bool discardEmptyFields )
335347
mDiscardEmptyFields = discardEmptyFields;
336348
}
337349

338-
QStringList &QgsDelimitedTextFile::columnNames()
350+
351+
void QgsDelimitedTextFile::setFieldNames( const QStringList &names )
352+
{
353+
mFieldNames.empty();
354+
foreach ( QString name, names )
355+
{
356+
bool nameOk = true;
357+
int fieldNo = mFieldNames.size() + 1;
358+
name = name.trimmed();
359+
360+
// If the name is invalid then reset it to default name
361+
if ( InvalidFieldRegexp.exactMatch( name ) )
362+
{
363+
name = DefaultFieldName.arg( fieldNo );
364+
}
365+
// If the name looks like a default field name (field_##), then it is
366+
// valid if the number matches its column number..
367+
else if ( DefaultFieldRegexp.indexIn( name ) == 0 )
368+
{
369+
int col = DefaultFieldRegexp.capturedTexts()[1].toInt();
370+
nameOk = col == fieldNo;
371+
}
372+
// Otherwise it is valid if isn't the name of an existing field...
373+
else
374+
{
375+
nameOk = ! mFieldNames.contains( name, Qt::CaseInsensitive );
376+
}
377+
// If it is not a valid name then try appending a number to generate
378+
// a valid name.
379+
if ( ! nameOk )
380+
{
381+
int suffix = 0;
382+
QString basename = name + "_%1";
383+
while ( true )
384+
{
385+
suffix++;
386+
name = basename.arg( suffix );
387+
// Not ok if it is already in the name list
388+
if ( mFieldNames.contains( name, Qt::CaseInsensitive ) ) continue;
389+
// Not ok if it is already in proposed names
390+
if ( names.contains( name, Qt::CaseInsensitive ) ) continue;
391+
break;
392+
}
393+
}
394+
mFieldNames.append( name );
395+
}
396+
}
397+
398+
399+
QStringList &QgsDelimitedTextFile::fieldNames()
339400
{
340401
// If not yet opened then reset file to read column headers
341402
//
342403
if ( mUseHeader && ! mFile ) reset();
343-
return mColumnNames;
404+
// If have read more fields than field names, then append field names
405+
// to match the field count (will only happen if parsed some records)
406+
if ( mMaxFieldCount > mFieldNames.size() )
407+
{
408+
for ( int i = mFieldNames.size() + 1; i <= mMaxFieldCount; i++ )
409+
{
410+
mFieldNames.append( DefaultFieldName.arg( i ) );
411+
}
412+
}
413+
return mFieldNames;
414+
}
415+
416+
int QgsDelimitedTextFile::fieldIndex( QString name )
417+
{
418+
// If not yet opened then reset file to read column headers
419+
//
420+
if ( mUseHeader && ! mFile ) reset();
421+
// Try to determine the field based on a default field name, includes
422+
// Field_### and simple integer fields.
423+
if ( DefaultFieldRegexp.indexIn( name ) == 0 )
424+
{
425+
return DefaultFieldRegexp.capturedTexts()[1].toInt() - 1;
426+
}
427+
for ( int i = 0; i < mFieldNames.size(); i++ )
428+
{
429+
if ( mFieldNames[i].compare( name, Qt::CaseInsensitive ) == 0 ) return i;
430+
}
431+
return -1;
432+
344433
}
345434

346435
QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextRecord( QStringList &record )
@@ -368,8 +457,9 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::reset()
368457
// Read the column names
369458
if ( mUseHeader )
370459
{
371-
QgsDelimitedTextFile::Status result = ( this->*mParser )( mColumnNames );
372-
mMaxFields = mColumnNames.size();
460+
QStringList names;
461+
QgsDelimitedTextFile::Status result = nextRecord( names );
462+
setFieldNames( names );
373463
return result;
374464
}
375465
return RecordOk;
@@ -396,6 +486,22 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextLine( QString &buffer, bo
396486
return RecordEOF;
397487
}
398488

489+
void QgsDelimitedTextFile::appendField( QStringList &record, QString field, bool quoted )
490+
{
491+
if ( mMaxFields > 0 && record.size() >= mMaxFields ) return;
492+
if ( quoted )
493+
{
494+
record.append( field );
495+
}
496+
else
497+
{
498+
if ( mTrimFields ) field = field.trimmed();
499+
if ( !( mDiscardEmptyFields && field.isEmpty() ) ) record.append( field );
500+
}
501+
// Keep track of maximum number of non-empty fields in a record
502+
if ( record.size() > mMaxFieldCount && ! field.isEmpty() ) mMaxFieldCount = record.size();
503+
}
504+
399505
QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QStringList &fields )
400506
{
401507
fields.clear();
@@ -404,12 +510,54 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QStringList &fie
404510
if ( status != RecordOk ) return status;
405511
mRecordLineNumber = mLineNumber;
406512

407-
QStringList parts = buffer.split( mDelimRegexp );
408-
foreach ( QString f, parts )
513+
// If match is anchored, then only interested in records which actually match
514+
// and extract capture groups
515+
if ( mAnchoredRegexp )
516+
{
517+
if ( mDelimRegexp.indexIn( buffer ) < 0 ) return RecordInvalid;
518+
QStringList groups = mDelimRegexp.capturedTexts();
519+
for ( int i = 1; i < groups.size(); i++ )
520+
{
521+
appendField( fields, groups[i] );
522+
}
523+
return RecordOk;
524+
}
525+
526+
int pos = 0;
527+
int size = buffer.size();
528+
while ( true )
409529
{
410-
if ( mTrimFields ) f = f.trimmed();
411-
if ( mDiscardEmptyFields && f.isEmpty() ) continue;
412-
fields.append( f );
530+
if ( pos >= size ) break;
531+
int matchPos = mDelimRegexp.indexIn( buffer, pos );
532+
// If match won't advance cursor, then need to force it along one place
533+
// to avoid infinite loop.
534+
int matchLen = mDelimRegexp.matchedLength();
535+
if ( matchPos == pos && matchLen == 0 )
536+
{
537+
matchPos = mDelimRegexp.indexIn( buffer, pos + 1 );
538+
matchLen = mDelimRegexp.matchedLength();
539+
}
540+
// If no match, then field is to end of record
541+
if ( matchPos < 0 )
542+
{
543+
appendField( fields, buffer.mid( pos ) );
544+
break;
545+
}
546+
// Else append up to matched string, then any capture
547+
// groups from match
548+
appendField( fields, buffer.mid( pos, matchPos - pos ) );
549+
if ( mDelimRegexp.captureCount() > 0 )
550+
{
551+
QStringList groups = mDelimRegexp.capturedTexts();
552+
for ( int i = 1; i < groups.size(); i++ )
553+
{
554+
appendField( fields, groups[i] );
555+
}
556+
}
557+
// Advance the buffer pointer
558+
pos = matchPos + matchLen;
559+
560+
// Quit loop if we have enough fields.
413561
if ( mMaxFields > 0 && fields.size() >= mMaxFields ) break;
414562
}
415563
return RecordOk;
@@ -445,7 +593,11 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
445593
if ( quoted || escaped )
446594
{
447595
status = nextLine( buffer, false );
448-
if ( status != RecordOk ) return status;
596+
if ( status != RecordOk )
597+
{
598+
status = RecordInvalid;
599+
break;
600+
}
449601
field.append( '\n' );
450602
cp = 0;
451603
cpmax = buffer.size();
@@ -530,12 +682,8 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
530682
// If it is a delimiter, then end of field...
531683
else if ( isDelim )
532684
{
533-
if ( mMaxFields <= 0 || fields.size() < mMaxFields )
534-
{
535-
// If wasn't quoted, then trim..
536-
if ( mTrimFields && ! ended ) field = field.trimmed();
537-
if ( ! field.isEmpty() || ended || ! mDiscardEmptyFields ) fields.append( field );
538-
}
685+
appendField( fields, field, ended );
686+
539687
// Clear the field
540688
field.clear();
541689
started = false;
@@ -560,12 +708,12 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
560708
}
561709
}
562710
// If reached the end of the record, then add the last field...
563-
if ( started && ( mMaxFields <= 0 || fields.size() < mMaxFields ) )
711+
if ( started )
564712
{
565-
if ( mTrimFields && ! ended ) field = field.trimmed();
566-
if ( ! field.isEmpty() || ended || ! mDiscardEmptyFields ) fields.append( field );
713+
appendField( fields, field, ended );
714+
567715
}
568-
return RecordOk;
716+
return status;
569717
}
570718

571719
bool QgsDelimitedTextFile::isValid()

‎src/providers/delimitedtext/qgsdelimitedtextfile.h

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -154,12 +154,12 @@ class QgsDelimitedTextFile
154154
return mSkipLines;
155155
}
156156

157-
/* Set reading column names from the first record
158-
* @param useheaders Column names will be read if true
157+
/* Set reading field names from the first record
158+
* @param useheaders Field names will be read if true
159159
*/
160160
void setUseHeader( bool useheader = true );
161-
/* Return the option for reading column names from the first record
162-
* @return useheaders Column names will be read if true
161+
/* Return the option for reading field names from the first record
162+
* @return useheaders Field names will be read if true
163163
*/
164164
bool useHeader()
165165
{
@@ -190,11 +190,26 @@ class QgsDelimitedTextFile
190190
return mTrimFields;
191191
}
192192

193-
/** Return the column names read from the header, or default names
193+
/** Set the field names
194+
* Field names are set from QStringList. Names may be modified
195+
* to ensure that they are unique, not empty, and do not conflict
196+
* with default field name (Field_##)
197+
*/
198+
void setFieldNames( const QStringList &names );
199+
200+
/** Return the field names read from the header, or default names
194201
* Col## if none defined. Will open and read the head of the file
195202
* if required, then reset..
196203
*/
197-
QStringList &columnNames();
204+
QStringList &fieldNames();
205+
206+
/** Return the index of a names field
207+
* @param name The name of the field to find. This will also accept an
208+
* integer string ("1" = first field).
209+
* @return index The zero based index of the field name, or -1 if the field
210+
* name does not exist or cannot be inferred
211+
*/
212+
int fieldIndex( QString name );
198213

199214
/** Reads the next record from the stream splits into string fields.
200215
* @param fields The string list to populate with the fields
@@ -242,9 +257,6 @@ class QgsDelimitedTextFile
242257
*/
243258
static QString decodeChars( QString string );
244259

245-
246-
247-
248260
private:
249261

250262
/** Open the file
@@ -258,7 +270,7 @@ class QgsDelimitedTextFile
258270
void close();
259271

260272
/** Reset the status if the definition is changing (eg clear
261-
* existing column names, etc...
273+
* existing field names, etc...
262274
*/
263275
void resetDefinition();
264276

@@ -273,6 +285,12 @@ class QgsDelimitedTextFile
273285
*/
274286
Status nextLine( QString &buffer, bool skipBlank = false );
275287

288+
/** Utility routine to add a field to a record, accounting for trimming
289+
* and discarding, and maximum field count
290+
*/
291+
292+
void appendField( QStringList &record, QString field, bool quoted = false );
293+
276294
// Pointer to the currently selected parser
277295
Status( QgsDelimitedTextFile::*mParser )( QStringList &fields );
278296

@@ -292,12 +310,14 @@ class QgsDelimitedTextFile
292310

293311
// Parameters used by parsers
294312
QRegExp mDelimRegexp;
313+
bool mAnchoredRegexp;
295314
QString mDelimChars;
296315
QString mQuoteChar;
297316
QString mEscapeChar;
298317

299318
// Information extracted from file
300-
QStringList mColumnNames;
319+
QStringList mFieldNames;
301320
int mLineNumber;
302321
int mRecordLineNumber;
322+
int mMaxFieldCount;
303323
};

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 188 additions & 199 deletions
Large diffs are not rendered by default.

‎src/providers/delimitedtext/qgsdelimitedtextprovider.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
172172

173173
void clearInvalidLines();
174174
void recordInvalidLine( QString message );
175-
void handleInvalidLines();
175+
void reportErrors( QStringList messages = QStringList() );
176176
void resetStream();
177177
bool recordIsEmpty( QStringList &record );
178178

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp

Lines changed: 42 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,8 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
3434
QDialog( parent, fl ),
3535
mFile( new QgsDelimitedTextFile() ),
3636
mExampleRowCount( 20 ),
37-
mColumnNamePrefix( "Column_" ),
3837
mPluginKey( "/Plugin-DelimitedText" ),
39-
mLastFileType("")
38+
mLastFileType( "" )
4039
{
4140

4241
setupUi( this );
@@ -51,7 +50,13 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
5150
}
5251

5352
cmbEncoding->clear();
53+
QStringList codecs;
5454
foreach ( QByteArray codec, QTextCodec::availableCodecs() )
55+
{
56+
codecs.append( codec );
57+
}
58+
codecs.sort();
59+
foreach( QString codec, codecs )
5560
{
5661
cmbEncoding->addItem( codec );
5762
}
@@ -62,6 +67,7 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
6267

6368
connect( txtFilePath, SIGNAL( textChanged( QString ) ), this, SLOT( updateFileName() ) );
6469
connect( txtLayerName, SIGNAL( textChanged( QString ) ), this, SLOT( enableAccept() ) );
70+
connect( cmbEncoding, SIGNAL( currentIndexChanged( int ) ), this, SLOT( updateFieldsAndEnable() ) );
6571

6672
connect( delimiterCSV, SIGNAL( toggled( bool ) ), this, SLOT( updateFieldsAndEnable() ) );
6773
connect( delimiterChars, SIGNAL( toggled( bool ) ), this, SLOT( updateFieldsAndEnable() ) );
@@ -153,10 +159,8 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
153159
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
154160
{
155161
QString field = cmbXField->currentText();
156-
if ( ! useHeader ) field.remove( mColumnNamePrefix );
157162
url.addQueryItem( "xField", field );
158163
field = cmbYField->currentText();
159-
if ( ! useHeader ) field.remove( mColumnNamePrefix );
160164
url.addQueryItem( "yField", field );
161165
}
162166
}
@@ -165,7 +169,6 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
165169
if ( ! cmbWktField->currentText().isEmpty() )
166170
{
167171
QString field = cmbWktField->currentText();
168-
if ( ! useHeader ) field.remove( mColumnNamePrefix );
169172
url.addQueryItem( "wktField", field );
170173
}
171174
if ( cmbGeometryType->currentIndex() > 0 )
@@ -314,9 +317,9 @@ void QgsDelimitedTextSourceSelect::loadSettingsForFile( QString filename )
314317
{
315318
if ( filename.isEmpty() ) return;
316319
QFileInfo fi( filename );
317-
QString filetype=fi.suffix();
320+
QString filetype = fi.suffix();
318321
// Don't expect to change settings if not changing file type
319-
if( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
322+
if ( filetype != mLastFileType ) loadSettings( fi.suffix(), true );
320323
mLastFileType = filetype;
321324
}
322325

@@ -383,27 +386,13 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
383386
if ( ! loadDelimitedFileDefinition() )
384387
return;
385388

386-
bool useHeader = mFile->useHeader();
387-
QStringList fieldList;
388-
QList<bool> isValidNumber;
389-
QList<bool> isValidWkt;
390-
QList<bool> isEmpty;
389+
// Put a sample set of records into the sample box. Also while scanning assess suitability of
390+
// fields for use as coordinate and WKT fields
391391

392-
if ( useHeader )
393-
{
394-
fieldList = mFile->columnNames();
395-
tblSample->setColumnCount( fieldList.size() );
396-
tblSample->resizeColumnsToContents();
397-
for ( int i = 0; i < fieldList.size(); i++ )
398-
{
399-
isValidNumber.append( false );
400-
isValidWkt.append( false );
401-
isEmpty.append( true );
402-
}
403-
}
404-
405-
// put a lines into the sample box
406392

393+
QList<bool> isValidCoordinate;
394+
QList<bool> isValidWkt;
395+
QList<bool> isEmpty;
407396
int counter = 0;
408397
QStringList values;
409398
QRegExp wktre( "^\\s*(?:MULTI)?(?:POINT|LINESTRING|POLYGON)\\s*Z?\\s*M?\\(", Qt::CaseInsensitive );
@@ -415,24 +404,20 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
415404
if ( status != QgsDelimitedTextFile::RecordOk ) continue;
416405
counter++;
417406

418-
// If don't have headers, then check column count and expand if necessary
419-
// Don't count blank columns
407+
// Look at count of non-blank fields
420408

421409
int nv = values.size();
422410
while ( nv > 0 && values[nv-1].isEmpty() ) nv--;
423411

424-
if ( nv > fieldList.size() )
412+
if ( isEmpty.size() < nv )
425413
{
426-
while ( fieldList.size() < nv )
414+
while ( isEmpty.size() < nv )
427415
{
428-
int nc = fieldList.size();
429-
QString column = mColumnNamePrefix + QString::number( nc + 1 );
430-
fieldList.append( column );
431416
isEmpty.append( true );
432-
isValidNumber.append( false );
417+
isValidCoordinate.append( false );
433418
isValidWkt.append( false );
434419
}
435-
tblSample->setColumnCount( fieldList.size() );
420+
tblSample->setColumnCount( nv );
436421
}
437422

438423
tblSample->setRowCount( counter );
@@ -449,10 +434,10 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
449434
if ( isEmpty[i] )
450435
{
451436
isEmpty[i] = false;
452-
isValidNumber[i] = true;
437+
isValidCoordinate[i] = true;
453438
isValidWkt[i] = true;
454439
}
455-
if ( isValidNumber[i] )
440+
if ( isValidCoordinate[i] )
456441
{
457442
bool ok = true;
458443
if ( cbxPointIsComma->isChecked() )
@@ -467,7 +452,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
467452
{
468453
value.toDouble( &ok );
469454
}
470-
isValidNumber[i] = ok;
455+
isValidCoordinate[i] = ok;
471456
}
472457
if ( isValidWkt[i] )
473458
{
@@ -478,6 +463,19 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
478463
}
479464
}
480465

466+
QStringList fieldList = mFile->fieldNames();
467+
468+
if ( isEmpty.size() < fieldList.size() )
469+
{
470+
while ( isEmpty.size() < fieldList.size() )
471+
{
472+
isEmpty.append( true );
473+
isValidCoordinate.append( false );
474+
isValidWkt.append( false );
475+
}
476+
tblSample->setColumnCount( fieldList.size() );
477+
}
478+
481479
tblSample->setHorizontalHeaderLabels( fieldList );
482480
tblSample->resizeColumnsToContents();
483481
tblSample->resizeRowsToContents();
@@ -508,11 +506,11 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
508506
// Now try setting optional X,Y fields - will only reset the fields if
509507
// not already set.
510508

511-
trySetXYField( fieldList, isValidNumber, "longitude", "latitude" );
512-
trySetXYField( fieldList, isValidNumber, "lon", "lat" );
513-
trySetXYField( fieldList, isValidNumber, "east", "north" );
514-
trySetXYField( fieldList, isValidNumber, "x", "y" );
515-
trySetXYField( fieldList, isValidNumber, "e", "n" );
509+
trySetXYField( fieldList, isValidCoordinate, "longitude", "latitude" );
510+
trySetXYField( fieldList, isValidCoordinate, "lon", "lat" );
511+
trySetXYField( fieldList, isValidCoordinate, "east", "north" );
512+
trySetXYField( fieldList, isValidCoordinate, "x", "y" );
513+
trySetXYField( fieldList, isValidCoordinate, "e", "n" );
516514

517515
// And also a WKT field if there is one
518516

@@ -581,7 +579,7 @@ bool QgsDelimitedTextSourceSelect::trySetXYField( QStringList &fields, QList<boo
581579
if ( ! fields.contains( yfield, Qt::CaseInsensitive ) ) continue;
582580
for ( int iy = 0; iy < fields.size(); iy++ )
583581
{
584-
if ( ! isValidNumber[i] ) continue;
582+
if ( ! isValidNumber[iy] ) continue;
585583
if ( iy == i ) continue;
586584
if ( fields[iy].compare( yfield, Qt::CaseInsensitive ) == 0 )
587585
{

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
5050
private:
5151
QgsDelimitedTextFile *mFile;
5252
int mExampleRowCount;
53-
QString mColumnNamePrefix;
5453
QString mPluginKey;
5554
QString mLastFileType;
5655

‎src/ui/qgsdelimitedtextsourceselectbase.ui

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,9 @@
794794
<property name="whatsThis">
795795
<string>Select the file encoding</string>
796796
</property>
797+
<property name="insertPolicy">
798+
<enum>QComboBox::InsertAtTop</enum>
799+
</property>
797800
</widget>
798801
</item>
799802
</layout>

‎tests/src/python/test_qgsdelimitedtextprovider.py

Lines changed: 265 additions & 27 deletions
Large diffs are not rendered by default.

‎tests/testdata/delimitedtext/test.badquote

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
id,description,data,info
2+
1,Unclosed quotes 1,"Quoted,data1
3+
2,Unclosed quotes 2,"Quoted,data2",info2
4+
3,Recovered after unclosed quore,"Data ok",inf3
5+
4,Unclosed quotes to end of file,"Never ending field ...

‎tests/testdata/delimitedtext/testfields.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,"description",data,,data,28,24.5,field_3,data_1
2+
1,Generation of field names,Some data,Some info,,,,,,,,last data
3+

‎tests/testdata/delimitedtext/testlatin1.csv

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id|description|name
2+
1|Correctly read latin1 encoding|This test is �

‎tests/testdata/delimitedtext/testre.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
idREGEXPdescriptionREGEXPdataREGEXPinfo
2+
1REGEXP Basic regular expression test REGEXP data1 REGEXP info
3+
2REGEXP Basic regular expression test 2 RE data2 RE info2

‎tests/testdata/delimitedtext/testre2.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
id description information
2+
1 Anchored regexp Some data
3+
2 Anchored regexp invalid
4+
3 Anchored regexp recovered Some data

‎tests/testdata/delimitedtext/testutf8.csv

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id|description|name
2+
1|Correctly read UTF8 encoding|Field has āccèntéd text

0 commit comments

Comments
 (0)
Please sign in to comment.