Skip to content

Commit b501c9a

Browse files
ccrookccrook
committedApr 18, 2013
Error message fixes and more regexp testing
·
ltr-3_40archive/master_2
1 parent 8570646 commit b501c9a

File tree

8 files changed

+174
-50
lines changed

8 files changed

+174
-50
lines changed
 

‎resources/context_help/QgsDelimitedTextSourceSelect-en_US

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Loads and displays delimited text files
88
<a href="#wkt">How WKT text is interpreted</a><br />
99
<a href="#example">Example of a text file with X,Y point coordinates</a><br/>
1010
<a href="#wkt_example">Example of a text file with WKT geometries</a><br/>
11-
<a href="#notes">Notes</a><br/>
11+
<a href="#python">Using delimited text layers in Python</a><br/>
1212
</p>
1313

1414
<h4><a name="re">Overview</a></h4>
@@ -170,3 +170,5 @@ id|wkt<br />
170170
<li>Uses <b>|</b> as a delimiter.</li>
171171
<li>Specifies each point using the WKT notation
172172
</ul>
173+
174+
<h4><a name="python">Using delimited text layers in Python</a></h4>

‎src/providers/delimitedtext/qgsdelimitedtextfile.cpp

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,10 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
180180
{
181181
mTrimFields = ! url.queryItemValue( "trimFields" ).toUpper().startsWith( 'N' );;
182182
}
183+
if ( url.hasQueryItem( "maxFields" ) )
184+
{
185+
mMaxFields = url.queryItemValue( "maxFields" ).toInt();
186+
}
183187

184188
QgsDebugMsg( "Delimited text file is: " + mFileName );
185189
QgsDebugMsg( "Encoding is: " + mEncoding );
@@ -188,6 +192,7 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
188192
QgsDebugMsg( "Quote character is: [" + quote + "]" );
189193
QgsDebugMsg( "Escape character is: [" + escape + "]" );
190194
QgsDebugMsg( "Skip lines: " + QString::number( mSkipLines ) );
195+
QgsDebugMsg( "Maximum number of fields in record: " + QString::number( mMaxFields ) );
191196
QgsDebugMsg( "Use headers: " + QString( mUseHeader ? "Yes" : "No" ) );
192197
QgsDebugMsg( "Discard empty fields: " + QString( mDiscardEmptyFields ? "Yes" : "No" ) );
193198
QgsDebugMsg( "Trim fields: " + QString( mTrimFields ? "Yes" : "No" ) );
@@ -246,6 +251,10 @@ QUrl QgsDelimitedTextFile::url()
246251
{
247252
url.addQueryItem( "skipEmptyFields", "Yes" );
248253
}
254+
if ( mMaxFields > 0 )
255+
{
256+
url.addQueryItem( "maxFields", QString::number( mMaxFields ) );
257+
}
249258
return url;
250259
}
251260

@@ -341,6 +350,12 @@ void QgsDelimitedTextFile::setTrimFields( bool trimFields )
341350
mTrimFields = trimFields;
342351
}
343352

353+
void QgsDelimitedTextFile::setMaxFields( int maxFields )
354+
{
355+
resetDefinition();
356+
mMaxFields = maxFields;
357+
}
358+
344359
void QgsDelimitedTextFile::setDiscardEmptyFields( bool discardEmptyFields )
345360
{
346361
resetDefinition();

‎src/providers/delimitedtext/qgsdelimitedtextfile.h

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -142,64 +142,79 @@ class QgsDelimitedTextFile
142142
*/
143143
void setTypeCSV( QString delim = QString( "," ), QString quote = QString( "\"" ), QString escape = QString( "\"" ) );
144144

145-
/* Set the number of header lines to skip
145+
/** Set the number of header lines to skip
146146
* @param skiplines The maximum lines to skip
147147
*/
148148
void setSkipLines( int skiplines );
149-
/* Return the number of header lines to skip
149+
/** Return the number of header lines to skip
150150
* @return skiplines The maximum lines to skip
151151
*/
152152
int skipLines()
153153
{
154154
return mSkipLines;
155155
}
156156

157-
/* Set reading field names from the first record
157+
/** Set reading field names from the first record
158158
* @param useheaders Field names will be read if true
159159
*/
160160
void setUseHeader( bool useheader = true );
161-
/* Return the option for reading field names from the first record
161+
/** Return the option for reading field names from the first record
162162
* @return useheaders Field names will be read if true
163163
*/
164164
bool useHeader()
165165
{
166166
return mUseHeader;
167167
}
168168

169-
/* Set the option for dicarding empty fields
169+
/** Set the option for dicarding empty fields
170170
* @param useheaders Empty fields will be discarded if true
171171
*/
172172
void setDiscardEmptyFields( bool discardEmptyFields = true );
173-
/* Return the option for discarding empty fields
173+
/** Return the option for discarding empty fields
174174
* @return useheaders Empty fields will be discarded if true
175175
*/
176176
bool discardEmptyFields()
177177
{
178178
return mDiscardEmptyFields;
179179
}
180180

181-
/* Set the option for trimming whitespace from fields
181+
/** Set the option for trimming whitespace from fields
182182
* @param trimFields Fields will be trimmed if true
183183
*/
184184
void setTrimFields( bool trimFields = true );
185-
/* Return the option for trimming empty fields
185+
/** Return the option for trimming empty fields
186186
* @return useheaders Empty fields will be trimmed if true
187187
*/
188188
bool trimFields()
189189
{
190190
return mTrimFields;
191191
}
192192

193+
/** Set the maximum number of fields that will be read from a record
194+
* By default the maximum number is unlimited (0)
195+
* @param maxFields The maximum number of fields that will be read
196+
*/
197+
void setMaxFields( int maxFields );
198+
/** Return the maximum number of fields that will be read
199+
* @return maxFields The maximum number of fields that will be read
200+
*/
201+
int maxFields() { return mMaxFields; }
202+
193203
/** Set the field names
194204
* Field names are set from QStringList. Names may be modified
195205
* to ensure that they are unique, not empty, and do not conflict
196-
* with default field name (Field_##)
206+
* with default field name (field_##)
207+
* @param names A list of proposed field names
197208
*/
198209
void setFieldNames( const QStringList &names );
199210

200211
/** Return the field names read from the header, or default names
201-
* Col## if none defined. Will open and read the head of the file
202-
* if required, then reset..
212+
* field_## if none defined. Will open and read the head of the file
213+
* if required, then reset. Note that if header record record has
214+
* not been read then the field names are empty until records have
215+
* been read. The list may be expanded as the file is read and records
216+
* with more fields are loaded.
217+
* @return names A list of field names in the file
203218
*/
204219
QStringList &fieldNames();
205220

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
129129
if ( ! mFile->isValid() )
130130
{
131131
// uri is invalid so the layer must be too...
132+
QStringList messages;
133+
messages.append( "File cannot be opened or delimiter parameters are not valid" );
134+
reportErrors( messages );
132135
QgsDebugMsg( "Delimited text source invalid - filename or delimiter parameters" );
133136
return;
134137
}
@@ -539,20 +542,22 @@ void QgsDelimitedTextProvider::recordInvalidLine( QString message )
539542

540543
void QgsDelimitedTextProvider::reportErrors( QStringList messages )
541544
{
542-
if ( !mInvalidLines.isEmpty() )
545+
if ( !mInvalidLines.isEmpty() || ! messages.isEmpty() )
543546
{
544547
QString tag( "DelimitedText" );
545548
QgsMessageLog::logMessage( tr( "Errors in file %1" ).arg( mFile->fileName() ), tag );
546549
foreach ( QString message, messages )
547550
{
548551
QgsMessageLog::logMessage( message );
549552
}
550-
QgsMessageLog::logMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg( mFile->fileName() ),
551-
tag );
552-
for ( int i = 0; i < mInvalidLines.size(); ++i )
553-
QgsMessageLog::logMessage( mInvalidLines.at( i ), tag );
554-
if ( mNExtraInvalidLines > 0 )
555-
QgsMessageLog::logMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ), tag );
553+
if ( ! mInvalidLines.isEmpty() )
554+
{
555+
QgsMessageLog::logMessage( tr( "The following lines were not loaded into QGIS due to errors:" ), tag );
556+
for ( int i = 0; i < mInvalidLines.size(); ++i )
557+
QgsMessageLog::logMessage( mInvalidLines.at( i ), tag );
558+
if ( mNExtraInvalidLines > 0 )
559+
QgsMessageLog::logMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ), tag );
560+
}
556561

557562

558563
// Display errors in a dialog...
@@ -565,11 +570,14 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
565570
{
566571
output->appendMessage( message );
567572
}
568-
output->appendMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg( mFile->fileName() ) );
569-
for ( int i = 0; i < mInvalidLines.size(); ++i )
570-
output->appendMessage( mInvalidLines.at( i ) );
571-
if ( mNExtraInvalidLines > 0 )
572-
output->appendMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ) );
573+
if ( ! mInvalidLines.isEmpty() )
574+
{
575+
output->appendMessage( tr( "The following lines were not loaded into QGIS due to errors:" ) );
576+
for ( int i = 0; i < mInvalidLines.size(); ++i )
577+
output->appendMessage( mInvalidLines.at( i ) );
578+
if ( mNExtraInvalidLines > 0 )
579+
output->appendMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ) );
580+
}
573581
output->showMessage();
574582
}
575583

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
5656
codecs.append( codec );
5757
}
5858
codecs.sort();
59-
foreach( QString codec, codecs )
59+
foreach ( QString codec, codecs )
6060
{
6161
cmbEncoding->addItem( codec );
6262
}
@@ -143,8 +143,6 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
143143

144144
QUrl url = mFile->url();
145145

146-
bool useHeader = mFile->useHeader();
147-
148146
if ( cbxPointIsComma->isChecked() )
149147
{
150148
url.addQueryItem( "decimalPoint", "," );
@@ -637,19 +635,11 @@ void QgsDelimitedTextSourceSelect::updateFileName()
637635

638636
void QgsDelimitedTextSourceSelect::updateFieldsAndEnable()
639637
{
640-
// Check the regular expression is valid
641-
lblRegexpError->setText( "" );
642-
if ( delimiterRegexp->isChecked() )
643-
{
644-
QRegExp re( txtDelimiterRegexp->text() );
645-
if ( ! re.isValid() ) lblRegexpError->setText( tr( "Expression is not valid" ) );
646-
}
647-
648638
updateFieldLists();
649639
enableAccept();
650640
}
651641

652-
void QgsDelimitedTextSourceSelect::enableAccept()
642+
bool QgsDelimitedTextSourceSelect::validate()
653643
{
654644
// Check that input data is valid - provide a status message if not..
655645

@@ -672,9 +662,23 @@ void QgsDelimitedTextSourceSelect::enableAccept()
672662
{
673663
message = tr( "At least one delimiter character must be specified" );
674664
}
675-
else if ( delimiterRegexp->isChecked() && ! QRegExp( txtDelimiterRegexp->text() ).isValid() )
665+
666+
if ( message.isEmpty() && delimiterRegexp->isChecked() )
667+
{
668+
QRegExp re( txtDelimiterRegexp->text() );
669+
if ( ! re.isValid() )
670+
{
671+
message = tr( "Regular expression is not valid" );
672+
}
673+
else if ( re.pattern().startsWith( "^" ) && re.captureCount() == 0 )
674+
{
675+
message = tr( "^.. expression needs capture groups" );
676+
}
677+
lblRegexpError->setText( message );
678+
}
679+
if ( ! message.isEmpty() )
676680
{
677-
message = tr( "Regular expression is not valid" );
681+
// continue...
678682
}
679683
// Hopefully won't hit this none-specific message, but just in case ...
680684
else if ( ! mFile->isValid() )
@@ -702,8 +706,12 @@ void QgsDelimitedTextSourceSelect::enableAccept()
702706
{
703707
enabled = true;
704708
}
705-
706709
lblStatus->setText( message );
710+
return enabled;
711+
}
707712

713+
void QgsDelimitedTextSourceSelect::enableAccept()
714+
{
715+
bool enabled = validate();
708716
buttonBox->button( QDialogButtonBox::Ok )->setEnabled( enabled );
709717
}

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
6666
void updateFileName();
6767
void updateFieldsAndEnable();
6868
void enableAccept();
69+
bool validate();
6970

7071
signals:
7172
void addVectorLayer( QString, QString, QString );

‎tests/src/python/test_qgsdelimitedtextprovider.py

Lines changed: 84 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,31 @@ def test_002a_field_naming(self):
362362
runTest(description,wanted,log_wanted,filename,**params)
363363

364364

365+
def test_002b_max_fields(self):
366+
description='Limiting maximum number of fields'
367+
filename='testfields.csv'
368+
params={'geomType': 'none', 'maxFields': '7', 'type': 'csv'}
369+
if printTests:
370+
createTest(description,filename,**params)
371+
assert False,"Set printTests to False to run delimited text tests"
372+
wanted={
373+
u'1': {
374+
'id': u'1',
375+
'description': u'Generation of field names',
376+
'data': u'Some data',
377+
'field_4': u'Some info',
378+
'data_1': u'',
379+
'field_6': u'',
380+
'field_7': u'',
381+
'#fid': 2L,
382+
'#geometry': 'None',
383+
},
384+
}
385+
log_wanted=[
386+
]
387+
runTest(description,wanted,log_wanted,filename,**params)
388+
389+
365390
def test_003_load_whitespace(self):
366391
description='Whitespace file parsing'
367392
filename='test.space'
@@ -589,7 +614,7 @@ def test_005_multiple_quote(self):
589614
}
590615
log_wanted=[
591616
u'Errors in file file',
592-
u'The following lines were not loaded from file into QGIS due to errors:\n',
617+
u'The following lines were not loaded into QGIS due to errors:',
593618
u'Invalid record format at line 7',
594619
u'Invalid record format at line 8',
595620
u'Invalid record format at line 9',
@@ -616,7 +641,7 @@ def test_005a_badly_formed_quotes(self):
616641
}
617642
log_wanted=[
618643
u'Errors in file file',
619-
u'The following lines were not loaded from file into QGIS due to errors:\n',
644+
u'The following lines were not loaded into QGIS due to errors:',
620645
u'Invalid record format at line 2',
621646
u'Invalid record format at line 5',
622647
]
@@ -681,7 +706,7 @@ def test_008_read_coordinates(self):
681706
}
682707
log_wanted=[
683708
u'Errors in file file',
684-
u'The following lines were not loaded from file into QGIS due to errors:\n',
709+
u'The following lines were not loaded into QGIS due to errors:',
685710
u'Invalid X or Y fields at line 4',
686711
]
687712
runTest(description,wanted,log_wanted,filename,**params)
@@ -728,7 +753,7 @@ def test_009_read_wkt(self):
728753
}
729754
log_wanted=[
730755
u'Errors in file file',
731-
u'The following lines were not loaded from file into QGIS due to errors:\n',
756+
u'The following lines were not loaded into QGIS due to errors:',
732757
u'Invalid WKT at line 8',
733758
]
734759
runTest(description,wanted,log_wanted,filename,**params)
@@ -775,7 +800,7 @@ def test_010_read_wkt_point(self):
775800
}
776801
log_wanted=[
777802
u'Errors in file file',
778-
u'The following lines were not loaded from file into QGIS due to errors:\n',
803+
u'The following lines were not loaded into QGIS due to errors:',
779804
u'Invalid WKT at line 8',
780805
]
781806
runTest(description,wanted,log_wanted,filename,**params)
@@ -822,7 +847,7 @@ def test_011_read_wkt_line(self):
822847
}
823848
log_wanted=[
824849
u'Errors in file file',
825-
u'The following lines were not loaded from file into QGIS due to errors:\n',
850+
u'The following lines were not loaded into QGIS due to errors:',
826851
u'Invalid WKT at line 8',
827852
]
828853
runTest(description,wanted,log_wanted,filename,**params)
@@ -851,7 +876,7 @@ def test_012_read_wkt_polygon(self):
851876
}
852877
log_wanted=[
853878
u'Errors in file file',
854-
u'The following lines were not loaded from file into QGIS due to errors:\n',
879+
u'The following lines were not loaded into QGIS due to errors:',
855880
u'Invalid WKT at line 8',
856881
]
857882
runTest(description,wanted,log_wanted,filename,**params)
@@ -1020,7 +1045,7 @@ def test_013_read_dms_xy(self):
10201045
}
10211046
log_wanted=[
10221047
u'Errors in file file',
1023-
u'The following lines were not loaded from file into QGIS due to errors:\n',
1048+
u'The following lines were not loaded into QGIS due to errors:',
10241049
u'Invalid X or Y fields at line 27',
10251050
u'Invalid X or Y fields at line 28',
10261051
u'Invalid X or Y fields at line 29',
@@ -1161,12 +1186,62 @@ def test_017_regular_expression_3(self):
11611186
}
11621187
log_wanted=[
11631188
u'Errors in file file',
1164-
u'The following lines were not loaded from file into QGIS due to errors:\n',
1189+
u'The following lines were not loaded into QGIS due to errors:',
11651190
u'Invalid record format at line 3',
11661191
]
11671192
runTest(description,wanted,log_wanted,filename,**params)
11681193

11691194

1195+
def test_017a_regular_expression_4(self):
1196+
description='Parsing zero length re'
1197+
filename='testre3.txt'
1198+
params={'geomType': 'none', 'delimiter': 'x?', 'type': 'regexp'}
1199+
if printTests:
1200+
createTest(description,filename,**params)
1201+
assert False,"Set printTests to False to run delimited text tests"
1202+
wanted={
1203+
u'f': {
1204+
'id': u'f',
1205+
'description': u'i',
1206+
's': u'f',
1207+
'm': u'i',
1208+
'a': u'.',
1209+
'l': u'.',
1210+
'l_1': u'i',
1211+
'field_6': u'l',
1212+
'field_7': u'e',
1213+
'#fid': 2L,
1214+
'#geometry': 'None',
1215+
},
1216+
}
1217+
log_wanted=[
1218+
]
1219+
runTest(description,wanted,log_wanted,filename,**params)
1220+
1221+
1222+
def test_017a_regular_expression_5(self):
1223+
description='Parsing zero length re 2'
1224+
filename='testre3.txt'
1225+
params={'geomType': 'none', 'delimiter': '\\b', 'type': 'regexp'}
1226+
if printTests:
1227+
createTest(description,filename,**params)
1228+
assert False,"Set printTests to False to run delimited text tests"
1229+
wanted={
1230+
u'fi': {
1231+
'id': u'fi',
1232+
'description': u'..',
1233+
'small': u'fi',
1234+
'field_2': u'..',
1235+
'field_3': u'ile',
1236+
'#fid': 2L,
1237+
'#geometry': 'None',
1238+
},
1239+
}
1240+
log_wanted=[
1241+
]
1242+
runTest(description,wanted,log_wanted,filename,**params)
1243+
1244+
11701245
def test_018_utf8_encoded_file(self):
11711246
description='UTF8 encoded file test'
11721247
filename='testutf8.csv'
@@ -1208,8 +1283,6 @@ def test_019_latin1_encoded_file(self):
12081283
]
12091284
runTest(description,wanted,log_wanted,filename,**params)
12101285

1211-
1212-
12131286
#END
12141287

12151288
if __name__ == '__main__':

‎tests/testdata/delimitedtext/testre3.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
small
2+
fi..ile

0 commit comments

Comments
 (0)
Please sign in to comment.