Skip to content

Commit b4f2069

Browse files
ccrooknyalldawson
ccrook
authored and
nyalldawson
committedJul 19, 2018
Delimited text detectTypes flag - fixes #18601
[FEATURE] Adds a detectTypes flag to the delimited text provider url. If set to "no" then type detection is not done and all attributes are treated as text fields. Otherwise the original behaviour of detecting field types is preserved. [needs-docs] Adds a "Detect field types" check box to the record and field options section of the delimited text provider GUI. This addresses (at least partially) issue #18601. A more complete solution would be to allow users to set field types.
·
ltr-3_40final-3_4_0
1 parent 8f5d968 commit b4f2069

File tree

6 files changed

+209
-87
lines changed

6 files changed

+209
-87
lines changed
 

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
120120
}
121121
}
122122

123+
mDetectTypes=true;
124+
if ( url.hasQueryItem( QStringLiteral( "detectTypes" ) ) )
125+
mDetectTypes = ! url.queryItemValue( QStringLiteral( "detectTypes" ) ).toLower().startsWith( 'n' );
126+
123127
if ( url.hasQueryItem( QStringLiteral( "decimalPoint" ) ) )
124128
mDecimalPoint = url.queryItemValue( QStringLiteral( "decimalPoint" ) );
125129

@@ -225,7 +229,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
225229
QgsDebugMsg( QString( "Field type string: %1" ).arg( strTypeList ) );
226230

227231
int pos = 0;
228-
QRegExp reType( "(integer|real|string|date|datetime|time)" );
232+
QRegExp reType( "(integer|real|double|string|date|datetime|time)" );
229233

230234
while ( ( pos = reType.indexIn( strTypeList, pos ) ) != -1 )
231235
{
@@ -240,10 +244,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
240244
// *message=tr("Reading field types from %1").arg(csvtInfo.fileName());
241245
}
242246

243-
244247
return types;
245-
246-
247248
}
248249

249250
void QgsDelimitedTextProvider::resetCachedSubset() const
@@ -556,6 +557,11 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
556557
couldBeDouble[i] = true;
557558
}
558559

560+
if( ! mDetectTypes )
561+
{
562+
continue;
563+
}
564+
559565
// Now test for still valid possible types for the field
560566
// Types are possible until first record which cannot be parsed
561567

@@ -603,40 +609,40 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
603609
QString typeName = QStringLiteral( "text" );
604610
if ( i < csvtTypes.size() )
605611
{
606-
if ( csvtTypes[i] == QLatin1String( "integer" ) )
607-
{
608-
fieldType = QVariant::Int;
609-
typeName = QStringLiteral( "integer" );
610-
}
611-
else if ( csvtTypes[i] == QLatin1String( "long" ) || csvtTypes[i] == QLatin1String( "longlong" ) || csvtTypes[i] == QLatin1String( "int8" ) )
612-
{
613-
fieldType = QVariant::LongLong; //QVariant doesn't support long
614-
typeName = QStringLiteral( "longlong" );
615-
}
616-
else if ( csvtTypes[i] == QLatin1String( "real" ) || csvtTypes[i] == QLatin1String( "double" ) )
617-
{
618-
fieldType = QVariant::Double;
619-
typeName = QStringLiteral( "double" );
620-
}
612+
typeName = csvtTypes[i];
621613
}
622-
else if ( i < couldBeInt.size() )
614+
else if ( mDetectTypes && i < couldBeInt.size() )
623615
{
624616
if ( couldBeInt[i] )
625617
{
626-
fieldType = QVariant::Int;
627618
typeName = QStringLiteral( "integer" );
628619
}
629620
else if ( couldBeLongLong[i] )
630621
{
631-
fieldType = QVariant::LongLong;
632622
typeName = QStringLiteral( "longlong" );
633623
}
634624
else if ( couldBeDouble[i] )
635625
{
636-
fieldType = QVariant::Double;
637626
typeName = QStringLiteral( "double" );
638627
}
639628
}
629+
if( typeName == QStringLiteral( "integer" ) )
630+
{
631+
fieldType = QVariant::Int;
632+
}
633+
else if ( typeName == QStringLiteral( "longlong" ) )
634+
{
635+
fieldType = QVariant::LongLong;
636+
}
637+
else if( typeName == QStringLiteral( "real" ) || typeName == QStringLiteral( "double" ) )
638+
{
639+
typeName = QStringLiteral( "double" );
640+
fieldType = QVariant::Double;
641+
}
642+
else
643+
{
644+
typeName = QStringLiteral( "text" );
645+
}
640646
attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
641647
}
642648

‎src/providers/delimitedtext/qgsdelimitedtextprovider.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
227227
QString mWktFieldName;
228228
QString mXFieldName;
229229
QString mYFieldName;
230+
bool mDetectTypes;
230231

231232
mutable int mXFieldIndex = -1;
232233
mutable int mYFieldIndex = -1;

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()
145145

146146
QUrl url = mFile->url();
147147

148+
url.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
149+
148150
if ( cbxPointIsComma->isChecked() )
149151
{
150152
url.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
@@ -192,9 +194,9 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()
192194

193195
}
194196

195-
if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? "yes" : "no" );
196-
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? "yes" : "no" );
197-
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? "yes" : "no" );
197+
if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
198+
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
199+
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
198200

199201
// store the settings
200202
saveSettings();
@@ -284,6 +286,7 @@ void QgsDelimitedTextSourceSelect::loadSettings( const QString &subkey, bool loa
284286

285287
rowCounter->setValue( settings.value( key + "/startFrom", 0 ).toInt() );
286288
cbxUseHeader->setChecked( settings.value( key + "/useHeader", "true" ) != "false" );
289+
cbxDetectTypes->setChecked( settings.value( key + "/detectTypes", "true" ) != "false" );
287290
cbxTrimFields->setChecked( settings.value( key + "/trimFields", "false" ) == "true" );
288291
cbxSkipEmptyFields->setChecked( settings.value( key + "/skipEmptyFields", "false" ) == "true" );
289292
cbxPointIsComma->setChecked( settings.value( key + "/decimalPoint", "." ).toString().contains( ',' ) );
@@ -329,6 +332,7 @@ void QgsDelimitedTextSourceSelect::saveSettings( const QString &subkey, bool sav
329332
settings.setValue( key + "/delimiterRegexp", txtDelimiterRegexp->text() );
330333
settings.setValue( key + "/startFrom", rowCounter->value() );
331334
settings.setValue( key + "/useHeader", cbxUseHeader->isChecked() ? "true" : "false" );
335+
settings.setValue( key + "/detectTypes", cbxDetectTypes->isChecked() ? "true" : "false" );
332336
settings.setValue( key + "/trimFields", cbxTrimFields->isChecked() ? "true" : "false" );
333337
settings.setValue( key + "/skipEmptyFields", cbxSkipEmptyFields->isChecked() ? "true" : "false" );
334338
settings.setValue( key + "/decimalPoint", cbxPointIsComma->isChecked() ? "," : "." );

‎src/ui/qgsdelimitedtextsourceselectbase.ui

Lines changed: 17 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -45,16 +45,7 @@
4545
<bool>true</bool>
4646
</property>
4747
<layout class="QVBoxLayout" name="verticalLayout">
48-
<property name="leftMargin">
49-
<number>0</number>
50-
</property>
51-
<property name="topMargin">
52-
<number>0</number>
53-
</property>
54-
<property name="rightMargin">
55-
<number>0</number>
56-
</property>
57-
<property name="bottomMargin">
48+
<property name="margin">
5849
<number>0</number>
5950
</property>
6051
<item>
@@ -295,16 +286,7 @@
295286
</widget>
296287
<widget class="QWidget" name="swpDelimOptions">
297288
<layout class="QVBoxLayout" name="verticalLayout_11">
298-
<property name="leftMargin">
299-
<number>0</number>
300-
</property>
301-
<property name="topMargin">
302-
<number>0</number>
303-
</property>
304-
<property name="rightMargin">
305-
<number>0</number>
306-
</property>
307-
<property name="bottomMargin">
289+
<property name="margin">
308290
<number>0</number>
309291
</property>
310292
<item>
@@ -325,16 +307,7 @@
325307
<property name="spacing">
326308
<number>0</number>
327309
</property>
328-
<property name="leftMargin">
329-
<number>2</number>
330-
</property>
331-
<property name="topMargin">
332-
<number>2</number>
333-
</property>
334-
<property name="rightMargin">
335-
<number>2</number>
336-
</property>
337-
<property name="bottomMargin">
310+
<property name="margin">
338311
<number>2</number>
339312
</property>
340313
<item>
@@ -593,16 +566,7 @@
593566
</widget>
594567
<widget class="QWidget" name="swpRegexpOptions">
595568
<layout class="QVBoxLayout" name="verticalLayout_12">
596-
<property name="leftMargin">
597-
<number>0</number>
598-
</property>
599-
<property name="topMargin">
600-
<number>0</number>
601-
</property>
602-
<property name="rightMargin">
603-
<number>0</number>
604-
</property>
605-
<property name="bottomMargin">
569+
<property name="margin">
606570
<number>0</number>
607571
</property>
608572
<item>
@@ -811,6 +775,16 @@
811775
</property>
812776
</widget>
813777
</item>
778+
<item row="2" column="0">
779+
<widget class="QCheckBox" name="cbxDetectTypes">
780+
<property name="text">
781+
<string>Detect field types</string>
782+
</property>
783+
<property name="checked">
784+
<bool>true</bool>
785+
</property>
786+
</widget>
787+
</item>
814788
</layout>
815789
</widget>
816790
</item>
@@ -900,16 +874,7 @@
900874
</property>
901875
<widget class="QWidget" name="swpGeomXY">
902876
<layout class="QVBoxLayout" name="verticalLayout_8">
903-
<property name="leftMargin">
904-
<number>0</number>
905-
</property>
906-
<property name="topMargin">
907-
<number>0</number>
908-
</property>
909-
<property name="rightMargin">
910-
<number>0</number>
911-
</property>
912-
<property name="bottomMargin">
877+
<property name="margin">
913878
<number>0</number>
914879
</property>
915880
<item>
@@ -1033,16 +998,7 @@
1033998
</widget>
1034999
<widget class="QWidget" name="swpGeomWKT">
10351000
<layout class="QVBoxLayout" name="verticalLayout_6">
1036-
<property name="leftMargin">
1037-
<number>0</number>
1038-
</property>
1039-
<property name="topMargin">
1040-
<number>0</number>
1041-
</property>
1042-
<property name="rightMargin">
1043-
<number>0</number>
1044-
</property>
1045-
<property name="bottomMargin">
1001+
<property name="margin">
10461002
<number>0</number>
10471003
</property>
10481004
<item>
@@ -1332,6 +1288,7 @@
13321288
<tabstop>recordOptionsGroupBox</tabstop>
13331289
<tabstop>rowCounter</tabstop>
13341290
<tabstop>cbxUseHeader</tabstop>
1291+
<tabstop>cbxDetectTypes</tabstop>
13351292
<tabstop>cbxPointIsComma</tabstop>
13361293
<tabstop>cbxTrimFields</tabstop>
13371294
<tabstop>cbxSkipEmptyFields</tabstop>

‎tests/src/python/test_qgsdelimitedtextprovider.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ def printWanted(self, testname, result):
337337
print((prefix + ' ' + repr(msg) + ','))
338338
print((prefix + ' ]'))
339339
print(' return wanted')
340-
print()
340+
print('',flush=True)
341341

342342
def recordDifference(self, record1, record2):
343343
# Compare a record defined as a dictionary
@@ -804,6 +804,21 @@ def test_040_issue_14666(self):
804804
params = {'yField': 'y', 'xField': 'x', 'type': 'csv', 'delimiter': '\\t'}
805805
requests = None
806806
self.runTest(filename, requests, **params)
807+
808+
def test_041_no_detect_type(self):
809+
# CSV file parsing
810+
# Skip lines
811+
filename = 'testtypes.csv'
812+
params = {'yField': 'lat', 'xField': 'lon', 'type': 'csv', 'detectTypes': 'no'}
813+
requests = None
814+
self.runTest(filename, requests, **params)
815+
816+
def test_042_no_detect_types_csvt(self):
817+
# CSVT field types
818+
filename = 'testcsvt.csv'
819+
params = {'geomType': 'none', 'type': 'csv', 'detectTypes': 'no' }
820+
requests = None
821+
self.runTest(filename, requests, **params)
807822

808823

809824
if __name__ == '__main__':

‎tests/src/python/test_qgsdelimitedtextprovider_wanted.py

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2449,3 +2449,142 @@ def test_040_issue_14666():
24492449
'2 records have missing geometry definitions',
24502450
]
24512451
return wanted
2452+
2453+
def test_041_no_detect_type():
2454+
wanted={}
2455+
wanted['uri']='file://testtypes.csv?yField=lat&xField=lon&type=csv&detectTypes=no'
2456+
wanted['fieldTypes']=['text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
2457+
wanted['geometryType']=0
2458+
wanted['data']={
2459+
2: {
2460+
'id': 'line1',
2461+
'description': '1.0',
2462+
'lon': '1.0',
2463+
'lat': '1.0',
2464+
'empty': 'NULL',
2465+
'text': 'NULL',
2466+
'int': '0',
2467+
'longlong': '0',
2468+
'real': 'NULL',
2469+
'text2': '1',
2470+
'#fid': 2,
2471+
'#geometry': 'Point (1 1)',
2472+
},
2473+
3: {
2474+
'id': 'line2',
2475+
'description': '1.0',
2476+
'lon': '1.0',
2477+
'lat': '5.0',
2478+
'empty': 'NULL',
2479+
'text': '1',
2480+
'int': 'NULL',
2481+
'longlong': '9189304972279762602',
2482+
'real': '1.3',
2483+
'text2': '-4',
2484+
'#fid': 3,
2485+
'#geometry': 'Point (1 5)',
2486+
},
2487+
4: {
2488+
'id': 'line3',
2489+
'description': '5.0',
2490+
'lon': '5.0',
2491+
'lat': '5.0',
2492+
'empty': 'NULL',
2493+
'text': '1xx',
2494+
'int': '2',
2495+
'longlong': '345',
2496+
'real': '2',
2497+
'text2': '1x',
2498+
'#fid': 4,
2499+
'#geometry': 'Point (5 5)',
2500+
},
2501+
5: {
2502+
'id': 'line4',
2503+
'description': '5.0',
2504+
'lon': '5.0',
2505+
'lat': '1.0',
2506+
'empty': 'NULL',
2507+
'text': 'A string',
2508+
'int': '-3456',
2509+
'longlong': '-3123724580211819352',
2510+
'real': '-123.56',
2511+
'text2': 'NULL',
2512+
'#fid': 5,
2513+
'#geometry': 'Point (5 1)',
2514+
},
2515+
6: {
2516+
'id': 'line5',
2517+
'description': '3.0',
2518+
'lon': '3.0',
2519+
'lat': '1.0',
2520+
'empty': 'NULL',
2521+
'text': 'NULL',
2522+
'int': 'NULL',
2523+
'longlong': 'NULL',
2524+
'real': '23e-5',
2525+
'text2': '23',
2526+
'#fid': 6,
2527+
'#geometry': 'Point (3 1)',
2528+
},
2529+
7: {
2530+
'id': 'line6',
2531+
'description': '1.0',
2532+
'lon': '1.0',
2533+
'lat': '3.0',
2534+
'empty': 'NULL',
2535+
'text': '1.5',
2536+
'int': '9',
2537+
'longlong': '42',
2538+
'real': '99',
2539+
'text2': '0',
2540+
'#fid': 7,
2541+
'#geometry': 'Point (1 3)',
2542+
},
2543+
}
2544+
wanted['log']=[
2545+
]
2546+
return wanted
2547+
2548+
def test_042_no_detect_types_csvt():
2549+
wanted={}
2550+
wanted['uri']='file://testcsvt.csv?geomType=none&type=csv&detectTypes=no'
2551+
wanted['fieldTypes']=['integer', 'text', 'integer', 'double', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
2552+
wanted['geometryType']=4
2553+
wanted['data']={
2554+
2: {
2555+
'id': '1',
2556+
'description': 'Test csvt 1',
2557+
'fint': '1',
2558+
'freal': '1.2',
2559+
'fstr': '1',
2560+
'fstr_1': 'text',
2561+
'fdatetime': '2015-03-02T12:30:00',
2562+
'fdate': '2014-12-30',
2563+
'ftime': '23:55',
2564+
'flong': '-456',
2565+
'flonglong': '-678',
2566+
'field_12': 'NULL',
2567+
'#fid': 2,
2568+
'#geometry': 'None',
2569+
},
2570+
3: {
2571+
'id': '2',
2572+
'description': 'Test csvt 2',
2573+
'fint': '3',
2574+
'freal': '1.5',
2575+
'fstr': '99',
2576+
'fstr_1': '23.5',
2577+
'fdatetime': '80',
2578+
'fdate': '2015-03-28',
2579+
'ftime': '2014-12-30',
2580+
'flong': '01:55',
2581+
'flonglong': '9189304972279762602',
2582+
'field_12': '-3123724580211819352',
2583+
'#fid': 3,
2584+
'#geometry': 'None',
2585+
},
2586+
}
2587+
wanted['log']=[
2588+
]
2589+
return wanted
2590+

0 commit comments

Comments
 (0)
Please sign in to comment.