Skip to content

Commit 54f13a6

Browse files
nyalldawsonnyalldawson
committedNov 18, 2015
Fix delimited text layers set to unknown geometry type if first row has
null geometry (fix #13749) (cherry-picked from 7439643)
·
final-2_12_3final-2_12_1
1 parent 0e0f473 commit 54f13a6

File tree

4 files changed

+112
-5
lines changed

4 files changed

+112
-5
lines changed
 

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
409409
QList<bool> couldBeInt;
410410
QList<bool> couldBeLongLong;
411411
QList<bool> couldBeDouble;
412+
bool foundFirstGeometry = false;
412413

413414
while ( true )
414415
{
@@ -458,11 +459,12 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
458459
if ( mGeometryType == QGis::UnknownGeometry || geom->type() == mGeometryType )
459460
{
460461
mGeometryType = geom->type();
461-
if ( mNumberFeatures == 0 )
462+
if ( !foundFirstGeometry )
462463
{
463464
mNumberFeatures++;
464465
mWkbType = type;
465466
mExtent = geom->boundingBox();
467+
foundFirstGeometry = true;
466468
}
467469
else
468470
{
@@ -517,7 +519,7 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
517519

518520
if ( ok )
519521
{
520-
if ( mNumberFeatures > 0 )
522+
if ( foundFirstGeometry )
521523
{
522524
mExtent.combineExtentWith( pt.x(), pt.y() );
523525
}
@@ -527,6 +529,7 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
527529
mExtent.set( pt.x(), pt.y(), pt.x(), pt.y() );
528530
mWkbType = QGis::WKBPoint;
529531
mGeometryType = QGis::Point;
532+
foundFirstGeometry = true;
530533
}
531534
mNumberFeatures++;
532535
if ( buildSpatialIndex && qIsFinite( pt.x() ) && qIsFinite( pt.y() ) )
@@ -778,13 +781,15 @@ void QgsDelimitedTextProvider::rescanFile()
778781
mNumberFeatures = 0;
779782
mExtent = QgsRectangle();
780783
QgsFeature f;
784+
bool foundFirstGeometry = false;
781785
while ( fi.nextFeature( f ) )
782786
{
783787
if ( mGeometryType != QGis::NoGeometry )
784788
{
785-
if ( mNumberFeatures == 0 )
789+
if ( !foundFirstGeometry )
786790
{
787791
mExtent = f.constGeometry()->boundingBox();
792+
foundFirstGeometry = true;
788793
}
789794
else
790795
{

‎tests/src/python/test_qgsdelimitedtextprovider.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
QgsVectorLayer,
4444
QgsFeatureRequest,
4545
QgsRectangle,
46-
QgsMessageLog
46+
QgsMessageLog,
47+
QGis
4748
)
4849

4950
from utilities import (getQgisTestApp,
@@ -190,7 +191,7 @@ def delimitedTextData(testname, filename, requests, verbose, **params):
190191
msg = re.sub(r'file\s+.*' + re.escape(filename), 'file ' + filelogname, msg)
191192
msg = msg.replace(filepath, filelogname)
192193
log.append(msg)
193-
return dict(fields=fields, fieldTypes=fieldTypes, data=data, log=log, uri=uri)
194+
return dict(fields=fields, fieldTypes=fieldTypes, data=data, log=log, uri=uri, geometryType=layer.geometryType())
194195

195196

196197
def printWanted(testname, result):
@@ -206,6 +207,7 @@ def printWanted(testname, result):
206207
print prefix + "wanted={}"
207208
print prefix + "wanted['uri']=" + repr(result['uri'])
208209
print prefix + "wanted['fieldTypes']=" + repr(result['fieldTypes'])
210+
print prefix + "wanted['geometryType']=" + repr(result['geometryType'])
209211
print prefix + "wanted['data']={"
210212
for k in sorted(data.keys()):
211213
row = data[k]
@@ -272,6 +274,10 @@ def runTest(file, requests, **params):
272274
msg = "Layer field types ({0}) doesn't match expected ({1})".format(
273275
result['fieldTypes'], wanted['fieldTypes'])
274276
failures.append(msg)
277+
if result['geometryType'] != wanted['geometryType']:
278+
msg = "Layer geometry type ({0}) doesn't match expected ({1})".format(
279+
result['geometryType'], wanted['geometryType'])
280+
failures.append(msg)
275281
wanted_data = wanted['data']
276282
for id in sorted(wanted_data.keys()):
277283
wrec = wanted_data[id]
@@ -728,5 +734,13 @@ def test_038_type_inference(self):
728734
requests = None
729735
runTest(filename, requests, **params)
730736

737+
def test_039_issue_13749(self):
738+
# First record contains missing geometry
739+
filename = 'test13749.csv'
740+
params = {'yField': 'geom_y', 'xField': 'geom_x', 'type': 'csv'}
741+
requests = None
742+
runTest(filename, requests, **params)
743+
744+
731745
if __name__ == '__main__':
732746
unittest.main()

‎tests/src/python/test_qgsdelimitedtextprovider_wanted.py

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ def test_002_load_csv_file():
33
wanted = {}
44
wanted['uri'] = u'file://test.csv?geomType=none&type=csv'
55
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text']
6+
wanted['geometryType'] = 4
67
wanted['data'] = {
78
2: {
89
'id': u'1',
@@ -67,6 +68,7 @@ def test_003_field_naming():
6768
wanted = {}
6869
wanted['uri'] = u'file://testfields.csv?geomType=none&type=csv'
6970
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
71+
wanted['geometryType'] = 4
7072
wanted['data'] = {
7173
2: {
7274
'id': u'1',
@@ -93,6 +95,7 @@ def test_004_max_fields():
9395
wanted = {}
9496
wanted['uri'] = u'file://testfields.csv?geomType=none&maxFields=7&type=csv'
9597
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text', 'text', 'text']
98+
wanted['geometryType'] = 4
9699
wanted['data'] = {
97100
2: {
98101
'id': u'1',
@@ -114,6 +117,7 @@ def test_005_load_whitespace():
114117
wanted = {}
115118
wanted['uri'] = u'file://test.space?geomType=none&type=whitespace'
116119
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text', 'text']
120+
wanted['geometryType'] = 4
117121
wanted['data'] = {
118122
2: {
119123
'id': u'1',
@@ -184,6 +188,7 @@ def test_006_quote_escape():
184188
wanted = {}
185189
wanted['uri'] = u'file://test.pipe?geomType=none&quote="&delimiter=|&escape=\\'
186190
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text', 'text']
191+
wanted['geometryType'] = 4
187192
wanted['data'] = {
188193
2: {
189194
'id': u'1',
@@ -274,6 +279,7 @@ def test_007_multiple_quote():
274279
wanted = {}
275280
wanted['uri'] = u'file://test.quote?geomType=none&quote=\'"&type=csv&escape="\''
276281
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text']
282+
wanted['geometryType'] = 4
277283
wanted['data'] = {
278284
2: {
279285
'id': u'1',
@@ -339,6 +345,7 @@ def test_008_badly_formed_quotes():
339345
wanted = {}
340346
wanted['uri'] = u'file://test.badquote?geomType=none&quote="&type=csv&escape="'
341347
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text']
348+
wanted['geometryType'] = 4
342349
wanted['data'] = {
343350
4: {
344351
'id': u'3',
@@ -363,6 +370,7 @@ def test_009_skip_lines():
363370
wanted = {}
364371
wanted['uri'] = u'file://test2.csv?geomType=none&skipLines=2&type=csv&useHeader=no'
365372
wanted['fieldTypes'] = ['integer', 'text', 'text']
373+
wanted['geometryType'] = 4
366374
wanted['data'] = {
367375
3: {
368376
'id': u'3',
@@ -382,6 +390,7 @@ def test_010_read_coordinates():
382390
wanted = {}
383391
wanted['uri'] = u'file://testpt.csv?yField=geom_y&xField=geom_x&type=csv'
384392
wanted['fieldTypes'] = ['integer', 'text', 'double', 'double']
393+
wanted['geometryType'] = 0
385394
wanted['data'] = {
386395
2: {
387396
'id': u'1',
@@ -421,6 +430,7 @@ def test_011_read_wkt():
421430
wanted = {}
422431
wanted['uri'] = u'file://testwkt.csv?delimiter=|&type=csv&wktField=geom_wkt'
423432
wanted['fieldTypes'] = ['integer', 'text']
433+
wanted['geometryType'] = 0
424434
wanted['data'] = {
425435
2: {
426436
'id': u'1',
@@ -467,6 +477,7 @@ def test_012_read_wkt_point():
467477
wanted = {}
468478
wanted['uri'] = u'file://testwkt.csv?geomType=point&delimiter=|&type=csv&wktField=geom_wkt'
469479
wanted['fieldTypes'] = ['integer', 'text']
480+
wanted['geometryType'] = 0
470481
wanted['data'] = {
471482
2: {
472483
'id': u'1',
@@ -513,6 +524,7 @@ def test_013_read_wkt_line():
513524
wanted = {}
514525
wanted['uri'] = u'file://testwkt.csv?geomType=line&delimiter=|&type=csv&wktField=geom_wkt'
515526
wanted['fieldTypes'] = ['integer', 'text']
527+
wanted['geometryType'] = 1
516528
wanted['data'] = {
517529
4: {
518530
'id': u'3',
@@ -559,6 +571,7 @@ def test_014_read_wkt_polygon():
559571
wanted = {}
560572
wanted['uri'] = u'file://testwkt.csv?geomType=polygon&delimiter=|&type=csv&wktField=geom_wkt'
561573
wanted['fieldTypes'] = ['integer', 'text']
574+
wanted['geometryType'] = 2
562575
wanted['data'] = {
563576
6: {
564577
'id': u'5',
@@ -587,6 +600,7 @@ def test_015_read_dms_xy():
587600
wanted = {}
588601
wanted['uri'] = u'file://testdms.csv?yField=lat&xField=lon&type=csv&xyDms=yes'
589602
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text']
603+
wanted['geometryType'] = 0
590604
wanted['data'] = {
591605
3: {
592606
'id': u'1',
@@ -758,6 +772,7 @@ def test_016_decimal_point():
758772
wanted = {}
759773
wanted['uri'] = u'file://testdp.csv?yField=geom_y&xField=geom_x&type=csv&delimiter=;&decimalPoint=,'
760774
wanted['fieldTypes'] = ['integer', 'text', 'double', 'double', 'double', 'text']
775+
wanted['geometryType'] = 0
761776
wanted['data'] = {
762777
2: {
763778
'id': u'1',
@@ -788,6 +803,7 @@ def test_017_regular_expression_1():
788803
wanted = {}
789804
wanted['uri'] = u'file://testre.txt?geomType=none&trimFields=Y&delimiter=RE(?:GEXP)?&type=regexp'
790805
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text']
806+
wanted['geometryType'] = 4
791807
wanted['data'] = {
792808
2: {
793809
'id': u'1',
@@ -814,6 +830,7 @@ def test_018_regular_expression_2():
814830
wanted = {}
815831
wanted['uri'] = u'file://testre.txt?geomType=none&trimFields=Y&delimiter=(RE)(GEXP)?&type=regexp'
816832
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
833+
wanted['geometryType'] = 4
817834
wanted['data'] = {
818835
2: {
819836
'id': u'1',
@@ -852,6 +869,7 @@ def test_019_regular_expression_3():
852869
wanted = {}
853870
wanted['uri'] = u'file://testre2.txt?geomType=none&trimFields=Y&delimiter=^(.{5})(.{30})(.{5,})&type=regexp'
854871
wanted['fieldTypes'] = ['integer', 'text', 'text']
872+
wanted['geometryType'] = 4
855873
wanted['data'] = {
856874
2: {
857875
'id': u'1',
@@ -881,6 +899,7 @@ def test_020_regular_expression_4():
881899
wanted = {}
882900
wanted['uri'] = u'file://testre3.txt?geomType=none&delimiter=x?&type=regexp'
883901
wanted['fieldTypes'] = ['text', 'text', 'text', 'text', 'text', 'text', 'text']
902+
wanted['geometryType'] = 4
884903
wanted['data'] = {
885904
2: {
886905
'id': u'f',
@@ -904,6 +923,7 @@ def test_021_regular_expression_5():
904923
wanted = {}
905924
wanted['uri'] = u'file://testre3.txt?geomType=none&delimiter=\\b&type=regexp'
906925
wanted['fieldTypes'] = ['text', 'text', 'text']
926+
wanted['geometryType'] = 4
907927
wanted['data'] = {
908928
2: {
909929
'id': u'fi',
@@ -923,6 +943,7 @@ def test_022_utf8_encoded_file():
923943
wanted = {}
924944
wanted['uri'] = u'file://testutf8.csv?geomType=none&delimiter=|&type=csv&encoding=utf-8'
925945
wanted['fieldTypes'] = ['integer', 'text', 'text']
946+
wanted['geometryType'] = 4
926947
wanted['data'] = {
927948
2: {
928949
'id': u'1',
@@ -940,6 +961,7 @@ def test_023_latin1_encoded_file():
940961
wanted = {}
941962
wanted['uri'] = u'file://testlatin1.csv?geomType=none&delimiter=|&type=csv&encoding=latin1'
942963
wanted['fieldTypes'] = ['integer', 'text', 'text']
964+
wanted['geometryType'] = 4
943965
wanted['data'] = {
944966
2: {
945967
'id': u'1',
@@ -957,6 +979,7 @@ def test_024_filter_rect_xy():
957979
wanted = {}
958980
wanted['uri'] = u'file://testextpt.txt?yField=y&delimiter=|&type=csv&xField=x'
959981
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'integer']
982+
wanted['geometryType'] = 0
960983
wanted['data'] = {
961984
2: {
962985
'id': u'1',
@@ -1001,6 +1024,7 @@ def test_025_filter_rect_wkt():
10011024
wanted = {}
10021025
wanted['uri'] = u'file://testextw.txt?delimiter=|&type=csv&wktField=wkt'
10031026
wanted['fieldTypes'] = ['integer', 'text']
1027+
wanted['geometryType'] = 1
10041028
wanted['data'] = {
10051029
2: {
10061030
'id': u'1',
@@ -1061,6 +1085,7 @@ def test_026_filter_fid():
10611085
wanted = {}
10621086
wanted['uri'] = u'file://test.csv?geomType=none&type=csv'
10631087
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text']
1088+
wanted['geometryType'] = 4
10641089
wanted['data'] = {
10651090
3: {
10661091
'id': u'2',
@@ -1100,6 +1125,7 @@ def test_027_filter_attributes():
11001125
wanted = {}
11011126
wanted['uri'] = u'file://test.csv?geomType=none&type=csv'
11021127
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text']
1128+
wanted['geometryType'] = 4
11031129
wanted['data'] = {
11041130
2: {
11051131
'id': u'None',
@@ -1209,6 +1235,7 @@ def test_028_substring_test():
12091235
wanted = {}
12101236
wanted['uri'] = u'file://test.csv?geomType=none&type=csv&subset=id%20%25%202%20%3D%201'
12111237
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text']
1238+
wanted['geometryType'] = 4
12121239
wanted['data'] = {
12131240
2: {
12141241
'id': u'1',
@@ -1246,6 +1273,7 @@ def test_029_file_watcher():
12461273
wanted = {}
12471274
wanted['uri'] = u'file://file?geomType=none&type=csv&watchFile=yes'
12481275
wanted['fieldTypes'] = ['integer', 'text']
1276+
wanted['geometryType'] = 4
12491277
wanted['data'] = {
12501278
3: {
12511279
'id': u'2',
@@ -1359,6 +1387,7 @@ def test_030_filter_rect_xy_spatial_index():
13591387
wanted = {}
13601388
wanted['uri'] = u'file://testextpt.txt?spatialIndex=Y&yField=y&delimiter=|&type=csv&xField=x'
13611389
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'integer']
1390+
wanted['geometryType'] = 0
13621391
wanted['data'] = {
13631392
2: {
13641393
'id': u'1',
@@ -1547,6 +1576,7 @@ def test_031_filter_rect_wkt_spatial_index():
15471576
wanted = {}
15481577
wanted['uri'] = u'file://testextw.txt?spatialIndex=Y&delimiter=|&type=csv&wktField=wkt'
15491578
wanted['fieldTypes'] = ['integer', 'text']
1579+
wanted['geometryType'] = 1
15501580
wanted['data'] = {
15511581
2: {
15521582
'id': u'1',
@@ -1679,6 +1709,7 @@ def test_032_filter_rect_wkt_create_spatial_index():
16791709
wanted = {}
16801710
wanted['uri'] = u'file://testextw.txt?delimiter=|&type=csv&wktField=wkt'
16811711
wanted['fieldTypes'] = ['integer', 'text']
1712+
wanted['geometryType'] = 1
16821713
wanted['data'] = {
16831714
2: {
16841715
'id': u'1',
@@ -1877,6 +1908,7 @@ def test_033_reset_subset_string():
18771908
wanted = {}
18781909
wanted['uri'] = u'file://test.csv?geomType=none&type=csv'
18791910
wanted['fieldTypes'] = ['integer', 'text', 'text', 'text', 'text']
1911+
wanted['geometryType'] = 4
18801912
wanted['data'] = {
18811913
2: {
18821914
'id': u'1',
@@ -2040,6 +2072,7 @@ def test_034_csvt_file():
20402072
wanted = {}
20412073
wanted['uri'] = u'file://testcsvt.csv?geomType=none&type=csv'
20422074
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'double', 'text', 'text', 'text', 'text', 'text', 'text', 'longlong', 'longlong']
2075+
wanted['geometryType'] = 4
20432076
wanted['data'] = {
20442077
2: {
20452078
'id': u'1',
@@ -2082,6 +2115,7 @@ def test_035_csvt_file2():
20822115
wanted = {}
20832116
wanted['uri'] = u'file://testcsvt2.txt?geomType=none&delimiter=|&type=csv'
20842117
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'double', 'integer', 'text', 'integer']
2118+
wanted['geometryType'] = 4
20852119
wanted['data'] = {
20862120
2: {
20872121
'id': u'1',
@@ -2114,6 +2148,7 @@ def test_036_csvt_file_invalid_types():
21142148
wanted = {}
21152149
wanted['uri'] = u'file://testcsvt3.csv?geomType=none&type=csv'
21162150
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'double', 'integer', 'text', 'text']
2151+
wanted['geometryType'] = 4
21172152
wanted['data'] = {
21182153
2: {
21192154
'id': u'1',
@@ -2149,6 +2184,7 @@ def test_037_csvt_file_invalid_file():
21492184
wanted = {}
21502185
wanted['uri'] = u'file://testcsvt4.csv?geomType=none&type=csv'
21512186
wanted['fieldTypes'] = ['integer', 'text', 'integer', 'double', 'integer', 'text', 'text']
2187+
wanted['geometryType'] = 4
21522188
wanted['data'] = {
21532189
2: {
21542190
'id': u'1',
@@ -2181,6 +2217,7 @@ def test_038_type_inference():
21812217
wanted = {}
21822218
wanted['uri'] = u'file://testtypes.csv?yField=lat&xField=lon&type=csv'
21832219
wanted['fieldTypes'] = ['text', 'double', 'double', 'text', 'text', 'integer', 'longlong', 'double', 'text']
2220+
wanted['geometryType'] = 0
21842221
wanted['data'] = {
21852222
2: {
21862223
'id': u'line1',
@@ -2269,3 +2306,49 @@ def test_038_type_inference():
22692306
}
22702307
wanted['log'] = []
22712308
return wanted
2309+
2310+
2311+
def test_039_issue_13749():
2312+
wanted = {}
2313+
wanted['uri'] = u'file://test13749.csv?yField=geom_y&xField=geom_x&type=csv'
2314+
wanted['fieldTypes'] = ['integer', 'text', 'double', 'double']
2315+
wanted['geometryType'] = 0
2316+
wanted['data'] = {
2317+
2: {
2318+
'id': u'1',
2319+
'description': u'No geom',
2320+
'geom_x': u'NULL',
2321+
'geom_y': u'NULL',
2322+
'#fid': 2,
2323+
'#geometry': 'None',
2324+
},
2325+
3: {
2326+
'id': u'2',
2327+
'description': u'Point1',
2328+
'geom_x': u'11.0',
2329+
'geom_y': u'22.0',
2330+
'#fid': 3,
2331+
'#geometry': 'Point (11 22)',
2332+
},
2333+
4: {
2334+
'id': u'3',
2335+
'description': u'Point2',
2336+
'geom_x': u'15.0',
2337+
'geom_y': u'23.0',
2338+
'#fid': 4,
2339+
'#geometry': 'Point (15 23)',
2340+
},
2341+
5: {
2342+
'id': u'4',
2343+
'description': u'Point3',
2344+
'geom_x': u'13.0',
2345+
'geom_y': u'23.0',
2346+
'#fid': 5,
2347+
'#geometry': 'Point (13 23)',
2348+
},
2349+
}
2350+
wanted['log'] = [
2351+
u'Errors in file test13749.csv',
2352+
u'1 records have missing geometry definitions',
2353+
]
2354+
return wanted

‎tests/testdata/delimitedtext/test13749.csv

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
id,description,geom_x,geom_y
2+
1,No geom,,
3+
2,Point1,11,22
4+
3,Point2,15,23
5+
4,Point3,13.0,23.0

0 commit comments

Comments
 (0)
Please sign in to comment.