Skip to content

Commit 7ed9308

Browse files
wonder-skwonder-sk
committedOct 20, 2012
Enabled delimitedtext provider again
·
ltr-3_40archive/master_2
1 parent 999df64 commit 7ed9308

File tree

6 files changed

+291
-221
lines changed

6 files changed

+291
-221
lines changed
 

‎src/providers/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ SET (CMAKE_LIBRARY_OUTPUT_DIRECTORY ${QGIS_OUTPUT_DIRECTORY}/${QGIS_PLUGIN_SUBDI
55
ADD_SUBDIRECTORY(memory)
66
ADD_SUBDIRECTORY(ogr)
77
ADD_SUBDIRECTORY(wms)
8-
#ADD_SUBDIRECTORY(delimitedtext) # TODO: enable when migrated to new api
8+
ADD_SUBDIRECTORY(delimitedtext)
99
#ADD_SUBDIRECTORY(osm) # TODO: enable when migrated to new api
1010
#ADD_SUBDIRECTORY(sqlanywhere) # TODO: enable when migrated to new api
1111
ADD_SUBDIRECTORY(gdal)

‎src/providers/delimitedtext/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Files
44

55
SET (DTEXT_SRCS
6+
qgsdelimitedtextfeatureiterator.cpp
67
qgsdelimitedtextprovider.cpp
78
qgsdelimitedtextsourceselect.cpp
89
)

‎src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
#include "qgsdelimitedtextfeatureiterator.h"
2+
3+
#include "qgsdelimitedtextprovider.h"
4+
5+
#include "qgsgeometry.h"
6+
7+
#include <QTextStream>
8+
9+
QgsDelimitedTextFeatureIterator::QgsDelimitedTextFeatureIterator( QgsDelimitedTextProvider* p, const QgsFeatureRequest& request )
10+
: QgsAbstractFeatureIterator( request ), P( p )
11+
{
12+
rewind();
13+
}
14+
15+
QgsDelimitedTextFeatureIterator::~QgsDelimitedTextFeatureIterator()
16+
{
17+
close();
18+
}
19+
20+
bool QgsDelimitedTextFeatureIterator::nextFeature( QgsFeature& feature )
21+
{
22+
// before we do anything else, assume that there's something wrong with
23+
// the feature
24+
feature.setValid( false );
25+
26+
if ( mClosed )
27+
return false;
28+
29+
while ( !P->mStream->atEnd() )
30+
{
31+
QString line = P->readLine( P->mStream ); // Default local 8 bit encoding
32+
if ( line.isEmpty() )
33+
continue;
34+
35+
// lex the tokens from the current data line
36+
QStringList tokens = P->splitLine( line );
37+
38+
while ( tokens.size() < P->mFieldCount )
39+
tokens.append( QString::null );
40+
41+
QgsGeometry *geom = 0;
42+
43+
if ( P->mWktFieldIndex >= 0 )
44+
{
45+
geom = loadGeometryWkt( tokens );
46+
}
47+
else if ( P->mXFieldIndex >= 0 && P->mYFieldIndex >= 0 )
48+
{
49+
geom = loadGeometryXY( tokens );
50+
}
51+
52+
if ( !geom && P->mWkbType != QGis::WKBNoGeometry )
53+
{
54+
P->mInvalidLines << line;
55+
continue;
56+
}
57+
58+
mFid++;
59+
60+
// At this point the current feature values are valid
61+
62+
feature.setValid( true );
63+
feature.setFields( &P->attributeFields ); // allow name-based attribute lookups
64+
feature.setFeatureId( mFid );
65+
feature.initAttributes( P->attributeFields.count() );
66+
67+
if ( geom )
68+
feature.setGeometry( geom );
69+
70+
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
71+
{
72+
const QgsAttributeList& attrs = mRequest.subsetOfAttributes();
73+
for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
74+
{
75+
int fieldIdx = *i;
76+
if ( fieldIdx < 0 || fieldIdx >= P->attributeColumns.count() )
77+
continue; // ignore non-existant fields
78+
fetchAttribute( feature, fieldIdx, tokens );
79+
}
80+
}
81+
else
82+
{
83+
for ( int idx = 0; idx < P->attributeFields.count(); ++idx )
84+
fetchAttribute( feature, idx, tokens );
85+
}
86+
87+
// We have a good line, so return
88+
return true;
89+
90+
} // !mStream->atEnd()
91+
92+
// End of the file. If there are any lines that couldn't be
93+
// loaded, display them now.
94+
P->handleInvalidLines();
95+
96+
return false;
97+
}
98+
99+
bool QgsDelimitedTextFeatureIterator::rewind()
100+
{
101+
if ( mClosed )
102+
return false;
103+
104+
// Reset feature id to 0
105+
mFid = 0;
106+
// Skip to first data record
107+
P->mStream->seek( 0 );
108+
int n = P->mFirstDataLine - 1;
109+
while ( n-- > 0 )
110+
P->readLine( P->mStream );
111+
112+
return true;
113+
}
114+
115+
bool QgsDelimitedTextFeatureIterator::close()
116+
{
117+
if ( mClosed )
118+
return false;
119+
120+
mClosed = true;
121+
return true;
122+
}
123+
124+
125+
QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryWkt(const QStringList& tokens)
126+
{
127+
QgsGeometry* geom = 0;
128+
try
129+
{
130+
QString sWkt = tokens[P->mWktFieldIndex];
131+
// Remove Z and M coordinates if present, as currently fromWkt doesn't
132+
// support these.
133+
if ( P->mWktHasZM )
134+
{
135+
sWkt.remove( P->mWktZMRegexp ).replace( P->mWktCrdRegexp, "\\1" );
136+
}
137+
138+
geom = QgsGeometry::fromWkt( sWkt );
139+
}
140+
catch ( ... )
141+
{
142+
geom = 0;
143+
}
144+
145+
if ( geom && geom->wkbType() != P->mWkbType )
146+
{
147+
delete geom;
148+
geom = 0;
149+
}
150+
if ( geom && !boundsCheck( geom ) )
151+
{
152+
delete geom;
153+
geom = 0;
154+
}
155+
return geom;
156+
}
157+
158+
159+
QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryXY( const QStringList& tokens )
160+
{
161+
QString sX = tokens[P->mXFieldIndex];
162+
QString sY = tokens[P->mYFieldIndex];
163+
164+
if ( !P->mDecimalPoint.isEmpty() )
165+
{
166+
sX.replace( P->mDecimalPoint, "." );
167+
sY.replace( P->mDecimalPoint, "." );
168+
}
169+
170+
bool xOk, yOk;
171+
double x = sX.toDouble( &xOk );
172+
double y = sY.toDouble( &yOk );
173+
if ( xOk && yOk )
174+
{
175+
if ( boundsCheck( x, y ) )
176+
{
177+
return QgsGeometry::fromPoint( QgsPoint( x, y ) );
178+
}
179+
}
180+
return 0;
181+
}
182+
183+
184+
185+
/**
186+
* Check to see if the point is within the selection rectangle
187+
*/
188+
bool QgsDelimitedTextFeatureIterator::boundsCheck( double x, double y )
189+
{
190+
// no selection rectangle or geometry => always in the bounds
191+
if ( mRequest.filterType() != QgsFeatureRequest::FilterRect || (mRequest.flags() & QgsFeatureRequest::NoGeometry) )
192+
return true;
193+
194+
return mRequest.filterRect().contains( QgsPoint( x, y ) );
195+
}
196+
197+
/**
198+
* Check to see if the geometry is within the selection rectangle
199+
*/
200+
bool QgsDelimitedTextFeatureIterator::boundsCheck( QgsGeometry *geom )
201+
{
202+
// no selection rectangle or geometry => always in the bounds
203+
if ( mRequest.filterType() != QgsFeatureRequest::FilterRect || (mRequest.flags() & QgsFeatureRequest::NoGeometry) )
204+
return true;
205+
206+
if ( mRequest.flags() & QgsFeatureRequest::ExactIntersect )
207+
return geom->intersects( mRequest.filterRect() );
208+
else
209+
return geom->boundingBox().intersects( mRequest.filterRect() );
210+
}
211+
212+
213+
void QgsDelimitedTextFeatureIterator::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
214+
{
215+
const QString &value = tokens[P->attributeColumns[fieldIdx]];
216+
QVariant val;
217+
switch ( P->attributeFields[fieldIdx].type() )
218+
{
219+
case QVariant::Int:
220+
if ( !value.isEmpty() )
221+
val = QVariant( value );
222+
else
223+
val = QVariant( P->attributeFields[fieldIdx].type() );
224+
break;
225+
case QVariant::Double:
226+
if ( !value.isEmpty() )
227+
val = QVariant( value.toDouble() );
228+
else
229+
val = QVariant( P->attributeFields[fieldIdx].type() );
230+
break;
231+
default:
232+
val = QVariant( value );
233+
break;
234+
}
235+
feature.setAttribute( fieldIdx, val );
236+
}

‎src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#ifndef QGSDELIMITEDTEXTFEATUREITERATOR_H
2+
#define QGSDELIMITEDTEXTFEATUREITERATOR_H
3+
4+
#include "qgsfeatureiterator.h"
5+
6+
class QgsDelimitedTextProvider;
7+
8+
class QgsDelimitedTextFeatureIterator : public QgsAbstractFeatureIterator
9+
{
10+
public:
11+
QgsDelimitedTextFeatureIterator( QgsDelimitedTextProvider* p, const QgsFeatureRequest& request );
12+
13+
~QgsDelimitedTextFeatureIterator();
14+
15+
//! fetch next feature, return true on success
16+
virtual bool nextFeature( QgsFeature& feature );
17+
18+
//! reset the iterator to the starting position
19+
virtual bool rewind();
20+
21+
//! end of iterating: free the resources / lock
22+
virtual bool close();
23+
24+
protected:
25+
QgsDelimitedTextProvider* P;
26+
27+
//! Feature id
28+
long mFid;
29+
30+
QgsGeometry* loadGeometryWkt( const QStringList& tokens );
31+
QgsGeometry* loadGeometryXY( const QStringList& tokens );
32+
33+
bool boundsCheck( double x, double y );
34+
bool boundsCheck( QgsGeometry *geom );
35+
36+
void fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens );
37+
};
38+
39+
40+
#endif // QGSDELIMITEDTEXTFEATUREITERATOR_H

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 6 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
#include "qgis.h"
3939

4040
#include "qgsdelimitedtextsourceselect.h"
41+
#include "qgsdelimitedtextfeatureiterator.h"
4142

4243
static const QString TEXT_PROVIDER_KEY = "delimitedtext";
4344
static const QString TEXT_PROVIDER_DESCRIPTION = "Delimited text data provider";
@@ -187,8 +188,6 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
187188
if ( mDelimiterType != "regexp" )
188189
mDelimiter.replace( "\\t", "\t" ); // replace "\t" with a real tabulator
189190

190-
// Set the selection rectangle to null
191-
mSelectionRectangle = QgsRectangle();
192191
// assume the layer is invalid until proven otherwise
193192
mValid = false;
194193
if ( mFileName.isEmpty() || mDelimiter.isEmpty() )
@@ -463,134 +462,14 @@ QString QgsDelimitedTextProvider::storageType() const
463462
}
464463

465464

466-
bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature )
465+
QgsFeatureIterator QgsDelimitedTextProvider::getFeatures( const QgsFeatureRequest& request )
467466
{
468-
// before we do anything else, assume that there's something wrong with
469-
// the feature
470-
feature.setValid( false );
471-
while ( !mStream->atEnd() )
472-
{
473-
QString line = readLine( mStream ); // Default local 8 bit encoding
474-
if ( line.isEmpty() )
475-
continue;
476-
477-
// lex the tokens from the current data line
478-
QStringList tokens = splitLine( line );
479-
480-
while ( tokens.size() < mFieldCount )
481-
tokens.append( QString::null );
482-
483-
QgsGeometry *geom = 0;
484-
485-
if ( mWktFieldIndex >= 0 )
486-
{
487-
try
488-
{
489-
QString &sWkt = tokens[mWktFieldIndex];
490-
// Remove Z and M coordinates if present, as currently fromWkt doesn't
491-
// support these.
492-
if ( mWktHasZM )
493-
{
494-
sWkt.remove( mWktZMRegexp ).replace( mWktCrdRegexp, "\\1" );
495-
}
496-
497-
geom = QgsGeometry::fromWkt( sWkt );
498-
}
499-
catch ( ... )
500-
{
501-
geom = 0;
502-
}
503-
504-
if ( geom && geom->wkbType() != mWkbType )
505-
{
506-
delete geom;
507-
geom = 0;
508-
}
509-
mFid++;
510-
if ( geom && !boundsCheck( geom ) )
511-
{
512-
delete geom;
513-
geom = 0;
514-
}
515-
}
516-
else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
517-
{
518-
QString sX = tokens[mXFieldIndex];
519-
QString sY = tokens[mYFieldIndex];
520-
521-
if ( !mDecimalPoint.isEmpty() )
522-
{
523-
sX.replace( mDecimalPoint, "." );
524-
sY.replace( mDecimalPoint, "." );
525-
}
526-
527-
bool xOk, yOk;
528-
double x = sX.toDouble( &xOk );
529-
double y = sY.toDouble( &yOk );
530-
if ( xOk && yOk )
531-
{
532-
mFid++;
533-
if ( boundsCheck( x, y ) )
534-
{
535-
geom = QgsGeometry::fromPoint( QgsPoint( x, y ) );
536-
}
537-
}
538-
}
539-
540-
if ( !geom && mWkbType != QGis::WKBNoGeometry )
541-
{
542-
mInvalidLines << line;
543-
continue;
544-
}
545-
546-
// At this point the current feature values are valid
547-
548-
feature.setValid( true );
549-
feature.setFields( &attributeFields ); // allow name-based attribute lookups
550-
feature.setFeatureId( mFid );
551-
feature.initAttributes( mFieldCount );
552-
553-
if ( geom )
554-
feature.setGeometry( geom );
555-
556-
for ( QgsAttributeList::const_iterator i = mAttributesToFetch.begin();
557-
i != mAttributesToFetch.end();
558-
++i )
559-
{
560-
int fieldIdx = *i;
561-
if ( fieldIdx < 0 || fieldIdx >= attributeColumns.count() )
562-
continue; // ignore non-existant fields
563-
564-
QString &value = tokens[attributeColumns[fieldIdx]];
565-
QVariant val;
566-
switch ( attributeFields[fieldIdx].type() )
567-
{
568-
case QVariant::Int:
569-
if ( !value.isEmpty() )
570-
val = QVariant( value );
571-
else
572-
val = QVariant( attributeFields[fieldIdx].type() );
573-
break;
574-
case QVariant::Double:
575-
if ( !value.isEmpty() )
576-
val = QVariant( value.toDouble() );
577-
else
578-
val = QVariant( attributeFields[fieldIdx].type() );
579-
break;
580-
default:
581-
val = QVariant( value );
582-
break;
583-
}
584-
feature.setAttribute( fieldIdx, val );
585-
}
586-
587-
// We have a good line, so return
588-
return true;
467+
return QgsFeatureIterator( new QgsDelimitedTextFeatureIterator( this, request ) );
468+
}
589469

590-
} // !mStream->atEnd()
591470

592-
// End of the file. If there are any lines that couldn't be
593-
// loaded, display them now.
471+
void QgsDelimitedTextProvider::handleInvalidLines()
472+
{
594473
if ( mShowInvalidLines && !mInvalidLines.isEmpty() )
595474
{
596475
mShowInvalidLines = false;
@@ -610,35 +489,10 @@ bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature )
610489
// We no longer need these lines.
611490
mInvalidLines.clear();
612491
}
613-
614-
return false;
615-
616-
} // nextFeature
617-
618-
619-
void QgsDelimitedTextProvider::select( QgsAttributeList fetchAttributes,
620-
QgsRectangle rect,
621-
bool fetchGeometry,
622-
bool useIntersect )
623-
{
624-
mSelectionRectangle = rect;
625-
mAttributesToFetch = fetchAttributes;
626-
mFetchGeom = fetchGeometry;
627-
mUseIntersect = useIntersect;
628-
if ( rect.isEmpty() )
629-
{
630-
mSelectionRectangle = mExtent;
631-
}
632-
else
633-
{
634-
mSelectionRectangle = rect;
635-
}
636-
rewind();
637492
}
638493

639494

640495

641-
642496
// Return the extent of the layer
643497
QgsRectangle QgsDelimitedTextProvider::extent()
644498
{
@@ -667,45 +521,11 @@ const QgsFields & QgsDelimitedTextProvider::fields() const
667521
return attributeFields;
668522
}
669523

670-
void QgsDelimitedTextProvider::rewind()
671-
{
672-
// Reset feature id to 0
673-
mFid = 0;
674-
// Skip to first data record
675-
mStream->seek( 0 );
676-
int n = mFirstDataLine - 1;
677-
while ( n-- > 0 )
678-
readLine( mStream );
679-
}
680-
681524
bool QgsDelimitedTextProvider::isValid()
682525
{
683526
return mValid;
684527
}
685528

686-
/**
687-
* Check to see if the point is within the selection rectangle
688-
*/
689-
bool QgsDelimitedTextProvider::boundsCheck( double x, double y )
690-
{
691-
// no selection rectangle or geometry => always in the bounds
692-
if ( mSelectionRectangle.isEmpty() || !mFetchGeom )
693-
return true;
694-
695-
return mSelectionRectangle.contains( QgsPoint( x, y ) );
696-
}
697-
/**
698-
* Check to see if the geometry is within the selection rectangle
699-
*/
700-
bool QgsDelimitedTextProvider::boundsCheck( QgsGeometry *geom )
701-
{
702-
// no selection rectangle or geometry => always in the bounds
703-
if ( mSelectionRectangle.isEmpty() || !mFetchGeom )
704-
return true;
705-
706-
return geom->boundingBox().intersects( mSelectionRectangle ) &&
707-
( !mUseIntersect || geom->intersects( mSelectionRectangle ) );
708-
}
709529

710530
int QgsDelimitedTextProvider::capabilities() const
711531
{

‎src/providers/delimitedtext/qgsdelimitedtextprovider.h

Lines changed: 7 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ class QgsField;
2626
class QFile;
2727
class QTextStream;
2828

29+
class QgsDelimitedTextFeatureIterator;
30+
2931

3032
/**
3133
\class QgsDelimitedTextProvider
@@ -57,27 +59,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
5759
*/
5860
virtual QString storageType() const;
5961

60-
/** Select features based on a bounding rectangle. Features can be retrieved with calls to nextFeature.
61-
* @param fetchAttributes list of attributes which should be fetched
62-
* @param rect spatial filter
63-
* @param fetchGeometry true if the feature geometry should be fetched
64-
* @param useIntersect true if an accurate intersection test should be used,
65-
* false if a test based on bounding box is sufficient
66-
*/
67-
virtual void select( QgsAttributeList fetchAttributes = QgsAttributeList(),
68-
QgsRectangle rect = QgsRectangle(),
69-
bool fetchGeometry = true,
70-
bool useIntersect = false );
71-
72-
/**
73-
* Get the next feature resulting from a select operation.
74-
* @param feature feature which will receive data from the provider
75-
* @return true when there was a feature to fetch, false when end was hit
76-
*
77-
* mFile should be open with the file pointer at the record of the next
78-
* feature, or EOF. The feature found on the current line is parsed.
79-
*/
80-
virtual bool nextFeature( QgsFeature& feature );
62+
virtual QgsFeatureIterator getFeatures( const QgsFeatureRequest& request );
8163

8264
/**
8365
* Get feature type.
@@ -97,9 +79,6 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
9779
*/
9880
virtual const QgsFields & fields() const;
9981

100-
/** Restart reading features from previous select operation */
101-
virtual void rewind();
102-
10382
/** Returns a bitmask containing the supported capabilities
10483
Note, some capabilities may change depending on whether
10584
a spatial filter is active on this provider, so it may
@@ -176,12 +155,12 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
176155

177156
private:
178157

158+
void handleInvalidLines();
159+
179160
//! Fields
180161
QList<int> attributeColumns;
181162
QgsFields attributeFields;
182163

183-
QgsAttributeList mAttributesToFetch;
184-
185164
QString mFileName;
186165
QString mDelimiter;
187166
QRegExp mDelimiterRegexp;
@@ -204,17 +183,12 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
204183
//! Layer extent
205184
QgsRectangle mExtent;
206185

207-
//! Current selection rectangle
208-
209-
QgsRectangle mSelectionRectangle;
210-
211186
//! Text file
212187
QFile *mFile;
213188

214189
QTextStream *mStream;
215190

216191
bool mValid;
217-
bool mUseIntersect;
218192

219193
int mGeomType;
220194

@@ -228,9 +202,6 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
228202
//! Only want to show the invalid lines once to the user
229203
bool mShowInvalidLines;
230204

231-
//! Feature id
232-
long mFid;
233-
234205
struct wkbPoint
235206
{
236207
unsigned char byteOrder;
@@ -246,4 +217,6 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
246217
QGis::WkbType mWkbType;
247218

248219
QStringList splitLine( QString line ) { return splitLine( line, mDelimiterType, mDelimiter ); }
220+
221+
friend class QgsDelimitedTextFeatureIterator;
249222
};

0 commit comments

Comments
 (0)
Please sign in to comment.