Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #569 from ccrook/delimited_text_substring_capability
Delimited text substring capability - also bug #7742
  • Loading branch information
timlinux committed May 1, 2013
2 parents 2e446a8 + 2fb50ab commit 5e9e98a
Show file tree
Hide file tree
Showing 12 changed files with 1,068 additions and 441 deletions.
2 changes: 2 additions & 0 deletions resources/context_help/QgsDelimitedTextSourceSelect-en_US
Expand Up @@ -270,6 +270,8 @@ The following options can be added
<li><tt>xField=fieldname</tt> specifies the name or number (starting at 1) of the field the X coordinate (only applies if wktField is not defined)</li>
<li><tt>yField=fieldname</tt> specifies the name or number (starting at 1) of the field the Y coordinate (only applies if wktField is not defined)</li>
<li><tt>geomType=(auto|point|line|polygon|none)</tt> specifies type of geometry for wkt fields, or none to load the file as an attribute-only table. The default is auto.</li>
<li><tt>subset=expression</tt> specifies an expression used to identify a subset of the records that will be
used.</li>
<li><tt>crs=...</tt> specifies the coordinate system to use for the vector layer, in a format accepted by QgsCoordinateReferenceSystem.createFromString (for example &quot;EPSG:4167&quot;). If this is not
specified then a dialog box may request this information from the user
when the layer is loaded (depending on QGIS CRS settings).</li>
Expand Down
4 changes: 4 additions & 0 deletions src/core/qgsvectorlayer.h
Expand Up @@ -366,6 +366,10 @@ struct CORE_EXPORT QgsVectorJoinInfo
* geomType can also be set to none, in which case the layer is loaded without
* geometries.
*
* - subset=expression
*
* Defines an expression that will identify a subset of records to display
*
* - crs=crsstring
*
* Defines the coordinate reference system used for the layer. This can be
Expand Down
3 changes: 3 additions & 0 deletions src/core/qgsvectorlayerfeatureiterator.cpp
Expand Up @@ -38,9 +38,12 @@ QgsVectorLayerFeatureIterator::QgsVectorLayerFeatureIterator( QgsVectorLayer* la
// prepare list of attributes to match provider fields
QgsAttributeList providerSubset;
QgsAttributeList subset = mProviderRequest.subsetOfAttributes();
const QgsFields &pendingFields = L->pendingFields();
int nPendingFields = pendingFields.count();
for ( int i = 0; i < subset.count(); ++i )
{
int attrIndex = subset[i];
if( attrIndex < 0 || attrIndex >= nPendingFields ) continue;
if ( L->pendingFields().fieldOrigin( attrIndex ) == QgsFields::OriginProvider )
providerSubset << L->pendingFields().fieldOriginIndex( attrIndex );
}
Expand Down
171 changes: 3 additions & 168 deletions src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp
Expand Up @@ -49,77 +49,9 @@ bool QgsDelimitedTextFeatureIterator::nextFeature( QgsFeature& feature )
if ( mClosed )
return false;

QStringList tokens;
while ( true )
{
QgsDelimitedTextFile::Status status = P->mFile->nextRecord( tokens );
int fid = P->mFile->recordLineNumber();
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;
if ( P->recordIsEmpty( tokens ) ) continue;

while ( tokens.size() < P->mFieldCount )
tokens.append( QString::null );

QgsGeometry *geom = 0;

if ( P->mWktFieldIndex >= 0 )
{
geom = loadGeometryWkt( tokens );
}
else if ( P->mXFieldIndex >= 0 && P->mYFieldIndex >= 0 )
{
geom = loadGeometryXY( tokens );
}

if ( !geom && P->mWkbType != QGis::WKBNoGeometry )
{
// Already dealt with invalid lines in provider - no need to repeat
// removed code (CC 2013-04-13) ...
// P->mInvalidLines << line;
// In any case it may be a valid line that is excluded because of
// bounds check...
continue;
}

// At this point the current feature values are valid

feature.setValid( true );
feature.setFields( &P->attributeFields ); // allow name-based attribute lookups
feature.setFeatureId( fid );
feature.initAttributes( P->attributeFields.count() );

if ( geom )
feature.setGeometry( geom );

if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
const QgsAttributeList& attrs = mRequest.subsetOfAttributes();
for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
{
int fieldIdx = *i;
if ( fieldIdx < 0 || fieldIdx >= P->attributeColumns.count() )
continue; // ignore non-existant fields
fetchAttribute( feature, fieldIdx, tokens );
}
}
else
{
for ( int idx = 0; idx < P->attributeFields.count(); ++idx )
fetchAttribute( feature, idx, tokens );
}

// We have a good line, so return
return true;

} // !mStream->atEnd()

// End of the file. If there are any lines that couldn't be
// loaded, display them now.
// P->handleInvalidLines();

close();
return false;
bool gotFeature = P->nextFeature(feature,P->mFile,mRequest);
if( ! gotFeature ) close();
return gotFeature;
}

bool QgsDelimitedTextFeatureIterator::rewind()
Expand All @@ -142,100 +74,3 @@ bool QgsDelimitedTextFeatureIterator::close()
mClosed = true;
return true;
}


QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryWkt( const QStringList& tokens )
{
QgsGeometry* geom = 0;
QString sWkt = tokens[P->mWktFieldIndex];

geom = P->geomFromWkt( sWkt );

if ( geom && geom->type() != P->mGeometryType )
{
delete geom;
geom = 0;
}
if ( geom && !boundsCheck( geom ) )
{
delete geom;
geom = 0;
}
return geom;
}


QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryXY( const QStringList& tokens )
{
QString sX = tokens[P->mXFieldIndex];
QString sY = tokens[P->mYFieldIndex];
QgsPoint pt;
bool ok = P->pointFromXY( sX, sY, pt );

if ( ok && boundsCheck( pt ) )
{
return QgsGeometry::fromPoint( pt );
}
return 0;
}

/**
* Check to see if the point is within the selection rectangle
*/
bool QgsDelimitedTextFeatureIterator::boundsCheck( const QgsPoint &pt )
{
// no selection rectangle or geometry => always in the bounds
if ( mRequest.filterType() != QgsFeatureRequest::FilterRect || ( mRequest.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

return mRequest.filterRect().contains( pt );
}

/**
* Check to see if the geometry is within the selection rectangle
*/
bool QgsDelimitedTextFeatureIterator::boundsCheck( QgsGeometry *geom )
{
// no selection rectangle or geometry => always in the bounds
if ( mRequest.filterType() != QgsFeatureRequest::FilterRect || ( mRequest.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

if ( mRequest.flags() & QgsFeatureRequest::ExactIntersect )
return geom->intersects( mRequest.filterRect() );
else
return geom->boundingBox().intersects( mRequest.filterRect() );
}


void QgsDelimitedTextFeatureIterator::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
{
const QString &value = tokens[P->attributeColumns[fieldIdx]];
QVariant val;
switch ( P->attributeFields[fieldIdx].type() )
{
case QVariant::Int:
if ( value.isEmpty() )
val = QVariant( P->attributeFields[fieldIdx].type() );
else
val = QVariant( value );
break;
case QVariant::Double:
if ( value.isEmpty() )
{
val = QVariant( P->attributeFields[fieldIdx].type() );
}
else if ( P->mDecimalPoint.isEmpty() )
{
val = QVariant( value.toDouble() );
}
else
{
val = QVariant( QString( value ).replace( P->mDecimalPoint, "." ).toDouble() );
}
break;
default:
val = QVariant( value );
break;
}
feature.setAttribute( fieldIdx, val );
}
8 changes: 0 additions & 8 deletions src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h
Expand Up @@ -37,14 +37,6 @@ class QgsDelimitedTextFeatureIterator : public QgsAbstractFeatureIterator

protected:
QgsDelimitedTextProvider* P;

QgsGeometry* loadGeometryWkt( const QStringList& tokens );
QgsGeometry* loadGeometryXY( const QStringList& tokens );

bool boundsCheck( const QgsPoint &pt );
bool boundsCheck( QgsGeometry *geom );

void fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens );
};


Expand Down
2 changes: 2 additions & 0 deletions src/providers/delimitedtext/qgsdelimitedtextfile.cpp
Expand Up @@ -43,6 +43,7 @@ QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
mTrimFields( false ),
mSkipLines( 0 ),
mMaxFields( 0 ),
mMaxNameLength( 200 ), // Don't want field names to be too unweildy!
mLineNumber( 0 ),
mRecordLineNumber( 0 ),
mMaxFieldCount( 0 )
Expand Down Expand Up @@ -377,6 +378,7 @@ void QgsDelimitedTextFile::setFieldNames( const QStringList &names )
bool nameOk = true;
int fieldNo = mFieldNames.size() + 1;
name = name.trimmed();
if( name.length() > mMaxNameLength ) name=name.mid(0,mMaxNameLength );

// If the name is invalid then reset it to default name
if ( InvalidFieldRegexp.exactMatch( name ) )
Expand Down
1 change: 1 addition & 0 deletions src/providers/delimitedtext/qgsdelimitedtextfile.h
Expand Up @@ -322,6 +322,7 @@ class QgsDelimitedTextFile
bool mTrimFields;
int mSkipLines;
int mMaxFields;
int mMaxNameLength;

// Parameters used by parsers
QRegExp mDelimRegexp;
Expand Down

0 comments on commit 5e9e98a

Please sign in to comment.