Skip to content

Commit

Permalink
Adding subset test and help information
Browse files Browse the repository at this point in the history
  • Loading branch information
@ccrook
ccrook committed Apr 30, 2013
1 parent ea8617d commit 1fb60b4
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 199 deletions.
2 changes: 2 additions & 0 deletions resources/context_help/QgsDelimitedTextSourceSelect-en_US
View file
Expand Up @@ -270,6 +270,8 @@ The following options can be added
<li><tt>xField=fieldname</tt> specifies the name or number (starting at 1) of the field the X coordinate (only applies if wktField is not defined)</li>
<li><tt>yField=fieldname</tt> specifies the name or number (starting at 1) of the field the Y coordinate (only applies if wktField is not defined)</li>
<li><tt>geomType=(auto|point|line|polygon|none)</tt> specifies type of geometry for wkt fields, or none to load the file as an attribute-only table. The default is auto.</li>
<li><tt>subset=expression</tt> specifies an expression used to identify a subset of the records that will be
used.</li>
<li><tt>crs=...</tt> specifies the coordinate system to use for the vector layer, in a format accepted by QgsCoordinateReferenceSystem.createFromString (for example &quot;EPSG:4167&quot;). If this is not
specified then a dialog box may request this information from the user
when the layer is loaded (depending on QGIS CRS settings).</li>
Expand Down
4 changes: 4 additions & 0 deletions src/core/qgsvectorlayer.h
View file
Expand Up @@ -366,6 +366,10 @@ struct CORE_EXPORT QgsVectorJoinInfo
* geomType can also be set to none, in which case the layer is loaded without
* geometries.
*
* - subset=expression
*
* Defines an expression that will identify a subset of records to display
*
* - crs=crsstring
*
* Defines the coordinate reference system used for the layer. This can be
Expand Down
199 changes: 0 additions & 199 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View file
Expand Up @@ -879,205 +879,6 @@ void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx
feature.setAttribute( fieldIdx, val );
}

void QgsDelimitedTextProvider::resetDataSummary()
{
QgsFeatureIterator fi = getFeatures(QgsFeatureRequest());
mNumberFeatures = 0;
mExtent = QgsRectangle();
QgsFeature f;
while( fi.nextFeature(f))
{
if( mGeometryType != QGis::NoGeometry )
{
if( mNumberFeatures == 0 )
{
mExtent = f.geometry()->boundingBox();
}
else
{
QgsRectangle bbox( f.geometry()->boundingBox());
mExtent.combineExtentWith( &bbox);
}
}
mNumberFeatures++;
}
}


bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request )
{
QStringList tokens;
while ( true )
{
// before we do anything else, assume that there's something wrong with
// the feature
feature.setValid( false );
QgsDelimitedTextFile::Status status = file->nextRecord( tokens );
if ( status == QgsDelimitedTextFile::RecordEOF ) break;
if ( status != QgsDelimitedTextFile::RecordOk ) continue;

int fid = file->recordLineNumber();
if( request.filterType() == QgsFeatureRequest::FilterFid && fid != request.filterFid()) continue;
if ( recordIsEmpty( tokens ) ) continue;

while ( tokens.size() < mFieldCount )
tokens.append( QString::null );

QgsGeometry *geom = 0;

if ( mWktFieldIndex >= 0 )
{
geom = loadGeometryWkt( tokens, request );
}
else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
{
geom = loadGeometryXY( tokens,request );
}

if ( !geom && mWkbType != QGis::WKBNoGeometry )
{
// Already dealt with invalid lines in provider - no need to repeat
// removed code (CC 2013-04-13) ...
// mInvalidLines << line;
// In any case it may be a valid line that is excluded because of
// bounds check...
continue;
}

// At this point the current feature values are valid

feature.setValid( true );
feature.setFields( &attributeFields ); // allow name-based attribute lookups
feature.setFeatureId( fid );
feature.initAttributes( attributeFields.count() );

if ( geom )
feature.setGeometry( geom );

if ( request.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
const QgsAttributeList& attrs = request.subsetOfAttributes();
for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
{
int fieldIdx = *i;
fetchAttribute( feature, fieldIdx, tokens );
}
}
else
{
for ( int idx = 0; idx < attributeFields.count(); ++idx )
fetchAttribute( feature, idx, tokens );
}

// We have a good line, so return
return true;

} // !mStream->atEnd()

return false;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request )
{
QgsGeometry* geom = 0;
QString sWkt = tokens[mWktFieldIndex];

geom = geomFromWkt( sWkt );

if ( geom && geom->type() != mGeometryType )
{
delete geom;
geom = 0;
}
if ( geom && !boundsCheck( geom, request ) )
{
delete geom;
geom = 0;
}
return geom;
}


QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request )
{
QString sX = tokens[mXFieldIndex];
QString sY = tokens[mYFieldIndex];
QgsPoint pt;
bool ok = pointFromXY( sX, sY, pt );

if ( ok && boundsCheck( pt, request ) )
{
return QgsGeometry::fromPoint( pt );
}
return 0;
}

/**
* Check to see if the point is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

return request.filterRect().contains( pt );
}

/**
* Check to see if the geometry is within the selection rectangle
*/
bool QgsDelimitedTextProvider::boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request )
{
// no selection rectangle or geometry => always in the bounds
if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
return true;

if ( request.flags() & QgsFeatureRequest::ExactIntersect )
return geom->intersects( request.filterRect() );
else
return geom->boundingBox().intersects( request.filterRect() );
}


void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
{
if( fieldIdx < 0 || fieldIdx >= attributeColumns.count()) return;
int column = attributeColumns[fieldIdx];
if( column < 0 || column >= tokens.count()) return;
const QString &value = tokens[column];
QVariant val;
switch ( attributeFields[fieldIdx].type() )
{
case QVariant::Int:
if ( value.isEmpty() )
val = QVariant( attributeFields[fieldIdx].type() );
else
val = QVariant( value );
break;
case QVariant::Double:
if ( value.isEmpty() )
{
val = QVariant( attributeFields[fieldIdx].type() );
}
else if ( mDecimalPoint.isEmpty() )
{
val = QVariant( value.toDouble() );
}
else
{
val = QVariant( QString( value ).replace( mDecimalPoint, "." ).toDouble() );
}
break;
default:
val = QVariant( value );
break;
}
feature.setAttribute( fieldIdx, val );
}


// Return the extent of the layer
QgsRectangle QgsDelimitedTextProvider::extent()
{
Expand Down
43 changes: 43 additions & 0 deletions tests/src/python/test_qgsdelimitedtextprovider.py
View file
Expand Up @@ -1692,6 +1692,49 @@ def test_033_filter_attributes(self):
]
runTest(description,wanted,filename,requests,**params)

def test_034_substring_test(self):
description='CSV file parsing'
filename='test.csv'
params={'geomType': 'none', 'subset': 'id % 2 = 1', 'type': 'csv'}
requests=None
if rebuildTests:
createTest(description,filename,requests,**params)
assert False,"Set rebuildTests to False to run delimited text tests"
wanted={}
wanted['uri']=u'file://file?geomType=none&type=csv&subset=id%20%25%202%20%3D%201'
wanted['data']={
2L: {
'id': u'1',
'description': u'Basic unquoted record',
'data': u'Some data',
'info': u'Some info',
'field_5': u'',
'#fid': 2L,
'#geometry': 'None',
},
4L: {
'id': u'3',
'description': u'Escaped quotes',
'data': u'Quoted "citation" data',
'info': u'Unquoted',
'field_5': u'',
'#fid': 4L,
'#geometry': 'None',
},
9L: {
'id': u'5',
'description': u'Extra fields',
'data': u'data',
'info': u'info',
'field_5': u'message',
'#fid': 9L,
'#geometry': 'None',
},
}
wanted['log']=[
]
runTest(description,wanted,filename,requests,**params)

#END

if __name__ == '__main__':
Expand Down

0 comments on commit 1fb60b4

Please sign in to comment.