Substring working - tests not in place yet

qgis · Apr 30, 2013 · ea8617d · ea8617d
1 parent 1fb1c9c
commit ea8617d
Show file tree

Hide file tree

Showing 3 changed files with 310 additions and 2 deletions.
diff --git a/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp b/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
@@ -29,6 +29,7 @@
 
 #include "qgsapplication.h"
 #include "qgsdataprovider.h"
+#include "qgsexpression.h"
 #include "qgsfeature.h"
 #include "qgsfield.h"
 #include "qgsgeometry.h"
@@ -60,6 +61,8 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
     , mWktHasZM( false )
     , mWktHasPrefix( false )
     , mXyDms( false )
+    , mSubsetString("")
+    , mSubsetExpression( 0 )
     , mMaxInvalidLines( 50 )
     , mShowInvalidLines( true )
     , mCrs()
@@ -507,6 +510,12 @@ QgsDelimitedTextProvider::~QgsDelimitedTextProvider()
     delete mFile;
     mFile = 0;
   }
+
+  if( mSubsetExpression )
+  {
+    delete mSubsetExpression;
+    mSubsetExpression = 0;
+  }
 }
 
 
@@ -597,6 +606,278 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
   }
 }
 
+//
+
+bool QgsDelimitedTextProvider::setSubsetString( QString subset, bool updateFeatureCount )
+{
+
+  bool valid = true;
+
+  // If there is a new subset string then encode it..
+
+  QgsExpression *expression = 0;
+  if( ! subset.isEmpty() )
+  {
+
+    expression = new QgsExpression( subset );
+    QString error;
+    if( expression->hasParserError())
+    {
+      error = expression->parserErrorString();
+    }
+    else
+    {
+      expression->prepare( fields() );
+      if( expression->hasEvalError() )
+      {
+        error = expression->evalErrorString();
+      }
+    }
+    if( ! error.isEmpty() )
+    {
+      valid = false;
+      delete expression;
+      expression = 0;
+      QString tag( "DelimitedText" );
+      QgsMessageLog::logMessage( tr( "Invalid subset string %1 for %2" ).arg(subset).arg( mFile->fileName() ), tag );
+    }
+  }
+
+
+  // if the expression is valid, then reset the subset string and data source Uri
+
+  if( valid )
+  {
+
+    if( mSubsetExpression ) delete mSubsetExpression;
+    mSubsetString = subset;
+    mSubsetExpression = expression;
+
+    // Encode the subset string into the data source URI.
+
+    QUrl url = QUrl::fromEncoded( dataSourceUri().toAscii() );
+    if( url.hasQueryItem("subset")) url.removeAllQueryItems("subset");
+    if( ! subset.isEmpty()) url.addQueryItem("subset",subset);
+    setDataSourceUri( QString::fromAscii( url.toEncoded() ) );
+
+    // Update the feature count and extents if requested
+    if( updateFeatureCount )
+    {
+      resetDataSummary();
+    }
+  }
+
+  return valid;
+}
+
+void QgsDelimitedTextProvider::resetDataSummary()
+{
+  QgsFeatureIterator fi = getFeatures(QgsFeatureRequest());
+  mNumberFeatures = 0;
+  mExtent = QgsRectangle();
+  QgsFeature f;
+  while( fi.nextFeature(f))
+  {
+    if( mGeometryType != QGis::NoGeometry )
+    {
+      if( mNumberFeatures == 0 )
+      {
+        mExtent = f.geometry()->boundingBox();
+      }
+      else
+      {
+        QgsRectangle bbox( f.geometry()->boundingBox());
+        mExtent.combineExtentWith( &bbox);
+      }
+    }
+    mNumberFeatures++;
+  }
+}
+
+
+bool QgsDelimitedTextProvider::nextFeature( QgsFeature& feature, QgsDelimitedTextFile *file, const QgsFeatureRequest& request )
+{
+  QStringList tokens;
+  while ( true )
+  {
+    // before we do anything else, assume that there's something wrong with
+    // the feature
+    feature.setValid( false );
+    QgsDelimitedTextFile::Status status = file->nextRecord( tokens );
+    if ( status == QgsDelimitedTextFile::RecordEOF ) break;
+    if ( status != QgsDelimitedTextFile::RecordOk ) continue;
+
+    int fid = file->recordLineNumber();
+    if( request.filterType() == QgsFeatureRequest::FilterFid && fid != request.filterFid()) continue;
+    if ( recordIsEmpty( tokens ) ) continue;
+
+    while ( tokens.size() < mFieldCount )
+      tokens.append( QString::null );
+
+    QgsGeometry *geom = 0;
+
+    if ( mWktFieldIndex >= 0 )
+    {
+      geom = loadGeometryWkt( tokens, request );
+    }
+    else if ( mXFieldIndex >= 0 && mYFieldIndex >= 0 )
+    {
+      geom = loadGeometryXY( tokens,request );
+    }
+
+    if ( !geom && mWkbType != QGis::WKBNoGeometry )
+    {
+      // Already dealt with invalid lines in provider - no need to repeat
+      // removed code (CC 2013-04-13) ...
+      //    mInvalidLines << line;
+      // In any case it may be a valid line that is excluded because of
+      // bounds check...
+      continue;
+    }
+
+    // At this point the current feature values are valid
+
+    feature.setValid( true );
+    feature.setFields( &attributeFields ); // allow name-based attribute lookups
+    feature.setFeatureId( fid );
+    feature.initAttributes( attributeFields.count() );
+
+    if ( geom )
+      feature.setGeometry( geom );
+
+    // If we have subset expression, then ened attributes
+    if ( ! mSubsetExpression && request.flags() & QgsFeatureRequest::SubsetOfAttributes )
+    {
+      const QgsAttributeList& attrs = request.subsetOfAttributes();
+      for ( QgsAttributeList::const_iterator i = attrs.begin(); i != attrs.end(); ++i )
+      {
+        int fieldIdx = *i;
+        fetchAttribute( feature, fieldIdx, tokens );
+      }
+    }
+    else
+    {
+      for ( int idx = 0; idx < attributeFields.count(); ++idx )
+        fetchAttribute( feature, idx, tokens );
+    }
+
+    // Are we using a subset expression, if so try and evaluate
+    // and accept result if passes.
+
+    if( mSubsetExpression )
+    {
+      QVariant isOk = mSubsetExpression->evaluate( &feature );
+      if( mSubsetExpression->hasEvalError() ) continue;
+      if( ! isOk.toBool() ) continue;
+    }
+
+    // We have a good line, so return
+    return true;
+
+  } // !mStream->atEnd()
+
+  return false;
+}
+
+
+QgsGeometry* QgsDelimitedTextProvider::loadGeometryWkt( const QStringList& tokens, const QgsFeatureRequest& request )
+{
+  QgsGeometry* geom = 0;
+  QString sWkt = tokens[mWktFieldIndex];
+
+  geom = geomFromWkt( sWkt );
+
+  if ( geom && geom->type() != mGeometryType )
+  {
+    delete geom;
+    geom = 0;
+  }
+  if ( geom && !boundsCheck( geom, request ) )
+  {
+    delete geom;
+    geom = 0;
+  }
+  return geom;
+}
+
+
+QgsGeometry* QgsDelimitedTextProvider::loadGeometryXY( const QStringList& tokens, const QgsFeatureRequest& request )
+{
+  QString sX = tokens[mXFieldIndex];
+  QString sY = tokens[mYFieldIndex];
+  QgsPoint pt;
+  bool ok = pointFromXY( sX, sY, pt );
+
+  if ( ok && boundsCheck( pt, request ) )
+  {
+    return QgsGeometry::fromPoint( pt );
+  }
+  return 0;
+}
+
+/**
+ * Check to see if the point is within the selection rectangle
+ */
+bool QgsDelimitedTextProvider::boundsCheck( const QgsPoint &pt, const QgsFeatureRequest& request )
+{
+  // no selection rectangle or geometry => always in the bounds
+  if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
+    return true;
+
+  return request.filterRect().contains( pt );
+}
+
+/**
+ * Check to see if the geometry is within the selection rectangle
+ */
+bool QgsDelimitedTextProvider::boundsCheck( QgsGeometry *geom, const QgsFeatureRequest& request )
+{
+  // no selection rectangle or geometry => always in the bounds
+  if ( request.filterType() != QgsFeatureRequest::FilterRect || ( request.flags() & QgsFeatureRequest::NoGeometry ) )
+    return true;
+
+  if ( request.flags() & QgsFeatureRequest::ExactIntersect )
+    return geom->intersects( request.filterRect() );
+  else
+    return geom->boundingBox().intersects( request.filterRect() );
+}
+
+
+void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx, const QStringList& tokens )
+{
+  if( fieldIdx < 0 || fieldIdx >= attributeColumns.count()) return;
+  int column = attributeColumns[fieldIdx];
+  if( column < 0 || column >= tokens.count()) return;
+  const QString &value = tokens[column];
+  QVariant val;
+  switch ( attributeFields[fieldIdx].type() )
+  {
+    case QVariant::Int:
+      if ( value.isEmpty() )
+        val = QVariant( attributeFields[fieldIdx].type() );
+      else
+        val = QVariant( value );
+      break;
+    case QVariant::Double:
+      if ( value.isEmpty() )
+      {
+        val = QVariant( attributeFields[fieldIdx].type() );
+      }
+      else if ( mDecimalPoint.isEmpty() )
+      {
+        val = QVariant( value.toDouble() );
+      }
+      else
+      {
+        val = QVariant( QString( value ).replace( mDecimalPoint, "." ).toDouble() );
+      }
+      break;
+    default:
+      val = QVariant( value );
+      break;
+  }
+  feature.setAttribute( fieldIdx, val );
+}
 
 void QgsDelimitedTextProvider::resetDataSummary()
 {

diff --git a/src/providers/delimitedtext/qgsdelimitedtextprovider.h b/src/providers/delimitedtext/qgsdelimitedtextprovider.h
@@ -30,6 +30,7 @@ class QTextStream;
 
 class QgsDelimitedTextFeatureIterator;
 class QgsDelimitedTextFile;
+class QgsExpression;
 
 /**
 \class QgsDelimitedTextProvider
@@ -142,7 +143,28 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
     bool isValid();
 
     virtual QgsCoordinateReferenceSystem crs();
+    /**
+     * Set the subset string used to create a subset of features in
+     * the layer.
+     */
+    virtual bool setSubsetString( QString subset, bool updateFeatureCount = true );
+
+    /**
+     * provider supports setting of subset strings
+
+     */
+    virtual bool supportsSubsetString() { return true; }
 
+    /**
+     * Returns the subset definition string (typically sql) currently in
+     * use by the layer and used by the provider to limit the feature set.
+     * Must be overridden in the dataprovider, otherwise returns a null
+     * QString.
+     */
+    virtual QString subsetString()
+    {
+      return mSubsetString;
+    }
     /* new functions */
 
     /**
@@ -223,6 +245,9 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
     QString mDecimalPoint;
     bool mXyDms;
 
+    QString mSubsetString;
+    QgsExpression *mSubsetExpression;
+
     //! Storage for any lines in the file that couldn't be loaded
     int mMaxInvalidLines;
     int mNExtraInvalidLines;

diff --git a/tests/src/python/test_qgsdelimitedtextprovider.py b/tests/src/python/test_qgsdelimitedtextprovider.py
@@ -1391,13 +1391,14 @@ def test_019_latin1_encoded_file(self):
         runTest(description,wanted,filename,requests,**params)
 
 
+
     def test_030_filter_rect_xy(self):
         description='Filter extents on XY layer'
         filename='testextpt.txt'
         params={'yField': 'y', 'delimiter': '|', 'type': 'csv', 'xField': 'x'}
         requests=[
             {'extents': [10, 30, 30, 50]}, 
-            {'exact': 1, 'extents': [10, 30, 30, 50]}, 
+            {'extents': [10, 30, 30, 50], 'exact': 1}, 
             {'extents': [110, 130, 130, 150]}]
         if rebuildTests:
             createTest(description,filename,requests,**params)
@@ -1449,7 +1450,7 @@ def test_031_filter_rect_wkt(self):
         params={'delimiter': '|', 'type': 'csv', 'wktField': 'wkt'}
         requests=[
             {'extents': [10, 30, 30, 50]}, 
-            {'exact': 1, 'extents': [10, 30, 30, 50]}, 
+            {'extents': [10, 30, 30, 50], 'exact': 1}, 
             {'extents': [110, 130, 130, 150]}]
         if rebuildTests:
             createTest(description,filename,requests,**params)
@@ -1569,6 +1570,7 @@ def test_032_filter_fid(self):
         runTest(description,wanted,filename,requests,**params)
 
 
+
     def test_033_filter_attributes(self):
         description='Filter on attributes'
         filename='test.csv'