Skip to content

Commit

Permalink
Tests completed successfully
Browse files Browse the repository at this point in the history
  • Loading branch information
@ccrook
ccrook committed Apr 13, 2013
1 parent 1abb42f commit aac902a
Show file tree
Hide file tree
Showing 12 changed files with 538 additions and 214 deletions.
9 changes: 9 additions & 0 deletions src/core/qgsvectorlayer.h
View file
Expand Up @@ -310,6 +310,15 @@ struct CORE_EXPORT QgsVectorJoinInfo
*
* Defines the characters used to escape delimiter, quote, and newline characters.
*
* - skipEmptyFields=(yes|no)
*
* If yes then empty fields will be discarded (eqivalent to concatenating consecutive
* delimiters)
*
* - trimFields=(yes|no)
*
* If yes then leading and trailing whitespace will be removed from fields
*
* - skipLines=n
*
* Defines the number of lines to ignore at the beginning of the file (default 0)
Expand Down
5 changes: 5 additions & 0 deletions src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp
View file
Expand Up @@ -152,6 +152,11 @@ QgsGeometry* QgsDelimitedTextFeatureIterator::loadGeometryWkt( const QStringList
try
{
QString sWkt = tokens[P->mWktFieldIndex];
// Remove prefix chars
if( P->mWktHasPrefix)
{
sWkt.remove(QgsDelimitedTextProvider::WktPrefixRegexp);
}
// Remove Z and M coordinates if present, as currently fromWkt doesn't
// support these.
if ( P->mWktHasZM )
Expand Down
96 changes: 72 additions & 24 deletions src/providers/delimitedtext/qgsdelimitedtextfile.cpp
View file
Expand Up @@ -34,6 +34,8 @@ QgsDelimitedTextFile::QgsDelimitedTextFile( QString url ) :
mStream(0),
mDefinitionValid(false),
mUseHeader(true),
mDiscardEmptyFields(false),
mTrimFields(false),
mSkipLines(0),
mMaxFields(0),
mLineNumber(0),
Expand Down Expand Up @@ -90,6 +92,7 @@ void QgsDelimitedTextFile::resetDefinition()
{
close();
mColumnNames.clear();
mMaxFields=0;
}

// Extract the provider definition from the url
Expand Down Expand Up @@ -159,7 +162,15 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
}
if ( url.hasQueryItem( "useHeader" ) )
{
mUseHeader = url.queryItemValue( "useHeader" ).toUpper().startsWith('Y');
mUseHeader = ! url.queryItemValue( "useHeader" ).toUpper().startsWith('N');
}
if( url.hasQueryItem("skipEmptyFields"))
{
mDiscardEmptyFields = ! url.queryItemValue( "useHeader" ).toUpper().startsWith('N');;
}
if( url.hasQueryItem("trimFields"))
{
mTrimFields = ! url.queryItemValue( "trimFields" ).toUpper().startsWith('N');;
}

QgsDebugMsg( "Delimited text file is: " + mFileName );
Expand All @@ -169,7 +180,9 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
QgsDebugMsg( "Quote character is: [" + quote +"]");
QgsDebugMsg( "Escape character is: [" + escape + "]");
QgsDebugMsg( "Skip lines: " + QString::number(mSkipLines) );
QgsDebugMsg( "Skip lines: " + QString(mUseHeader ? "Yes" : "No") );
QgsDebugMsg( "Use headers: " + QString(mUseHeader ? "Yes" : "No") );
QgsDebugMsg( "Discard empty fields: " + QString(mDiscardEmptyFields ? "Yes" : "No") );
QgsDebugMsg( "Trim fields: " + QString(mTrimFields ? "Yes" : "No") );

// Support for previous version of plain characters
if( type == "csv" || type == "plain" )
Expand Down Expand Up @@ -201,13 +214,13 @@ QUrl QgsDelimitedTextFile::url()
url.addQueryItem("type",type());
if( mType == DelimTypeRegexp )
{
url.addQueryItem("delimiter",delimiterDefinitionString());
url.addQueryItem("delimiter",mDelimRegexp.pattern());
}
if( mType == DelimTypeCSV )
{
if( mDelimChars != "," ) url.addQueryItem("delimiter",delimiterDefinitionString());
if( mQuoteChar != "\"" ) url.addQueryItem("quote",mQuoteChar);
if( mEscapeChar != "\"" ) url.addQueryItem("escape",mEscapeChar);
if( mDelimChars != "," ) url.addQueryItem("delimiter",encodeChars(mDelimChars));
if( mQuoteChar != "\"" ) url.addQueryItem("quote",encodeChars(mQuoteChar));
if( mEscapeChar != "\"" ) url.addQueryItem("escape",encodeChars(mEscapeChar));
}
if( mSkipLines > 0 )
{
Expand All @@ -217,6 +230,14 @@ QUrl QgsDelimitedTextFile::url()
{
url.addQueryItem("useHeader","No");
}
if( mTrimFields )
{
url.addQueryItem("trimFields","Yes");
}
if( mDiscardEmptyFields && mType != DelimTypeWhitespace )
{
url.addQueryItem("skipEmptyFields","Yes");
}
return url;
}

Expand All @@ -243,16 +264,15 @@ QString QgsDelimitedTextFile::type()
void QgsDelimitedTextFile::setTypeWhitespace()
{
setTypeRegexp("\\s+");
mDiscardEmptyFields=true;
mType=DelimTypeWhitespace;
mDelimDefinition = "";
}

void QgsDelimitedTextFile::setTypeRegexp( QString regexp )
{
resetDefinition();
mType=DelimTypeRegexp;
mDelimRegexp.setPattern(regexp);
mDelimDefinition=regexp;
mParser=&QgsDelimitedTextFile::parseRegexp;
mDefinitionValid = regexp.size() > 0 && mDelimRegexp.isValid();
if( ! mDefinitionValid )
Expand All @@ -261,18 +281,28 @@ void QgsDelimitedTextFile::setTypeRegexp( QString regexp )
}
}

QString QgsDelimitedTextFile::decodeChars( QString chars )
{
chars = chars.replace("\\t","\t");
return chars;
}

QString QgsDelimitedTextFile::encodeChars( QString chars )
{
chars = chars.replace("\t","\\t");
return chars;
}

void QgsDelimitedTextFile::setTypeCSV( QString delim, QString quote, QString escape )
{
resetDefinition();
mType=DelimTypeRegexp;
mType=DelimTypeCSV;
mDelimDefinition=delim;
mDelimChars=delim;
mQuoteChar=quote;
mEscapeChar= escape;
mDelimChars.replace("\\t","\t");
mDelimChars=decodeChars(delim);
mQuoteChar=decodeChars(quote);
mEscapeChar= decodeChars(escape);
mParser=&QgsDelimitedTextFile::parseQuoted;
mDefinitionValid = delim.size() > 0;
mDefinitionValid = mDelimChars.size() > 0;
if( ! mDefinitionValid )
{
QgsDebugMsg("Invalid empty delimiter defined for text file delimiter");
Expand All @@ -291,6 +321,18 @@ void QgsDelimitedTextFile::setUseHeader( bool useheader )
mUseHeader = useheader;
}

void QgsDelimitedTextFile::setTrimFields( bool trimFields )
{
resetDefinition();
mTrimFields = trimFields;
}

void QgsDelimitedTextFile::setDiscardEmptyFields( bool discardEmptyFields )
{
resetDefinition();
mDiscardEmptyFields = discardEmptyFields;
}

QStringList &QgsDelimitedTextFile::columnNames()
{
// If not yet opened then reset file to read column headers
Expand Down Expand Up @@ -324,7 +366,9 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::reset()
// Read the column names
if( mUseHeader )
{
return (this->*mParser)(mColumnNames);
QgsDelimitedTextFile::Status result = (this->*mParser)(mColumnNames);
mMaxFields = mColumnNames.size();
return result;
}
return RecordOk;
}
Expand Down Expand Up @@ -352,16 +396,19 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::nextLine( QString &buffer, bo

QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseRegexp( QStringList &fields )
{
fields.clear();
QString buffer;
Status status = nextLine(buffer,true);
if( status != RecordOk ) return status;
mRecordLineNumber = mLineNumber;

if( mType == DelimTypeWhitespace ) buffer=buffer.trimmed();
fields = buffer.split(mDelimRegexp);
if( mMaxFields > 0 && fields.size() > mMaxFields )
QStringList parts = buffer.split(mDelimRegexp);
foreach( QString f, parts )
{
fields = fields.mid(0,mMaxFields);
if( mTrimFields ) f = f.trimmed();
if( mDiscardEmptyFields && f.isEmpty()) continue;
fields.append(f);
if( mMaxFields > 0 && fields.size() >= mMaxFields ) break;
}
return RecordOk;
}
Expand Down Expand Up @@ -456,6 +503,7 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
// quote char at start of field .. start of quoted fields
else if( ! started )
{
field.clear();
quoteChar = c;
quoted = true;
started = true;
Expand Down Expand Up @@ -483,8 +531,8 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
if( mMaxFields <= 0 || fields.size() < mMaxFields )
{
// If wasn't quoted, then trim..
if( ! ended ) field = field.trimmed();
fields.append(field);
if( mTrimFields && ! ended) field = field.trimmed();
if( ! field.isEmpty() || ended || ! mDiscardEmptyFields ) fields.append(field);
}
// Clear the field
field.clear();
Expand All @@ -495,7 +543,7 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
// after the end..
else if( c.isSpace() )
{
if( started && ! ended ) field.append(c);
if( ! ended ) field.append(c);
}
// Other chars permitted if not after quoted field
else
Expand All @@ -512,8 +560,8 @@ QgsDelimitedTextFile::Status QgsDelimitedTextFile::parseQuoted( QStringList &fie
// If reached the end of the record, then add the last field...
if( started && (mMaxFields <=0 || fields.size() < mMaxFields) )
{
if( ! ended ) field = field.trimmed();
fields.append(field);
if( mTrimFields && ! ended ) field = field.trimmed();
if( ! field.isEmpty() || ended || ! mDiscardEmptyFields ) fields.append(field);
}
return RecordOk;
}
Expand Down
76 changes: 40 additions & 36 deletions src/providers/delimitedtext/qgsdelimitedtextfile.h
View file
Expand Up @@ -141,22 +141,6 @@ class QgsDelimitedTextFile
*/
void setTypeCSV( QString delim=QString(","), QString quote=QString("\""), QString escape=QString("\"") );

/* Specify the maximum number of fields that will be read into a record.
*
* Any fields after this a record will be silently ignored. Use 0 to
* return an unlimited number of fields
* @param maxFields The maximum number of fields into which a record will be split
*/
void setMaxFields( int maxFields ) {
mMaxFields = maxFields;
}
/* Return the maximum number of fields to return
* @return maxFields The maximum number of fields to return
*/
int maxFields() {
return mMaxFields;
}

/* Set the number of header lines to skip
* @param skiplines The maximum lines to skip
*/
Expand All @@ -179,6 +163,28 @@ class QgsDelimitedTextFile
return mUseHeader;
}

/* Set the option for dicarding empty fields
* @param useheaders Empty fields will be discarded if true
*/
void setDiscardEmptyFields( bool discardEmptyFields=true );
/* Return the option for discarding empty fields
* @return useheaders Empty fields will be discarded if true
*/
bool discardEmptyFields() {
return mDiscardEmptyFields;
}

/* Set the option for trimming whitespace from fields
* @param trimFields Fields will be trimmed if true
*/
void setTrimFields( bool trimFields=true );
/* Return the option for trimming empty fields
* @return useheaders Empty fields will be trimmed if true
*/
bool trimFields() {
return mTrimFields;
}

/** Return the column names read from the header, or default names
* Col## if none defined. Will open and read the head of the file
* if required, then reset..
Expand Down Expand Up @@ -209,32 +215,29 @@ class QgsDelimitedTextFile
* @return type The delimiter type as a string
*/
QString type();
/** Return the string defining the delimiter (either a regexp pattern
* or a string)
* @return def The delimiter definition string
*/
QString delimiterDefinitionString() {
return mDelimDefinition;
}
/** Return the quote character
* @return quote The quoet character
*/
QString quoteChar() {
return mQuoteChar;
}
/** Return the escape character
* @return escape The escape character
*/
QString escapeChar() {
return mEscapeChar;
}

/** Check that provider is valid (filename and definition valid)
*
* @return valid True if the provider is valid
*/
bool isValid();

/** Encode characters - used to convert delimiter/quote/escape characters to
* encoded form (eg replace tab with \t)
* @param string The unencoded string
* @return encstring The encoded string
*/
static QString encodeChars( QString string );

/** Encode characters - used to encoded character strings to
* decoded form (eg replace \t with tab)
* @param string The unencoded string
* @return decstring The decoded string
*/
static QString decodeChars( QString string );




private:

Expand Down Expand Up @@ -275,8 +278,9 @@ class QgsDelimitedTextFile
// Parameters common to parsers
bool mDefinitionValid;
DelimiterType mType;
QString mDelimDefinition;
bool mUseHeader;
bool mDiscardEmptyFields;
bool mTrimFields;
int mSkipLines;
int mMaxFields;

Expand Down
5 changes: 2 additions & 3 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View file
Expand Up @@ -45,8 +45,8 @@
static const QString TEXT_PROVIDER_KEY = "delimitedtext";
static const QString TEXT_PROVIDER_DESCRIPTION = "Delimited text data provider";

QRegExp QgsDelimitedTextProvider::WktPrefixRegexp("^\\s*(?:\\d+\\s+|SRID\\=\\d+\\;)" );
QRegExp QgsDelimitedTextProvider::WktZMRegexp("\\s+(?:z|m|zm)(?=\\s*\\()", Qt::CaseInsensitive );
QRegExp QgsDelimitedTextProvider::WktPrefixRegexp("^\\s*(?:\\d+\\s+|SRID\\=\\d+\\;)", Qt::CaseInsensitive );
QRegExp QgsDelimitedTextProvider::WktZMRegexp("\\s*(?:z|m|zm)(?=\\s*\\()", Qt::CaseInsensitive );
QRegExp QgsDelimitedTextProvider::WktCrdRegexp("(\\-?\\d+(?:\\.\\d*)?\\s+\\-?\\d+(?:\\.\\d*)?)\\s[\\s\\d\\.\\-]+" );

QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
Expand Down Expand Up @@ -152,7 +152,6 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
QStringList &fieldList = mFile->columnNames();

mFieldCount = fieldList.count();
mFile->setMaxFields(mFieldCount);
fieldsCounted = true;

// We don't know anything about a text based field other
Expand Down

0 comments on commit aac902a

Please sign in to comment.