Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Delimited text detectTypes flag - fixes #18601
[FEATURE] Adds a detectTypes flag to the delimited text provider url. If
set to "no" then type detection is not done and all attributes are
treated as text fields.  Otherwise the original behaviour of
detecting field types is preserved.

[needs-docs] Adds a "Detect field types" check box to the record and
field options section of the delimited text provider GUI.

This addresses (at least partially) issue #18601.  A more complete
solution would be to allow users to set field types.
  • Loading branch information
ccrook authored and nyalldawson committed Jul 19, 2018
1 parent 8f5d968 commit b4f2069
Show file tree
Hide file tree
Showing 6 changed files with 209 additions and 87 deletions.
52 changes: 29 additions & 23 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
Expand Up @@ -120,6 +120,10 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
}
}

mDetectTypes=true;
if ( url.hasQueryItem( QStringLiteral( "detectTypes" ) ) )
mDetectTypes = ! url.queryItemValue( QStringLiteral( "detectTypes" ) ).toLower().startsWith( 'n' );

if ( url.hasQueryItem( QStringLiteral( "decimalPoint" ) ) )
mDecimalPoint = url.queryItemValue( QStringLiteral( "decimalPoint" ) );

Expand Down Expand Up @@ -225,7 +229,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
QgsDebugMsg( QString( "Field type string: %1" ).arg( strTypeList ) );

int pos = 0;
QRegExp reType( "(integer|real|string|date|datetime|time)" );
QRegExp reType( "(integer|real|double|string|date|datetime|time)" );

while ( ( pos = reType.indexIn( strTypeList, pos ) ) != -1 )
{
Expand All @@ -240,10 +244,7 @@ QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( const QString &filenam
// *message=tr("Reading field types from %1").arg(csvtInfo.fileName());
}


return types;


}

void QgsDelimitedTextProvider::resetCachedSubset() const
Expand Down Expand Up @@ -556,6 +557,11 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
couldBeDouble[i] = true;
}

if( ! mDetectTypes )
{
continue;
}

// Now test for still valid possible types for the field
// Types are possible until first record which cannot be parsed

Expand Down Expand Up @@ -603,40 +609,40 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
QString typeName = QStringLiteral( "text" );
if ( i < csvtTypes.size() )
{
if ( csvtTypes[i] == QLatin1String( "integer" ) )
{
fieldType = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( csvtTypes[i] == QLatin1String( "long" ) || csvtTypes[i] == QLatin1String( "longlong" ) || csvtTypes[i] == QLatin1String( "int8" ) )
{
fieldType = QVariant::LongLong; //QVariant doesn't support long
typeName = QStringLiteral( "longlong" );
}
else if ( csvtTypes[i] == QLatin1String( "real" ) || csvtTypes[i] == QLatin1String( "double" ) )
{
fieldType = QVariant::Double;
typeName = QStringLiteral( "double" );
}
typeName = csvtTypes[i];
}
else if ( i < couldBeInt.size() )
else if ( mDetectTypes && i < couldBeInt.size() )
{
if ( couldBeInt[i] )
{
fieldType = QVariant::Int;
typeName = QStringLiteral( "integer" );
}
else if ( couldBeLongLong[i] )
{
fieldType = QVariant::LongLong;
typeName = QStringLiteral( "longlong" );
}
else if ( couldBeDouble[i] )
{
fieldType = QVariant::Double;
typeName = QStringLiteral( "double" );
}
}
if( typeName == QStringLiteral( "integer" ) )
{
fieldType = QVariant::Int;
}
else if ( typeName == QStringLiteral( "longlong" ) )
{
fieldType = QVariant::LongLong;
}
else if( typeName == QStringLiteral( "real" ) || typeName == QStringLiteral( "double" ) )
{
typeName = QStringLiteral( "double" );
fieldType = QVariant::Double;
}
else
{
typeName = QStringLiteral( "text" );
}
attributeFields.append( QgsField( fieldNames[i], fieldType, typeName ) );
}

Expand Down
1 change: 1 addition & 0 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.h
Expand Up @@ -227,6 +227,7 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
QString mWktFieldName;
QString mXFieldName;
QString mYFieldName;
bool mDetectTypes;

mutable int mXFieldIndex = -1;
mutable int mYFieldIndex = -1;
Expand Down
10 changes: 7 additions & 3 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp
Expand Up @@ -145,6 +145,8 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()

QUrl url = mFile->url();

url.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );

if ( cbxPointIsComma->isChecked() )
{
url.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
Expand Down Expand Up @@ -192,9 +194,9 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()

}

if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? "yes" : "no" );
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? "yes" : "no" );
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? "yes" : "no" );
if ( ! geomTypeNone->isChecked() ) url.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
url.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );
url.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral("yes") : QStringLiteral("no") );

// store the settings
saveSettings();
Expand Down Expand Up @@ -284,6 +286,7 @@ void QgsDelimitedTextSourceSelect::loadSettings( const QString &subkey, bool loa

rowCounter->setValue( settings.value( key + "/startFrom", 0 ).toInt() );
cbxUseHeader->setChecked( settings.value( key + "/useHeader", "true" ) != "false" );
cbxDetectTypes->setChecked( settings.value( key + "/detectTypes", "true" ) != "false" );
cbxTrimFields->setChecked( settings.value( key + "/trimFields", "false" ) == "true" );
cbxSkipEmptyFields->setChecked( settings.value( key + "/skipEmptyFields", "false" ) == "true" );
cbxPointIsComma->setChecked( settings.value( key + "/decimalPoint", "." ).toString().contains( ',' ) );
Expand Down Expand Up @@ -329,6 +332,7 @@ void QgsDelimitedTextSourceSelect::saveSettings( const QString &subkey, bool sav
settings.setValue( key + "/delimiterRegexp", txtDelimiterRegexp->text() );
settings.setValue( key + "/startFrom", rowCounter->value() );
settings.setValue( key + "/useHeader", cbxUseHeader->isChecked() ? "true" : "false" );
settings.setValue( key + "/detectTypes", cbxDetectTypes->isChecked() ? "true" : "false" );
settings.setValue( key + "/trimFields", cbxTrimFields->isChecked() ? "true" : "false" );
settings.setValue( key + "/skipEmptyFields", cbxSkipEmptyFields->isChecked() ? "true" : "false" );
settings.setValue( key + "/decimalPoint", cbxPointIsComma->isChecked() ? "," : "." );
Expand Down
77 changes: 17 additions & 60 deletions src/ui/qgsdelimitedtextsourceselectbase.ui
Expand Up @@ -45,16 +45,7 @@
<bool>true</bool>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
Expand Down Expand Up @@ -295,16 +286,7 @@
</widget>
<widget class="QWidget" name="swpDelimOptions">
<layout class="QVBoxLayout" name="verticalLayout_11">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
Expand All @@ -325,16 +307,7 @@
<property name="spacing">
<number>0</number>
</property>
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>2</number>
</property>
<item>
Expand Down Expand Up @@ -593,16 +566,7 @@
</widget>
<widget class="QWidget" name="swpRegexpOptions">
<layout class="QVBoxLayout" name="verticalLayout_12">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
Expand Down Expand Up @@ -811,6 +775,16 @@
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QCheckBox" name="cbxDetectTypes">
<property name="text">
<string>Detect field types</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
</item>
Expand Down Expand Up @@ -900,16 +874,7 @@
</property>
<widget class="QWidget" name="swpGeomXY">
<layout class="QVBoxLayout" name="verticalLayout_8">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
Expand Down Expand Up @@ -1033,16 +998,7 @@
</widget>
<widget class="QWidget" name="swpGeomWKT">
<layout class="QVBoxLayout" name="verticalLayout_6">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<property name="margin">
<number>0</number>
</property>
<item>
Expand Down Expand Up @@ -1332,6 +1288,7 @@
<tabstop>recordOptionsGroupBox</tabstop>
<tabstop>rowCounter</tabstop>
<tabstop>cbxUseHeader</tabstop>
<tabstop>cbxDetectTypes</tabstop>
<tabstop>cbxPointIsComma</tabstop>
<tabstop>cbxTrimFields</tabstop>
<tabstop>cbxSkipEmptyFields</tabstop>
Expand Down
17 changes: 16 additions & 1 deletion tests/src/python/test_qgsdelimitedtextprovider.py
Expand Up @@ -337,7 +337,7 @@ def printWanted(self, testname, result):
print((prefix + ' ' + repr(msg) + ','))
print((prefix + ' ]'))
print(' return wanted')
print()
print('',flush=True)

def recordDifference(self, record1, record2):
# Compare a record defined as a dictionary
Expand Down Expand Up @@ -804,6 +804,21 @@ def test_040_issue_14666(self):
params = {'yField': 'y', 'xField': 'x', 'type': 'csv', 'delimiter': '\\t'}
requests = None
self.runTest(filename, requests, **params)

def test_041_no_detect_type(self):
# CSV file parsing
# Skip lines
filename = 'testtypes.csv'
params = {'yField': 'lat', 'xField': 'lon', 'type': 'csv', 'detectTypes': 'no'}
requests = None
self.runTest(filename, requests, **params)

def test_042_no_detect_types_csvt(self):
# CSVT field types
filename = 'testcsvt.csv'
params = {'geomType': 'none', 'type': 'csv', 'detectTypes': 'no' }
requests = None
self.runTest(filename, requests, **params)


if __name__ == '__main__':
Expand Down

0 comments on commit b4f2069

Please sign in to comment.