Skip to content

Commit

Permalink
Support type override
Browse files Browse the repository at this point in the history
  • Loading branch information
@elpaso @nyalldawson
elpaso authored and nyalldawson committed Nov 29, 2021
1 parent c1932e0 commit c2492b6
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 78 deletions.
8 changes: 4 additions & 4 deletions src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View file
Expand Up @@ -66,9 +66,9 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
// Add supported types to enable creating expression fields in field calculator
setNativeTypes( QList< NativeType >()
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer)" ), QStringLiteral( "integer" ), QVariant::Int, 0, 10 )
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer - 64 bit)" ), QStringLiteral( "int8" ), QVariant::LongLong )
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (double)" ), QStringLiteral( "double precision" ), QVariant::Double, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "text" ), QVariant::String, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer - 64 bit)" ), QStringLiteral( "integer64" ), QVariant::LongLong )
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (double)" ), QStringLiteral( "double" ), QVariant::Double, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "string" ), QVariant::String, -1, -1, -1, -1 )

// date type
<< QgsVectorDataProvider::NativeType( tr( "Date" ), QStringLiteral( "date" ), QVariant::Date, -1, -1, -1, -1 )
Expand Down Expand Up @@ -162,7 +162,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
// avoid redundant building indexes if we will be building a subset string,
// in which case indexes will be rebuilt.

scanFile( subset.isEmpty() );
scanFile( subset.isEmpty() && ! flags.testFlag( QgsDataProvider::ReadFlag::SkipGetExtent ) );

if ( ! subset.isEmpty() )
{
Expand Down
2 changes: 1 addition & 1 deletion src/providers/delimitedtext/qgsdelimitedtextprovider.h
View file
Expand Up @@ -133,7 +133,7 @@ class QgsDelimitedTextProvider final: public QgsVectorDataProvider
* \param message Pointer to a string to receive a status message
* \returns A list of field type strings, empty if not found or not valid
*/
QStringList readCsvtFieldTypes( const QString &filename, QString *message = nullptr );
static QStringList readCsvtFieldTypes( const QString &filename, QString *message = nullptr );

static QString providerKey();

Expand Down
214 changes: 141 additions & 73 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp
View file
Expand Up @@ -21,6 +21,7 @@
#include "qgssettings.h"
#include "qgsproviderregistry.h"
#include "qgsgui.h"
#include "qgsapplication.h"

#include <QButtonGroup>
#include <QFile>
Expand Down Expand Up @@ -138,86 +139,15 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()
}

//Build the delimited text URI from the user provided information
const QString datasourceUrl { url( )};

QUrl url = mFile->url();
QUrlQuery query( url );

query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );

if ( cbxPointIsComma->isChecked() )
{
query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
}
if ( cbxXyDms->isChecked() )
{
query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
}

bool haveGeom = true;
if ( geomTypeXY->isChecked() )
{
QString field;
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
{
field = cmbXField->currentText();
query.addQueryItem( QStringLiteral( "xField" ), field );
field = cmbYField->currentText();
query.addQueryItem( QStringLiteral( "yField" ), field );
}
if ( !cmbZField->currentText().isEmpty() )
{
field = cmbZField->currentText();
query.addQueryItem( QStringLiteral( "zField" ), field );
}
if ( !cmbMField->currentText().isEmpty() )
{
field = cmbMField->currentText();
query.addQueryItem( QStringLiteral( "mField" ), field );
}
}
else if ( geomTypeWKT->isChecked() )
{
if ( ! cmbWktField->currentText().isEmpty() )
{
const QString field = cmbWktField->currentText();
query.addQueryItem( QStringLiteral( "wktField" ), field );
}
if ( cmbGeometryType->currentIndex() > 0 )
{
query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
}
}
else
{
haveGeom = false;
query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
}
if ( haveGeom )
{
const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
if ( crs.isValid() )
{
query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
}

}

if ( ! geomTypeNone->isChecked() )
{
query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
}

query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );

url.setQuery( query );
// store the settings
saveSettings();
saveSettingsForFile( mFileWidget->filePath() );


// add the layer to the map
emit addVectorLayer( QString::fromLatin1( url.toEncoded() ), txtLayerName->text() );
emit addVectorLayer( datasourceUrl, txtLayerName->text() );

// clear the file and layer name show something has happened, ready for another file

Expand Down Expand Up @@ -462,6 +392,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
if ( status != QgsDelimitedTextFile::RecordOk ) { mBadRowCount++; continue; }
counter++;


// Look at count of non-blank fields

int nv = values.size();
Expand Down Expand Up @@ -524,6 +455,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
}
}


QStringList fieldList = mFile->fieldNames();

if ( isEmpty.size() < fieldList.size() )
Expand All @@ -537,6 +469,46 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
tblSample->setColumnCount( fieldList.size() );
}

tblSample->insertRow( 0 );
QStringList verticalHeaderLabels;
verticalHeaderLabels.push_back( QString( ) );

for ( int i = 1; i <= tblSample->rowCount(); i++ )
{
verticalHeaderLabels.push_back( QString::number( i ) );
}

tblSample->setVerticalHeaderLabels( verticalHeaderLabels );

// This may be slow on huge files, maybe we need a separate thread
mFields = QgsDelimitedTextProvider(
url(),
QgsDataProvider::ProviderOptions(),
QgsDataProvider::ReadFlag::SkipFeatureCount | QgsDataProvider::ReadFlag::SkipGetExtent )
.fields();

for ( int i = 0; i < tblSample->columnCount(); i++ )
{
QComboBox *typeCombo = new QComboBox( tblSample );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldText.svg" ) ), tr( "Text" ), "string" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldInteger.svg" ) ), tr( "Whole Number" ), "integer" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldFloat.svg" ) ), tr( "Decimal Number" ), "double" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldBool.svg" ) ), tr( "Boolean" ), "bool" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldDate.svg" ) ), tr( "Date" ), "date" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldTime.svg" ) ), tr( "Time" ), "time" );
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldDateTime.svg" ) ), tr( "Date and Time" ), "datetime" );
if ( mFields.lookupField( fieldList[ i ] ) >= 0 )
{
const QString typeName { mFields.field( fieldList[ i ] ).typeName() };
const int idx {typeCombo->findData( typeName )};
if ( idx >= 0 )
{
typeCombo->setCurrentIndex( idx );
}
}
tblSample->setCellWidget( 0, i, typeCombo );
}

tblSample->setHorizontalHeaderLabels( fieldList );
tblSample->resizeColumnsToContents();
tblSample->resizeRowsToContents();
Expand Down Expand Up @@ -792,3 +764,99 @@ void QgsDelimitedTextSourceSelect::showCrsWidget()
crsGeometry->setVisible( !geomTypeNone->isChecked() );
textLabelCrs->setVisible( !geomTypeNone->isChecked() );
}

QString QgsDelimitedTextSourceSelect::url()
{
if ( ! validate() )
{
return QString();
}
QUrl url = mFile->url();
QUrlQuery query( url );

query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );

if ( cbxPointIsComma->isChecked() )
{
query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
}
if ( cbxXyDms->isChecked() )
{
query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
}

bool haveGeom = true;
if ( geomTypeXY->isChecked() )
{
QString field;
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
{
field = cmbXField->currentText();
query.addQueryItem( QStringLiteral( "xField" ), field );
field = cmbYField->currentText();
query.addQueryItem( QStringLiteral( "yField" ), field );
}
if ( !cmbZField->currentText().isEmpty() )
{
field = cmbZField->currentText();
query.addQueryItem( QStringLiteral( "zField" ), field );
}
if ( !cmbMField->currentText().isEmpty() )
{
field = cmbMField->currentText();
query.addQueryItem( QStringLiteral( "mField" ), field );
}
}
else if ( geomTypeWKT->isChecked() )
{
if ( ! cmbWktField->currentText().isEmpty() )
{
const QString field = cmbWktField->currentText();
query.addQueryItem( QStringLiteral( "wktField" ), field );
}
if ( cmbGeometryType->currentIndex() > 0 )
{
query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
}
}
else
{
haveGeom = false;
query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
}
if ( haveGeom )
{
const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
if ( crs.isValid() )
{
query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
}

}

if ( ! geomTypeNone->isChecked() )
{
query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
}

query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );

// Set field types if overridden
for ( int column = 0; column < tblSample->columnCount(); column++ )
{
const QString fieldName { tblSample->horizontalHeaderItem( column )->text() };
const int fieldIdx { mFields.lookupField( fieldName ) };
if ( fieldIdx >= 0 )
{
QComboBox *typeCombo { qobject_cast<QComboBox *>( tblSample->cellWidget( 0, column ) ) };
if ( typeCombo && typeCombo->currentData().toString() != mFields.field( fieldName ).typeName() )
{
query.addQueryItem( QStringLiteral( "field" ), QString( fieldName ).replace( ':', QStringLiteral( "\\:" ) ) + ':' + mFields.field( fieldName ).typeName() );
}
}
}

url.setQuery( query );
return QString::fromLatin1( url.toEncoded() );
}
3 changes: 3 additions & 0 deletions src/providers/delimitedtext/qgsdelimitedtextsourceselect.h
View file
Expand Up @@ -18,6 +18,7 @@
#include <QTextStream>
#include "qgshelp.h"
#include "qgsguiutils.h"
#include "qgsfields.h"
#include "qgsproviderregistry.h"
#include "qgsabstractdatasourcewidget.h"
#include "qgsdelimitedtextfile.h"
Expand Down Expand Up @@ -50,6 +51,7 @@ class QgsDelimitedTextSourceSelect : public QgsAbstractDataSourceWidget, private
std::unique_ptr<QgsDelimitedTextFile> mFile;
int mExampleRowCount = 20;
int mBadRowCount = 0;
QgsFields mFields; // Stores the fields as returned by the provider to determine if their types were overridden
static constexpr int DEFAULT_MAX_FIELDS = 10000;
int mMaxFields = DEFAULT_MAX_FIELDS; // to avoid Denial Of Service (at least in source select). Configurable through /max_fields settings sub-key.
QString mSettingsKey;
Expand All @@ -58,6 +60,7 @@ class QgsDelimitedTextSourceSelect : public QgsAbstractDataSourceWidget, private
QButtonGroup *bgGeomType = nullptr;
void showHelp();
void showCrsWidget();
QString url();

public slots:
void addButtonClicked() override;
Expand Down

0 comments on commit c2492b6

Please sign in to comment.