Skip to content

Commit c2492b6

Browse files
elpasonyalldawson
elpaso
authored and
nyalldawson
committedNov 29, 2021
Support type override
·
ltr-3_40final-3_24_0
1 parent c1932e0 commit c2492b6

File tree

4 files changed

+149
-78
lines changed

4 files changed

+149
-78
lines changed
 

‎src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
6666
// Add supported types to enable creating expression fields in field calculator
6767
setNativeTypes( QList< NativeType >()
6868
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer)" ), QStringLiteral( "integer" ), QVariant::Int, 0, 10 )
69-
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer - 64 bit)" ), QStringLiteral( "int8" ), QVariant::LongLong )
70-
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (double)" ), QStringLiteral( "double precision" ), QVariant::Double, -1, -1, -1, -1 )
71-
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "text" ), QVariant::String, -1, -1, -1, -1 )
69+
<< QgsVectorDataProvider::NativeType( tr( "Whole number (integer - 64 bit)" ), QStringLiteral( "integer64" ), QVariant::LongLong )
70+
<< QgsVectorDataProvider::NativeType( tr( "Decimal number (double)" ), QStringLiteral( "double" ), QVariant::Double, -1, -1, -1, -1 )
71+
<< QgsVectorDataProvider::NativeType( tr( "Text, unlimited length (text)" ), QStringLiteral( "string" ), QVariant::String, -1, -1, -1, -1 )
7272

7373
// date type
7474
<< QgsVectorDataProvider::NativeType( tr( "Date" ), QStringLiteral( "date" ), QVariant::Date, -1, -1, -1, -1 )
@@ -162,7 +162,7 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr
162162
// avoid redundant building indexes if we will be building a subset string,
163163
// in which case indexes will be rebuilt.
164164

165-
scanFile( subset.isEmpty() );
165+
scanFile( subset.isEmpty() && ! flags.testFlag( QgsDataProvider::ReadFlag::SkipGetExtent ) );
166166

167167
if ( ! subset.isEmpty() )
168168
{

‎src/providers/delimitedtext/qgsdelimitedtextprovider.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ class QgsDelimitedTextProvider final: public QgsVectorDataProvider
133133
* \param message Pointer to a string to receive a status message
134134
* \returns A list of field type strings, empty if not found or not valid
135135
*/
136-
QStringList readCsvtFieldTypes( const QString &filename, QString *message = nullptr );
136+
static QStringList readCsvtFieldTypes( const QString &filename, QString *message = nullptr );
137137

138138
static QString providerKey();
139139

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp

Lines changed: 141 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "qgssettings.h"
2222
#include "qgsproviderregistry.h"
2323
#include "qgsgui.h"
24+
#include "qgsapplication.h"
2425

2526
#include <QButtonGroup>
2627
#include <QFile>
@@ -138,86 +139,15 @@ void QgsDelimitedTextSourceSelect::addButtonClicked()
138139
}
139140

140141
//Build the delimited text URI from the user provided information
142+
const QString datasourceUrl { url( )};
141143

142-
QUrl url = mFile->url();
143-
QUrlQuery query( url );
144-
145-
query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
146-
147-
if ( cbxPointIsComma->isChecked() )
148-
{
149-
query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
150-
}
151-
if ( cbxXyDms->isChecked() )
152-
{
153-
query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
154-
}
155-
156-
bool haveGeom = true;
157-
if ( geomTypeXY->isChecked() )
158-
{
159-
QString field;
160-
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
161-
{
162-
field = cmbXField->currentText();
163-
query.addQueryItem( QStringLiteral( "xField" ), field );
164-
field = cmbYField->currentText();
165-
query.addQueryItem( QStringLiteral( "yField" ), field );
166-
}
167-
if ( !cmbZField->currentText().isEmpty() )
168-
{
169-
field = cmbZField->currentText();
170-
query.addQueryItem( QStringLiteral( "zField" ), field );
171-
}
172-
if ( !cmbMField->currentText().isEmpty() )
173-
{
174-
field = cmbMField->currentText();
175-
query.addQueryItem( QStringLiteral( "mField" ), field );
176-
}
177-
}
178-
else if ( geomTypeWKT->isChecked() )
179-
{
180-
if ( ! cmbWktField->currentText().isEmpty() )
181-
{
182-
const QString field = cmbWktField->currentText();
183-
query.addQueryItem( QStringLiteral( "wktField" ), field );
184-
}
185-
if ( cmbGeometryType->currentIndex() > 0 )
186-
{
187-
query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
188-
}
189-
}
190-
else
191-
{
192-
haveGeom = false;
193-
query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
194-
}
195-
if ( haveGeom )
196-
{
197-
const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
198-
if ( crs.isValid() )
199-
{
200-
query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
201-
}
202-
203-
}
204-
205-
if ( ! geomTypeNone->isChecked() )
206-
{
207-
query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
208-
}
209-
210-
query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
211-
query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
212-
213-
url.setQuery( query );
214144
// store the settings
215145
saveSettings();
216146
saveSettingsForFile( mFileWidget->filePath() );
217147

218148

219149
// add the layer to the map
220-
emit addVectorLayer( QString::fromLatin1( url.toEncoded() ), txtLayerName->text() );
150+
emit addVectorLayer( datasourceUrl, txtLayerName->text() );
221151

222152
// clear the file and layer name show something has happened, ready for another file
223153

@@ -462,6 +392,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
462392
if ( status != QgsDelimitedTextFile::RecordOk ) { mBadRowCount++; continue; }
463393
counter++;
464394

395+
465396
// Look at count of non-blank fields
466397

467398
int nv = values.size();
@@ -524,6 +455,7 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
524455
}
525456
}
526457

458+
527459
QStringList fieldList = mFile->fieldNames();
528460

529461
if ( isEmpty.size() < fieldList.size() )
@@ -537,6 +469,46 @@ void QgsDelimitedTextSourceSelect::updateFieldLists()
537469
tblSample->setColumnCount( fieldList.size() );
538470
}
539471

472+
tblSample->insertRow( 0 );
473+
QStringList verticalHeaderLabels;
474+
verticalHeaderLabels.push_back( QString( ) );
475+
476+
for ( int i = 1; i <= tblSample->rowCount(); i++ )
477+
{
478+
verticalHeaderLabels.push_back( QString::number( i ) );
479+
}
480+
481+
tblSample->setVerticalHeaderLabels( verticalHeaderLabels );
482+
483+
// This may be slow on huge files, maybe we need a separate thread
484+
mFields = QgsDelimitedTextProvider(
485+
url(),
486+
QgsDataProvider::ProviderOptions(),
487+
QgsDataProvider::ReadFlag::SkipFeatureCount | QgsDataProvider::ReadFlag::SkipGetExtent )
488+
.fields();
489+
490+
for ( int i = 0; i < tblSample->columnCount(); i++ )
491+
{
492+
QComboBox *typeCombo = new QComboBox( tblSample );
493+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldText.svg" ) ), tr( "Text" ), "string" );
494+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldInteger.svg" ) ), tr( "Whole Number" ), "integer" );
495+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldFloat.svg" ) ), tr( "Decimal Number" ), "double" );
496+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldBool.svg" ) ), tr( "Boolean" ), "bool" );
497+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldDate.svg" ) ), tr( "Date" ), "date" );
498+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldTime.svg" ) ), tr( "Time" ), "time" );
499+
typeCombo->addItem( QgsApplication::getThemeIcon( QStringLiteral( "/mIconFieldDateTime.svg" ) ), tr( "Date and Time" ), "datetime" );
500+
if ( mFields.lookupField( fieldList[ i ] ) >= 0 )
501+
{
502+
const QString typeName { mFields.field( fieldList[ i ] ).typeName() };
503+
const int idx {typeCombo->findData( typeName )};
504+
if ( idx >= 0 )
505+
{
506+
typeCombo->setCurrentIndex( idx );
507+
}
508+
}
509+
tblSample->setCellWidget( 0, i, typeCombo );
510+
}
511+
540512
tblSample->setHorizontalHeaderLabels( fieldList );
541513
tblSample->resizeColumnsToContents();
542514
tblSample->resizeRowsToContents();
@@ -792,3 +764,99 @@ void QgsDelimitedTextSourceSelect::showCrsWidget()
792764
crsGeometry->setVisible( !geomTypeNone->isChecked() );
793765
textLabelCrs->setVisible( !geomTypeNone->isChecked() );
794766
}
767+
768+
QString QgsDelimitedTextSourceSelect::url()
769+
{
770+
if ( ! validate() )
771+
{
772+
return QString();
773+
}
774+
QUrl url = mFile->url();
775+
QUrlQuery query( url );
776+
777+
query.addQueryItem( QStringLiteral( "detectTypes" ), cbxDetectTypes->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
778+
779+
if ( cbxPointIsComma->isChecked() )
780+
{
781+
query.addQueryItem( QStringLiteral( "decimalPoint" ), QStringLiteral( "," ) );
782+
}
783+
if ( cbxXyDms->isChecked() )
784+
{
785+
query.addQueryItem( QStringLiteral( "xyDms" ), QStringLiteral( "yes" ) );
786+
}
787+
788+
bool haveGeom = true;
789+
if ( geomTypeXY->isChecked() )
790+
{
791+
QString field;
792+
if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() )
793+
{
794+
field = cmbXField->currentText();
795+
query.addQueryItem( QStringLiteral( "xField" ), field );
796+
field = cmbYField->currentText();
797+
query.addQueryItem( QStringLiteral( "yField" ), field );
798+
}
799+
if ( !cmbZField->currentText().isEmpty() )
800+
{
801+
field = cmbZField->currentText();
802+
query.addQueryItem( QStringLiteral( "zField" ), field );
803+
}
804+
if ( !cmbMField->currentText().isEmpty() )
805+
{
806+
field = cmbMField->currentText();
807+
query.addQueryItem( QStringLiteral( "mField" ), field );
808+
}
809+
}
810+
else if ( geomTypeWKT->isChecked() )
811+
{
812+
if ( ! cmbWktField->currentText().isEmpty() )
813+
{
814+
const QString field = cmbWktField->currentText();
815+
query.addQueryItem( QStringLiteral( "wktField" ), field );
816+
}
817+
if ( cmbGeometryType->currentIndex() > 0 )
818+
{
819+
query.addQueryItem( QStringLiteral( "geomType" ), cmbGeometryType->currentText() );
820+
}
821+
}
822+
else
823+
{
824+
haveGeom = false;
825+
query.addQueryItem( QStringLiteral( "geomType" ), QStringLiteral( "none" ) );
826+
}
827+
if ( haveGeom )
828+
{
829+
const QgsCoordinateReferenceSystem crs = crsGeometry->crs();
830+
if ( crs.isValid() )
831+
{
832+
query.addQueryItem( QStringLiteral( "crs" ), crs.authid() );
833+
}
834+
835+
}
836+
837+
if ( ! geomTypeNone->isChecked() )
838+
{
839+
query.addQueryItem( QStringLiteral( "spatialIndex" ), cbxSpatialIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
840+
}
841+
842+
query.addQueryItem( QStringLiteral( "subsetIndex" ), cbxSubsetIndex->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
843+
query.addQueryItem( QStringLiteral( "watchFile" ), cbxWatchFile->isChecked() ? QStringLiteral( "yes" ) : QStringLiteral( "no" ) );
844+
845+
// Set field types if overridden
846+
for ( int column = 0; column < tblSample->columnCount(); column++ )
847+
{
848+
const QString fieldName { tblSample->horizontalHeaderItem( column )->text() };
849+
const int fieldIdx { mFields.lookupField( fieldName ) };
850+
if ( fieldIdx >= 0 )
851+
{
852+
QComboBox *typeCombo { qobject_cast<QComboBox *>( tblSample->cellWidget( 0, column ) ) };
853+
if ( typeCombo && typeCombo->currentData().toString() != mFields.field( fieldName ).typeName() )
854+
{
855+
query.addQueryItem( QStringLiteral( "field" ), QString( fieldName ).replace( ':', QStringLiteral( "\\:" ) ) + ':' + mFields.field( fieldName ).typeName() );
856+
}
857+
}
858+
}
859+
860+
url.setQuery( query );
861+
return QString::fromLatin1( url.toEncoded() );
862+
}

‎src/providers/delimitedtext/qgsdelimitedtextsourceselect.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <QTextStream>
1919
#include "qgshelp.h"
2020
#include "qgsguiutils.h"
21+
#include "qgsfields.h"
2122
#include "qgsproviderregistry.h"
2223
#include "qgsabstractdatasourcewidget.h"
2324
#include "qgsdelimitedtextfile.h"
@@ -50,6 +51,7 @@ class QgsDelimitedTextSourceSelect : public QgsAbstractDataSourceWidget, private
5051
std::unique_ptr<QgsDelimitedTextFile> mFile;
5152
int mExampleRowCount = 20;
5253
int mBadRowCount = 0;
54+
QgsFields mFields; // Stores the fields as returned by the provider to determine if their types were overridden
5355
static constexpr int DEFAULT_MAX_FIELDS = 10000;
5456
int mMaxFields = DEFAULT_MAX_FIELDS; // to avoid Denial Of Service (at least in source select). Configurable through /max_fields settings sub-key.
5557
QString mSettingsKey;
@@ -58,6 +60,7 @@ class QgsDelimitedTextSourceSelect : public QgsAbstractDataSourceWidget, private
5860
QButtonGroup *bgGeomType = nullptr;
5961
void showHelp();
6062
void showCrsWidget();
63+
QString url();
6164

6265
public slots:
6366
void addButtonClicked() override;

0 commit comments

Comments
 (0)
Please sign in to comment.