Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[untwine] Fix precision issues when indexing
Offset and scale of X/Y/Z dimensions were not written and
some datasets would simply fail to get indexed completely.
Other datasets may have had the data slightly misaligned
after indexing.

References:
hobuinc/untwine#28
hobuinc/untwine#46
  • Loading branch information
wonder-sk authored and nyalldawson committed Jan 28, 2021
1 parent d3f0f3d commit 70dd223
Show file tree
Hide file tree
Showing 12 changed files with 114 additions and 165 deletions.
91 changes: 12 additions & 79 deletions external/untwine/bu/BuPyramid.cpp
Expand Up @@ -8,7 +8,6 @@
#include <pdal/util/ProgramArgs.hpp>

#include "BuPyramid.hpp"
#include "BuTypes.hpp"
#include "FileInfo.hpp"
#include "OctantInfo.hpp"
#include "../untwine/Common.hpp"
Expand All @@ -21,7 +20,7 @@ namespace bu

/// BuPyramid

BuPyramid::BuPyramid() : m_manager(m_b)
BuPyramid::BuPyramid(BaseInfo& common) : m_b(common), m_manager(m_b)
{}


Expand All @@ -31,7 +30,6 @@ void BuPyramid::run(const Options& options, ProgressWriter& progress)
m_b.outputDir = options.outputDir;
m_b.stats = options.stats;

readBaseInfo();
getInputFiles();
size_t count = queueWork();

Expand All @@ -44,76 +42,6 @@ void BuPyramid::run(const Options& options, ProgressWriter& progress)
}


void BuPyramid::readBaseInfo()
{
auto nextblock = [](std::istream& in)
{
std::string s;
bool firstnl = false;

while (in)
{
char c;
in.get(c);
if (c == '\n')
{
if (firstnl)
{
// Remove trailing newline.
s.resize(s.size() - 1);
return s;
}
else
firstnl = true;
}
else
firstnl = false;
s += c;
}
return s;
};

std::string baseFilename = m_b.inputDir + "/" + MetadataFilename;
std::ifstream in(baseFilename);

if (!in)
fatal("Can't open '" + MetadataFilename + "' in directory '" + m_b.inputDir + "'.");

std::stringstream ss(nextblock(in));
ss >> m_b.bounds.minx >> m_b.bounds.miny >> m_b.bounds.minz;
ss >> m_b.bounds.maxx >> m_b.bounds.maxy >> m_b.bounds.maxz;

ss.str(nextblock(in));
ss.clear();
ss >> m_b.trueBounds.minx >> m_b.trueBounds.miny >> m_b.trueBounds.minz;
ss >> m_b.trueBounds.maxx >> m_b.trueBounds.maxy >> m_b.trueBounds.maxz;

std::string srs = nextblock(in);
if (srs != "NONE")
m_b.srs.set(srs);

if (!in)
throw "Couldn't read info file.";

ss.str(nextblock(in));
ss.clear();
m_b.pointSize = 0;
while (true)
{
FileDimInfo fdi;
ss >> fdi;
if (!ss)
break;
if (fdi.name.empty())
fatal("Invalid dimension in info.txt.");
m_b.pointSize += pdal::Dimension::size(fdi.type);
m_b.dimInfo.push_back(fdi);
}
if (m_b.pointSize == 0)
throw "Couldn't read info file.";
}


void BuPyramid::writeInfo()
{
auto typeString = [](pdal::Dimension::BaseType b)
Expand All @@ -138,9 +66,9 @@ void BuPyramid::writeInfo()
out << "{\n";

pdal::BOX3D& b = m_b.bounds;
std::ios init(NULL);
init.copyfmt(out);
out << std::fixed << std::setw(12);

// Set fixed output for bounds output to get sufficient precision.
out << std::fixed;
out << "\"bounds\": [" <<
b.minx << ", " << b.miny << ", " << b.minz << ", " <<
b.maxx << ", " << b.maxy << ", " << b.maxz << "],\n";
Expand All @@ -149,6 +77,8 @@ void BuPyramid::writeInfo()
out << "\"boundsConforming\": [" <<
tb.minx << ", " << tb.miny << ", " << tb.minz << ", " <<
tb.maxx << ", " << tb.maxy << ", " << tb.maxz << "],\n";
// Reset to default float output to match PDAL option handling for now.
out << std::defaultfloat;

out << "\"dataType\": \"laszip\",\n";
out << "\"hierarchyType\": \"json\",\n";
Expand All @@ -163,8 +93,12 @@ void BuPyramid::writeInfo()
out << "\t{";
out << "\"name\": \"" << fdi.name << "\", ";
out << "\"type\": \"" << typeString(pdal::Dimension::base(fdi.type)) << "\", ";
if (fdi.name == "X" || fdi.name == "Y" || fdi.name == "Z")
out << "\"scale\": 0.01, \"offset\": 0, ";
if (fdi.name == "X")
out << "\"scale\": " << m_b.scale[0] << ", \"offset\": " << m_b.offset[0] << ", ";
if (fdi.name == "Y")
out << "\"scale\": " << m_b.scale[1] << ", \"offset\": " << m_b.offset[1] << ", ";
if (fdi.name == "Z")
out << "\"scale\": " << m_b.scale[2] << ", \"offset\": " << m_b.offset[2] << ", ";
out << "\"size\": " << pdal::Dimension::size(fdi.type);
const Stats *stats = m_manager.stats(fdi.name);
if (stats)
Expand Down Expand Up @@ -205,7 +139,6 @@ void BuPyramid::writeInfo()
out << "}\n";

out << "}\n";
out.copyfmt(init);
}


Expand Down
5 changes: 2 additions & 3 deletions external/untwine/bu/BuPyramid.hpp
Expand Up @@ -18,7 +18,6 @@
#include <vector>

#include "PyramidManager.hpp"
#include "BuTypes.hpp"

namespace pdal
{
Expand All @@ -39,7 +38,7 @@ class FileInfo;
class BuPyramid
{
public:
BuPyramid();
BuPyramid(BaseInfo& common);
void run(const Options& options, ProgressWriter& progress);

private:
Expand All @@ -48,8 +47,8 @@ class BuPyramid
size_t queueWork();
void writeInfo();

PyramidManager m_manager;
BaseInfo m_b;
PyramidManager m_manager;
std::unordered_map<VoxelKey, FileInfo> m_allFiles;
};

Expand Down
42 changes: 0 additions & 42 deletions external/untwine/bu/BuTypes.hpp

This file was deleted.

1 change: 0 additions & 1 deletion external/untwine/bu/PointAccessor.hpp
Expand Up @@ -16,7 +16,6 @@
#include "../untwine/MapFile.hpp"
#include "../untwine/Point.hpp"

#include "BuTypes.hpp"
#include "FileInfo.hpp"

namespace untwine
Expand Down
7 changes: 6 additions & 1 deletion external/untwine/bu/Processor.cpp
Expand Up @@ -368,7 +368,6 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim
}
}


void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr view,
const OctantInfo& oi, IndexedStats& stats)
{
Expand Down Expand Up @@ -412,6 +411,12 @@ void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr vi
wopts.add("software_id", "Entwine 1.0");
wopts.add("compression", "laszip");
wopts.add("filename", filename);
wopts.add("offset_x", m_b.offset[0]);
wopts.add("offset_y", m_b.offset[1]);
wopts.add("offset_z", m_b.offset[2]);
wopts.add("scale_x", m_b.scale[0]);
wopts.add("scale_y", m_b.scale[1]);
wopts.add("scale_z", m_b.scale[2]);
w->setOptions(wopts);
w->setInput(*prev);
// Set dataformat ID based on time/rgb, but for now accept the default.
Expand Down
1 change: 0 additions & 1 deletion external/untwine/bu/Processor.hpp
Expand Up @@ -15,7 +15,6 @@
#include <pdal/PointTable.hpp>
#include <pdal/PointView.hpp>

#include "BuTypes.hpp"
#include "PointAccessor.hpp"
#include "Stats.hpp"
#include "VoxelInfo.hpp"
Expand Down
1 change: 0 additions & 1 deletion external/untwine/bu/PyramidManager.hpp
Expand Up @@ -17,7 +17,6 @@
#include <unordered_map>
#include <vector>

#include "BuTypes.hpp"
#include "OctantInfo.hpp"
#include "Stats.hpp"
#include "../untwine/ThreadPool.hpp"
Expand Down
6 changes: 5 additions & 1 deletion external/untwine/bu/VoxelInfo.hpp
Expand Up @@ -123,7 +123,8 @@ class VoxelInfo
double y = p.y() - m_bounds.miny;
double z = p.z() - m_bounds.minz;

return GridKey((int)(x / m_gridCellWidth), (int)(y / m_gridCellWidth), (int)(z / m_gridCellWidth));
return GridKey((int)(x / m_gridCellWidth), (int)(y / m_gridCellWidth),
(int)(z / m_gridCellWidth));
}

//ABELL - Really torn WRT making Grid its own thing.
Expand All @@ -140,6 +141,9 @@ class VoxelInfo
int gridZCount() const
{ return m_gridZCount; }

pdal::BOX3D bounds() const
{ return m_bounds; }

private:
pdal::BOX3D m_fullBounds;
pdal::BOX3D m_bounds;
Expand Down

0 comments on commit 70dd223

Please sign in to comment.