MVE - Multi-View Environment mve-devel
Loading...
Searching...
No Matches
dmrecon.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 2015, Ronny Klowsky, Simon Fuhrmann
3 * TU Darmstadt - Graphics, Capture and Massively Parallel Computing
4 * All rights reserved.
5 *
6 * This software may be modified and distributed under the terms
7 * of the BSD 3-Clause license. See the LICENSE.txt file for details.
8 */
9
10#include <fstream>
11#include <iostream>
12#include <iomanip>
13#include <stdexcept>
14#include <set>
15#include <ctime>
16
17#include "math/vector.h"
18#include "math/functions.h"
19#include "math/octree_tools.h"
20#include "mve/image.h"
21#include "mve/image_tools.h"
22#include "util/file_system.h"
23#include "util/strings.h"
24#include "dmrecon/settings.h"
25#include "dmrecon/dmrecon.h"
27
29
30DMRecon::DMRecon(mve::Scene::Ptr _scene, Settings const& _settings)
31 : scene(_scene)
32 , settings(_settings)
33{
34 mve::Scene::ViewList const& mve_views(scene->get_views());
35
36 /* Check if master image exists */
37 if (settings.refViewNr >= mve_views.size())
38 throw std::invalid_argument("Master view index out of bounds");
39
40 /* Check for meaningful scale factor */
41 if (settings.scale < 0.f)
42 throw std::invalid_argument("Invalid scale factor");
43
44 /* Check if image embedding is set. */
45 if (settings.imageEmbedding.empty())
46 throw std::invalid_argument("Invalid image embedding");
47
48 // TODO: Implement more sanity checks on the settings!
49
50 /* Fetch bundle file. */
51 try
52 {
53 this->bundle = this->scene->get_bundle();
54 }
55 catch (std::exception& e)
56 {
57 throw std::runtime_error(std::string("Error reading bundle file: ")
58 + e.what());
59 }
60
61 /* Create list of SingleView pointers from MVE views. */
62 views.resize(mve_views.size());
63 for (std::size_t i = 0; i < mve_views.size(); ++i)
64 {
65 if (mve_views[i] == nullptr || !mve_views[i]->is_camera_valid() ||
66 !mve_views[i]->has_image(this->settings.imageEmbedding,
68 continue;
69 views[i] = mvs::SingleView::create(scene, mve_views[i],
70 this->settings.imageEmbedding);
71 }
72
73 SingleView::Ptr refV = views[settings.refViewNr];
74 if (refV == nullptr)
75 throw std::invalid_argument("Invalid master view");
76
77 /* Prepare reconstruction */
78 refV->loadColorImage(this->settings.scale);
79 refV->prepareMasterView(settings.scale);
80 mve::ByteImage::ConstPtr scaled_img = refV->getScaledImg();
81 this->width = scaled_img->width();
82 this->height = scaled_img->height();
83
84 if (!settings.quiet)
85 std::cout << "scaled image size: " << this->width << " x "
86 << this->height << std::endl;
87}
88
89void
91{
92 try
93 {
94 progress.start_time = std::time(nullptr);
95
96 analyzeFeatures();
97 globalViewSelection();
98 processFeatures();
99 processQueue();
100
101 if (progress.cancelled)
102 {
103 progress.status = RECON_CANCELLED;
104 return;
105 }
106
107 progress.status = RECON_SAVING;
108 SingleView::Ptr refV(views[settings.refViewNr]);
109 if (settings.writePlyFile)
110 {
111 if (!settings.quiet)
112 std::cout << "Saving ply file as "
113 << settings.plyPath << "/"
114 << refV->createFileName(settings.scale)
115 << ".ply" << std::endl;
116 refV->saveReconAsPly(settings.plyPath, settings.scale);
117 }
118
119 // Save images to view
120 mve::View::Ptr view = refV->getMVEView();
121
122 std::string name("depth-L");
123 name += util::string::get(settings.scale);
124 view->set_image(refV->depthImg, name);
125
126 if (settings.keepDzMap)
127 {
128 name = "dz-L";
129 name += util::string::get(settings.scale);
130 view->set_image(refV->dzImg, name);
131 }
132
133 if (settings.keepConfidenceMap)
134 {
135 name = "conf-L";
136 name += util::string::get(settings.scale);
137 view->set_image(refV->confImg, name);
138 }
139
140 if (settings.scale != 0)
141 {
142 name = "undist-L";
143 name += util::string::get(settings.scale);
144 view->set_image(refV->getScaledImg()->duplicate(), name);
145 }
146
147 progress.status = RECON_IDLE;
148
149 /* Output percentage of filled pixels */
150 {
151 int nrPix = this->width * this->height;
152 float percent = (float) progress.filled / (float) nrPix;
153 if (!settings.quiet)
154 std::cout << "Filled " << progress.filled << " pixels, i.e. "
155 << util::string::get_fixed(percent * 100.f, 1)
156 << " %." << std::endl;
157 }
158
159 /* Output required time to process the image */
160 size_t mvs_time = std::time(nullptr) - progress.start_time;
161 if (!settings.quiet)
162 std::cout << "MVS took " << mvs_time << " seconds." << std::endl;
163 }
164 catch (util::Exception& e)
165 {
166 if (!settings.quiet)
167 std::cout << "Reconstruction failed: " << e << std::endl;
168
169 progress.status = RECON_CANCELLED;
170 return;
171 }
172}
173
174/*
175 * Attach features that are visible in the reference view (according to
176 * the bundle) to all other views if inside the frustum.
177 */
178void
179DMRecon::analyzeFeatures()
180{
181 progress.status = RECON_FEATURES;
182
183 SingleView::ConstPtr refV = views[settings.refViewNr];
184 mve::Bundle::Features const& features = bundle->get_features();
185 for (std::size_t i = 0; i < features.size() && !progress.cancelled; ++i)
186 {
187 if (!features[i].contains_view_id(settings.refViewNr))
188 continue;
189
190 math::Vec3f featurePos(features[i].pos);
191 if (!refV->pointInFrustum(featurePos))
192 continue;
193
194 if (!math::geom::point_box_overlap(featurePos,
195 this->settings.aabbMin, this->settings.aabbMax))
196 continue;
197
198 for (std::size_t j = 0; j < features[i].refs.size(); ++j)
199 {
200 int view_id = features[i].refs[j].view_id;
201 if (view_id < 0 || view_id >= static_cast<int>(views.size())
202 || views[view_id] == nullptr)
203 continue;
204 if (views[view_id]->pointInFrustum(featurePos))
205 views[view_id]->addFeature(i);
206 }
207 }
208}
209
210void
211DMRecon::globalViewSelection()
212{
213 progress.status = RECON_GLOBALVS;
214 if (progress.cancelled)
215 return;
216
217 /* Perform global view selection. */
218 GlobalViewSelection globalVS(views, bundle->get_features(), settings);
219 globalVS.performVS();
220 neighViews = globalVS.getSelectedIDs();
221
222 if (neighViews.empty())
223 throw std::runtime_error("Global View Selection failed");
224
225 /* Print result of global view selection. */
226 if (!settings.quiet)
227 {
228 std::cout << "Global View Selection:";
229 for (IndexSet::const_iterator iter = neighViews.begin();
230 iter != neighViews.end(); ++iter)
231 std::cout << " " << *iter;
232 std::cout << std::endl;
233 }
234
235 /* Load selected images. */
236 if (!settings.quiet)
237 std::cout << "Loading color images..." << std::endl;
238 for (IndexSet::const_iterator iter = neighViews.begin();
239 iter != neighViews.end() && !progress.cancelled; ++iter)
240 views[*iter]->loadColorImage(0);
241}
242
243void
244DMRecon::processFeatures()
245{
246 progress.status = RECON_FEATURES;
247 if (progress.cancelled)
248 return;
249 SingleView::Ptr refV = views[settings.refViewNr];
250 mve::Bundle::Features const& features = bundle->get_features();
251
252 if (!settings.quiet)
253 std::cout << "Processing " << features.size()
254 << " features..." << std::endl;
255
256 std::size_t success = 0;
257 std::size_t processed = 0;
258 for (std::size_t i = 0; i < features.size() && !progress.cancelled; ++i)
259 {
260 /*
261 * Use feature if visible in reference view or
262 * at least one neighboring view.
263 */
264 bool useFeature = false;
265 if (features[i].contains_view_id(settings.refViewNr))
266 useFeature = true;
267
268 for (IndexSet::const_iterator id = neighViews.begin();
269 useFeature == false && id != neighViews.end(); ++id)
270 {
271 if (features[i].contains_view_id(*id))
272 useFeature = true;
273 }
274 if (!useFeature)
275 continue;
276
277 math::Vec3f featPos(features[i].pos);
278 if (!refV->pointInFrustum(featPos))
279 continue;
280
281 /* Check if feature is inside AABB. */
283 this->settings.aabbMin, this->settings.aabbMax))
284 continue;
285
286 /* Start processing the feature. */
287 processed += 1;
288
289 math::Vec2f pixPosF = refV->worldToScreenScaled(featPos);
290 int const x = math::round(pixPosF[0]);
291 int const y = math::round(pixPosF[1]);
292 float initDepth = (featPos - refV->camPos).norm();
293 PatchOptimization patch(views, settings, x, y, initDepth,
294 0.f, 0.f, neighViews, IndexSet());
295 patch.doAutoOptimization();
296 float conf = patch.computeConfidence();
297 if (conf <= 0.0f)
298 continue;
299
300 /* Feature depth optimization was successful. */
301 success += 1;
302 int const index = y * this->width + x;
303 float depth = patch.getDepth();
304 math::Vec3f normal = patch.getNormal();
305 if (refV->confImg->at(index) < conf)
306 {
307 if (refV->confImg->at(index) <= 0)
308 ++progress.filled;
309
310 refV->depthImg->at(index) = depth;
311 refV->normalImg->at(index, 0) = normal[0];
312 refV->normalImg->at(index, 1) = normal[1];
313 refV->normalImg->at(index, 2) = normal[2];
314 refV->dzImg->at(index, 0) = patch.getDzI();
315 refV->dzImg->at(index, 1) = patch.getDzJ();
316 refV->confImg->at(index) = conf;
317 QueueData tmpData;
318 tmpData.confidence = conf;
319 tmpData.depth = depth;
320 tmpData.dz_i = patch.getDzI();
321 tmpData.dz_j = patch.getDzJ();
322 tmpData.localViewIDs = patch.getLocalViewIDs();
323 tmpData.x = x;
324 tmpData.y = y;
325 prQueue.push(tmpData);
326 }
327 }
328 if (!settings.quiet)
329 std::cout << "Processed " << processed << " features, from which "
330 << success << " succeeded optimization." << std::endl;
331}
332
333void
334DMRecon::processQueue()
335{
336 progress.status = RECON_QUEUE;
337 if (progress.cancelled) return;
338
339 SingleView::Ptr refV = this->views[settings.refViewNr];
340
341 if (!settings.quiet)
342 std::cout << "Process queue ..." << std::endl;
343
344 size_t count = 0, lastStatus = 1;
345 progress.queueSize = prQueue.size();
346 if (!settings.quiet)
347 std::cout << "Count: " << std::setw(8) << count
348 << " filled: " << std::setw(8) << progress.filled
349 << " Queue: " << std::setw(8) << progress.queueSize
350 << std::endl;
351 lastStatus = progress.filled;
352
353 while (!prQueue.empty() && !progress.cancelled)
354 {
355 progress.queueSize = prQueue.size();
356 if ((progress.filled % 1000 == 0) && (progress.filled != lastStatus))
357 {
358 if (!settings.quiet)
359 std::cout << "Count: " << std::setw(8) << count
360 << " filled: " << std::setw(8) << progress.filled
361 << " Queue: " << std::setw(8) << progress.queueSize
362 << std::endl;
363 lastStatus = progress.filled;
364 }
365 QueueData tmpData = prQueue.top();
366 prQueue.pop();
367 ++count;
368 float x = tmpData.x;
369 float y = tmpData.y;
370 int index = y * this->width + x;
371 if (refV->confImg->at(index) > tmpData.confidence) {
372 continue ;
373 }
374 PatchOptimization patch(views, settings, x, y, tmpData.depth,
375 tmpData.dz_i, tmpData.dz_j, neighViews, tmpData.localViewIDs);
376 patch.doAutoOptimization();
377 tmpData.confidence = patch.computeConfidence();
378 if (tmpData.confidence == 0) {
379 continue;
380 }
381
382 float new_depth = patch.getDepth();
383 tmpData.depth = new_depth;
384 tmpData.dz_i = patch.getDzI();
385 tmpData.dz_j = patch.getDzJ();
386 math::Vec3f normal = patch.getNormal();
387 tmpData.localViewIDs = patch.getLocalViewIDs();
388 if (refV->confImg->at(index) <= 0) {
389 ++progress.filled;
390 }
391 if (refV->confImg->at(index) < tmpData.confidence) {
392 refV->depthImg->at(index) = tmpData.depth;
393 refV->normalImg->at(index, 0) = normal[0];
394 refV->normalImg->at(index, 1) = normal[1];
395 refV->normalImg->at(index, 2) = normal[2];
396 refV->dzImg->at(index, 0) = tmpData.dz_i;
397 refV->dzImg->at(index, 1) = tmpData.dz_j;
398 refV->confImg->at(index) = tmpData.confidence;
399
400 // left
401 tmpData.x = x - 1; tmpData.y = y;
402 index = tmpData.y * this->width + tmpData.x;
403 if (refV->confImg->at(index) < tmpData.confidence - 0.05f ||
404 refV->confImg->at(index) == 0.f)
405 {
406 prQueue.push(tmpData);
407 }
408 // right
409 tmpData.x = x + 1; tmpData.y = y;
410 index = tmpData.y * this->width + tmpData.x;
411 if (refV->confImg->at(index) < tmpData.confidence - 0.05f ||
412 refV->confImg->at(index) == 0.f)
413 {
414 prQueue.push(tmpData);
415 }
416 // top
417 tmpData.x = x; tmpData.y = y - 1;
418 index = tmpData.y * this->width + tmpData.x;
419 if (refV->confImg->at(index) < tmpData.confidence - 0.05f ||
420 refV->confImg->at(index) == 0.f)
421 {
422 prQueue.push(tmpData);
423 }
424 // bottom
425 tmpData.x = x; tmpData.y = y + 1;
426 index = tmpData.y * this->width + tmpData.x;
427 if (refV->confImg->at(index) < tmpData.confidence - 0.05f ||
428 refV->confImg->at(index) == 0.f)
429 {
430 prQueue.push(tmpData);
431 }
432 }
433 }
434}
435
Vector class for arbitrary dimensions and types.
Definition vector.h:87
std::vector< Feature3D > Features
Definition bundle.h:62
std::shared_ptr< Image< T > const > ConstPtr
Definition image.h:43
std::shared_ptr< Scene > Ptr
Definition scene.h:37
std::vector< View::Ptr > ViewList
Definition scene.h:38
std::shared_ptr< View > Ptr
Definition view.h:68
void start()
Definition dmrecon.cc:90
static Ptr create(mve::Scene::Ptr scene, mve::View::Ptr view, std::string const &embedding)
Definition single_view.h:94
std::shared_ptr< SingleView const > ConstPtr
Definition single_view.h:32
std::shared_ptr< SingleView > Ptr
Definition single_view.h:31
Universal, simple exception class.
Definition exception.h:24
#define MVS_NAMESPACE_BEGIN
Definition defines.h:18
#define MVS_NAMESPACE_END
Definition defines.h:19
bool point_box_overlap(math::Vector< T, N > const &point, math::Vector< T, N > const &aabb_min, math::Vector< T, N > const &aabb_max)
Returns true if the given point overlaps with the axis-aligned box.
T round(T const &x)
Removes the fractional part of the value to the closest integer.
Definition functions.h:70
@ IMAGE_TYPE_UINT8
Definition image_base.h:31
std::set< std::size_t > IndexSet
Definition defines.h:24
@ RECON_QUEUE
Definition progress.h:22
@ RECON_IDLE
Definition progress.h:19
@ RECON_GLOBALVS
Definition progress.h:20
@ RECON_SAVING
Definition progress.h:23
@ RECON_CANCELLED
Definition progress.h:24
@ RECON_FEATURES
Definition progress.h:21
std::string get_fixed(T const &value, int digits)
Returns string with 'digits' of fixed precision (fills with zeros).
Definition strings.h:124
std::string get(T const &value)
From arbitrary types to string conversion.
Definition strings.h:108
std::size_t queueSize
current size of MVS pixel queue
Definition progress.h:31
std::size_t start_time
start time of MVS reconstruction, or 0
Definition progress.h:32
bool cancelled
set from extern to true to cancel reconstruction
Definition progress.h:33
std::size_t filled
amount of pixels with reconstructed depth value
Definition progress.h:30
ReconStatus status
current status of MVS algorithm
Definition progress.h:29
bool keepDzMap
Definition settings.h:49
bool writePlyFile
Definition settings.h:41
std::string imageEmbedding
Input image emebdding.
Definition settings.h:28
math::Vec3f aabbMin
Features outside the AABB are ignored.
Definition settings.h:44
bool keepConfidenceMap
Definition settings.h:50
std::size_t refViewNr
The reference view ID to reconstruct.
Definition settings.h:25
std::string plyPath
Definition settings.h:47