Skip to content

Commit ebd27cb

Browse files
committed
FlexGlue: share one matcher across multiple worker threads per GPU
1 parent e1317dd commit ebd27cb

9 files changed

Lines changed: 332 additions & 33 deletions

File tree

‎doc/features.rst‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,10 @@ models run on a CUDA GPU. On CUDA GPUs with compute capability 7.0 or newer, a
217217
variant of the matcher whose transformer layers run in float16 is used by
218218
default (``--FlexGlueMatching.use_fp16``), which is faster than the float32
219219
matcher with practically identical matches; the float32 matcher is used on the CPU, on
220-
older GPUs, or if the float16 variant fails to initialize. On the CPU, a
220+
older GPUs, or if the float16 variant fails to initialize. On CUDA GPUs,
221+
``--FlexGlueMatching.num_workers_per_gpu`` (default 3) matching threads share a
222+
single matcher instance and GPU feature cache per GPU so CPU image decoding
223+
overlaps with GPU inference (lower it to reduce GPU memory usage). On the CPU, a
221224
single matching thread runs the models with ``--FeatureMatching.num_threads``
222225
threads, rather than one copy of the models per thread. The
223226
``automatic_reconstructor`` provides the ``--feature flexglue`` preset, which

‎src/colmap/controllers/feature_matching_utils.cc‎

Lines changed: 72 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "colmap/controllers/feature_matching_utils.h"
44

55
#include "colmap/estimators/two_view_geometry.h"
6+
#include "colmap/feature/flexglue.h"
67
#include "colmap/feature/sift.h"
78
#include "colmap/feature/utils.h"
89
#include "colmap/util/cuda.h"
@@ -11,17 +12,49 @@
1112

1213
namespace colmap {
1314

15+
std::shared_ptr<FeatureMatcher> SharedFeatureMatcher::Create(
16+
const FeatureMatchingOptions& options) {
17+
std::shared_ptr<FeatureMatcher> matcher;
18+
try {
19+
matcher = FeatureMatcher::Create(options);
20+
} catch (...) {
21+
{
22+
std::lock_guard<std::mutex> lock(mutex_);
23+
done_ = true;
24+
}
25+
cv_.notify_all();
26+
throw;
27+
}
28+
{
29+
std::lock_guard<std::mutex> lock(mutex_);
30+
matcher_ = matcher;
31+
done_ = true;
32+
}
33+
cv_.notify_all();
34+
return matcher;
35+
}
36+
37+
std::shared_ptr<FeatureMatcher> SharedFeatureMatcher::Wait() {
38+
std::unique_lock<std::mutex> lock(mutex_);
39+
cv_.wait(lock, [this]() { return done_; });
40+
return matcher_;
41+
}
42+
1443
FeatureMatcherWorker::FeatureMatcherWorker(
1544
const FeatureMatchingOptions& matching_options,
1645
const TwoViewGeometryOptions& geometry_options,
1746
const std::shared_ptr<FeatureMatcherCache>& cache,
1847
JobQueue<Input>* input_queue,
19-
JobQueue<Output>* output_queue)
48+
JobQueue<Output>* output_queue,
49+
std::shared_ptr<SharedFeatureMatcher> shared_matcher,
50+
const bool is_shared_creator)
2051
: matching_options_(matching_options),
2152
geometry_options_(geometry_options),
2253
cache_(cache),
2354
input_queue_(input_queue),
24-
output_queue_(output_queue) {
55+
output_queue_(output_queue),
56+
shared_matcher_(std::move(shared_matcher)),
57+
is_shared_creator_(is_shared_creator) {
2558
THROW_CHECK(matching_options_.Check());
2659

2760
if (matching_options_.RequiresOpenGL()) {
@@ -63,10 +96,18 @@ void FeatureMatcherWorker::Run() {
6396
matching_options_.max_num_matches = std::min<int>(
6497
matching_options_.max_num_matches, cache_->MaxNumKeypoints());
6598

66-
std::unique_ptr<FeatureMatcher> matcher =
67-
FeatureMatcher::Create(matching_options_);
99+
std::shared_ptr<FeatureMatcher> matcher;
100+
if (shared_matcher_ == nullptr) {
101+
matcher = FeatureMatcher::Create(matching_options_);
102+
} else if (is_shared_creator_) {
103+
matcher = shared_matcher_->Create(matching_options_);
104+
} else {
105+
matcher = shared_matcher_->Wait();
106+
}
68107
if (matcher == nullptr) {
69-
LOG(ERROR) << "Failed to create feature matcher.";
108+
if (shared_matcher_ == nullptr || is_shared_creator_) {
109+
LOG(ERROR) << "Failed to create feature matcher.";
110+
}
70111
SignalInvalidSetup();
71112
return;
72113
}
@@ -321,15 +362,34 @@ FeatureMatcherController::FeatureMatcherController(
321362
auto worker_matching_options = matching_options_;
322363
// The first matching is always without guided matching.
323364
worker_matching_options.guided_matching = false;
324-
matchers_.reserve(gpu_indices.size());
365+
int num_workers_per_gpu = 1;
366+
#ifdef COLMAP_CUDA_ENABLED
367+
if (matching_options_.type == FeatureMatcherType::FLEXGLUE) {
368+
num_workers_per_gpu =
369+
THROW_CHECK_NOTNULL(matching_options_.flexglue)->num_workers_per_gpu;
370+
worker_matching_options.num_threads = std::max(
371+
1,
372+
num_threads /
373+
(static_cast<int>(gpu_indices.size()) * num_workers_per_gpu));
374+
}
375+
#endif
376+
matchers_.reserve(gpu_indices.size() * num_workers_per_gpu);
325377
for (const auto& gpu_index : gpu_indices) {
326378
worker_matching_options.gpu_index = std::to_string(gpu_index);
327-
matchers_.emplace_back(
328-
std::make_unique<FeatureMatcherWorker>(worker_matching_options,
329-
geometry_options_,
330-
cache_,
331-
&matcher_queue_,
332-
matcher_output_queue));
379+
std::shared_ptr<SharedFeatureMatcher> shared_matcher;
380+
if (num_workers_per_gpu > 1) {
381+
shared_matcher = std::make_shared<SharedFeatureMatcher>();
382+
}
383+
for (int w = 0; w < num_workers_per_gpu; ++w) {
384+
matchers_.emplace_back(std::make_unique<FeatureMatcherWorker>(
385+
worker_matching_options,
386+
geometry_options_,
387+
cache_,
388+
&matcher_queue_,
389+
matcher_output_queue,
390+
shared_matcher,
391+
/*is_shared_creator=*/w == 0));
392+
}
333393
}
334394
} else {
335395
auto worker_matching_options = matching_options_;

‎src/colmap/controllers/feature_matching_utils.h‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
#include "colmap/util/opengl_utils.h"
99
#include "colmap/util/threading.h"
1010

11+
#include <condition_variable>
1112
#include <exception>
1213
#include <memory>
14+
#include <mutex>
1315
#include <vector>
1416

1517
namespace colmap {
@@ -25,16 +27,36 @@ struct FeatureMatcherData {
2527
std::exception_ptr error;
2628
};
2729

30+
// Synchronizes creation of a thread-safe FeatureMatcher shared across multiple
31+
// FeatureMatcherWorker threads (e.g., multiple workers per GPU for FlexGlue).
32+
// One designated creator worker calls Create() (propagating any exception on
33+
// that worker alone so Setup() rethrows once), while sibling workers call
34+
// Wait() and receive the shared instance (or nullptr on failure).
35+
class SharedFeatureMatcher {
36+
public:
37+
std::shared_ptr<FeatureMatcher> Create(const FeatureMatchingOptions& options);
38+
std::shared_ptr<FeatureMatcher> Wait();
39+
40+
private:
41+
std::mutex mutex_;
42+
std::condition_variable cv_;
43+
bool done_ = false;
44+
std::shared_ptr<FeatureMatcher> matcher_;
45+
};
46+
2847
class FeatureMatcherWorker : public Thread {
2948
public:
3049
using Input = FeatureMatcherData;
3150
using Output = FeatureMatcherData;
3251

33-
FeatureMatcherWorker(const FeatureMatchingOptions& matching_options,
34-
const TwoViewGeometryOptions& geometry_options,
35-
const std::shared_ptr<FeatureMatcherCache>& cache,
36-
JobQueue<Input>* input_queue,
37-
JobQueue<Output>* output_queue);
52+
FeatureMatcherWorker(
53+
const FeatureMatchingOptions& matching_options,
54+
const TwoViewGeometryOptions& geometry_options,
55+
const std::shared_ptr<FeatureMatcherCache>& cache,
56+
JobQueue<Input>* input_queue,
57+
JobQueue<Output>* output_queue,
58+
std::shared_ptr<SharedFeatureMatcher> shared_matcher = nullptr,
59+
bool is_shared_creator = false);
3860

3961
private:
4062
void Run() override;
@@ -48,6 +70,8 @@ class FeatureMatcherWorker : public Thread {
4870
std::shared_ptr<FeatureMatcherCache> cache_;
4971
JobQueue<Input>* input_queue_;
5072
JobQueue<Output>* output_queue_;
73+
std::shared_ptr<SharedFeatureMatcher> shared_matcher_;
74+
bool is_shared_creator_;
5175

5276
std::unique_ptr<OpenGLContextManager> opengl_context_;
5377
};

‎src/colmap/controllers/feature_matching_utils_test.cc‎

Lines changed: 130 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,12 @@
44

55
#include "colmap/controllers/matcher_cache.h"
66
#include "colmap/feature/aliked.h"
7+
#include "colmap/feature/flexglue.h"
8+
#include "colmap/feature/sift.h"
9+
#include "colmap/feature/utils.h"
710
#include "colmap/scene/synthetic.h"
811
#include "colmap/sensor/models.h"
12+
#include "colmap/util/cuda.h"
913
#include "colmap/util/testing.h"
1014

1115
#include <functional>
@@ -333,19 +337,140 @@ TEST(FeatureMatcherController, SetupPropagatesMatcherCreationErrors) {
333337
matching_options.aliked->lightglue.model_path =
334338
(data.test_dir / "missing_model.onnx").string();
335339
TwoViewGeometryOptions geometry_options;
340+
#ifdef COLMAP_ONNX_ENABLED
341+
const std::string expected_message = "missing_model.onnx";
342+
#else
343+
const std::string expected_message = "requires ONNX support";
344+
#endif
336345
{
337346
FeatureMatcherController controller(
338347
matching_options, geometry_options, data.cache);
339348
// The worker fails to create the matcher. Setup rethrows its error, and
340349
// the controller then destructs without terminating.
341-
#ifdef COLMAP_ONNX_ENABLED
342-
const std::string expected_message = "missing_model.onnx";
343-
#else
344-
const std::string expected_message = "requires ONNX support";
345-
#endif
346350
EXPECT_THAT(ThrownMessage([&]() { controller.Setup(); }),
347351
testing::HasSubstr(expected_message));
348352
}
353+
354+
// Also test SharedFeatureMatcher error propagation across multiple sibling
355+
// workers: only the creator worker rethrows on Wait(), while sibling workers
356+
// signal invalid setup and join without throwing.
357+
{
358+
JobQueue<FeatureMatcherData> input_queue;
359+
JobQueue<FeatureMatcherData> output_queue;
360+
auto shared_matcher = std::make_shared<SharedFeatureMatcher>();
361+
std::vector<std::unique_ptr<FeatureMatcherWorker>> workers;
362+
workers.reserve(3);
363+
for (int w = 0; w < 3; ++w) {
364+
workers.push_back(
365+
std::make_unique<FeatureMatcherWorker>(matching_options,
366+
geometry_options,
367+
data.cache,
368+
&input_queue,
369+
&output_queue,
370+
shared_matcher,
371+
/*is_shared_creator=*/w == 0));
372+
}
373+
for (auto& worker : workers) {
374+
worker->Start();
375+
}
376+
EXPECT_FALSE(workers[0]->CheckValidSetup());
377+
EXPECT_THAT(ThrownMessage([&]() { workers[0]->Wait(); }),
378+
testing::HasSubstr(expected_message));
379+
for (size_t w = 1; w < workers.size(); ++w) {
380+
EXPECT_FALSE(workers[w]->CheckValidSetup());
381+
EXPECT_NO_THROW(workers[w]->Wait());
382+
}
383+
}
384+
385+
#ifdef COLMAP_CUDA_ENABLED
386+
if (GetNumCudaDevices() > 0) {
387+
FeatureMatchingOptions flexglue_options(FeatureMatcherType::FLEXGLUE);
388+
flexglue_options.use_gpu = true;
389+
flexglue_options.gpu_index = "0";
390+
flexglue_options.num_threads = 2;
391+
flexglue_options.flexglue->image_path = data.test_dir.string();
392+
flexglue_options.flexglue->num_workers_per_gpu = 3;
393+
flexglue_options.flexglue->descriptor_model_path =
394+
(data.test_dir / "missing_model.onnx").string();
395+
FeatureMatcherController controller(
396+
flexglue_options, geometry_options, data.cache);
397+
EXPECT_THAT(ThrownMessage([&]() { controller.Setup(); }),
398+
testing::HasSubstr(expected_message));
399+
}
400+
#endif
401+
}
402+
403+
TEST(FeatureMatcherController, SharedFeatureMatcherWorkersMatchPairs) {
404+
auto data = CreateTestData(4);
405+
data.database->ClearMatches();
406+
data.database->ClearTwoViewGeometries();
407+
std::vector<FeatureDescriptors> normalized_descriptors;
408+
normalized_descriptors.reserve(data.image_ids.size());
409+
for (const image_t image_id : data.image_ids) {
410+
FeatureDescriptors desc = data.database->ReadDescriptors(image_id);
411+
FeatureDescriptorsFloatData float_desc = desc.ToFloat().data;
412+
L2NormalizeFeatureDescriptors(&float_desc);
413+
desc.data = FeatureDescriptorsToUnsignedByte(float_desc);
414+
normalized_descriptors.push_back(std::move(desc));
415+
}
416+
data.database->ClearDescriptors();
417+
for (size_t i = 0; i < data.image_ids.size(); ++i) {
418+
data.database->WriteDescriptors(data.image_ids[i],
419+
normalized_descriptors[i]);
420+
}
421+
data.cache = std::make_shared<FeatureMatcherCache>(100, data.database);
422+
423+
FeatureMatchingOptions matching_options = DefaultMatchingOptions();
424+
matching_options.skip_geometric_verification = true;
425+
// Brute-force SIFT avoids SiftCPUFeatureMatcher's non-thread-safe index
426+
// cache so the shared matcher is safe across concurrent workers.
427+
matching_options.sift->cpu_brute_force_matcher = true;
428+
TwoViewGeometryOptions geometry_options;
429+
JobQueue<FeatureMatcherData> input_queue;
430+
JobQueue<FeatureMatcherData> output_queue;
431+
auto shared_matcher = std::make_shared<SharedFeatureMatcher>();
432+
433+
std::vector<std::unique_ptr<FeatureMatcherWorker>> workers;
434+
workers.reserve(3);
435+
for (int w = 0; w < 3; ++w) {
436+
workers.push_back(
437+
std::make_unique<FeatureMatcherWorker>(matching_options,
438+
geometry_options,
439+
data.cache,
440+
&input_queue,
441+
&output_queue,
442+
shared_matcher,
443+
/*is_shared_creator=*/w == 0));
444+
workers.back()->Start();
445+
}
446+
for (auto& worker : workers) {
447+
ASSERT_TRUE(worker->CheckValidSetup());
448+
}
449+
450+
const auto pairs = AllPairs(data.image_ids);
451+
for (const auto& [id1, id2] : pairs) {
452+
FeatureMatcherData job;
453+
job.image_id1 = id1;
454+
job.image_id2 = id2;
455+
ASSERT_TRUE(input_queue.Push(std::move(job)));
456+
}
457+
458+
for (size_t i = 0; i < pairs.size(); ++i) {
459+
auto out = output_queue.Pop();
460+
ASSERT_TRUE(out.IsValid());
461+
EXPECT_EQ(out.Data().error, nullptr);
462+
EXPECT_FALSE(out.Data().matches.empty());
463+
}
464+
465+
input_queue.Wait();
466+
for (auto& worker : workers) {
467+
worker->Stop();
468+
}
469+
input_queue.Stop();
470+
output_queue.Stop();
471+
for (auto& worker : workers) {
472+
worker->Wait();
473+
}
349474
}
350475

351476
TEST(FeatureMatcherController, MatchSkipGeometricVerification) {

‎src/colmap/controllers/option_manager.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,8 @@ void OptionManager::AddFeatureMatchingOptions() {
464464
&feature_matching->flexglue->matcher_model_path_fp16);
465465
AddDefaultOption("FlexGlueMatching.use_fp16",
466466
&feature_matching->flexglue->use_fp16);
467+
AddDefaultOption("FlexGlueMatching.num_workers_per_gpu",
468+
&feature_matching->flexglue->num_workers_per_gpu);
467469
}
468470

469471
void OptionManager::AddTwoViewGeometryOptions() {

0 commit comments

Comments
 (0)