Skip to content

Commit 5009431

Browse files
committed
FlexGlue: bound concurrent GPU inferences with a semaphore
1 parent ebd27cb commit 5009431

9 files changed

Lines changed: 419 additions & 72 deletions

File tree

‎doc/features.rst‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,12 @@ variant of the matcher whose transformer layers run in float16 is used by
218218
default (``--FlexGlueMatching.use_fp16``), which is faster than the float32
219219
matcher with practically identical matches; the float32 matcher is used on the CPU, on
220220
older GPUs, or if the float16 variant fails to initialize. On CUDA GPUs,
221-
``--FlexGlueMatching.num_workers_per_gpu`` (default 3) matching threads share a
221+
``--FlexGlueMatching.num_workers_per_gpu`` (default 4) matching threads share a
222222
single matcher instance and GPU feature cache per GPU so CPU image decoding
223-
overlaps with GPU inference (lower it to reduce GPU memory usage). On the CPU, a
223+
overlaps with GPU inference, while ``--FlexGlueMatching.num_inferences_per_gpu``
224+
(default 2) bounds the number of concurrent model inferences on the GPU (lower
225+
it, e.g. to 1, to reduce GPU memory usage while keeping parallel CPU image
226+
decoding). On the CPU, a
224227
single matching thread runs the models with ``--FeatureMatching.num_threads``
225228
threads, rather than one copy of the models per thread. The
226229
``automatic_reconstructor`` provides the ``--feature flexglue`` preset, which

‎src/colmap/controllers/option_manager.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -466,6 +466,8 @@ void OptionManager::AddFeatureMatchingOptions() {
466466
&feature_matching->flexglue->use_fp16);
467467
AddDefaultOption("FlexGlueMatching.num_workers_per_gpu",
468468
&feature_matching->flexglue->num_workers_per_gpu);
469+
AddDefaultOption("FlexGlueMatching.num_inferences_per_gpu",
470+
&feature_matching->flexglue->num_inferences_per_gpu);
469471
}
470472

471473
void OptionManager::AddTwoViewGeometryOptions() {

‎src/colmap/feature/flexglue.cc‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -139,19 +139,31 @@ bool UseFlexGlueFp16(const FeatureMatchingOptions& options) {
139139
return false;
140140
}
141141

142+
std::shared_ptr<ONNXInferenceLimiter> CreateInferenceLimiter(
143+
const FeatureMatchingOptions& options) {
144+
if (SelectONNXExecutionProvider(options.use_gpu) !=
145+
ONNXExecutionProvider::CUDA) {
146+
return nullptr;
147+
}
148+
return std::make_shared<ONNXInferenceLimiter>(
149+
THROW_CHECK_NOTNULL(options.flexglue)->num_inferences_per_gpu);
150+
}
151+
142152
// Matcher with float16 or float32 transformer layers (see UseFlexGlueFp16).
143153
// The float16 matcher is loaded as a regular session, with a fallback to the
144154
// float32 matcher if its initialization fails.
145155
std::unique_ptr<ONNXModel> CreateMatcherModel(
146-
const FeatureMatchingOptions& options) {
156+
const FeatureMatchingOptions& options,
157+
const std::shared_ptr<ONNXInferenceLimiter>& inference_limiter) {
147158
const FlexGlueMatchingOptions& flexglue_options = *options.flexglue;
148159
if (UseFlexGlueFp16(options)) {
149160
try {
150161
return std::make_unique<ONNXModel>(
151162
flexglue_options.matcher_model_path_fp16,
152163
options.num_threads,
153164
options.use_gpu,
154-
options.gpu_index);
165+
options.gpu_index,
166+
inference_limiter);
155167
} catch (const Ort::Exception& e) {
156168
LOG(WARNING) << "Failed to initialize the fp16 FlexGlue matcher ("
157169
<< e.what() << "); falling back to fp32";
@@ -160,18 +172,22 @@ std::unique_ptr<ONNXModel> CreateMatcherModel(
160172
return std::make_unique<ONNXModel>(flexglue_options.matcher_model_path,
161173
options.num_threads,
162174
options.use_gpu,
163-
options.gpu_index);
175+
options.gpu_index,
176+
inference_limiter);
164177
}
165178

166179
class FlexGlueFeatureMatcher : public FeatureMatcher {
167180
public:
168181
explicit FlexGlueFeatureMatcher(const FeatureMatchingOptions& options)
169182
: options_(*THROW_CHECK_NOTNULL(options.flexglue)),
183+
inference_limiter_(CreateInferenceLimiter(options)),
170184
descriptor_(options_.descriptor_model_path,
171185
options.num_threads,
172186
options.use_gpu,
173-
options.gpu_index),
174-
matcher_(CreateMatcherModel(options)),
187+
options.gpu_index,
188+
inference_limiter_,
189+
/*allow_session_replicas=*/false),
190+
matcher_(CreateMatcherModel(options, inference_limiter_)),
175191
dense_cache_(options_.feature_cache_size,
176192
[this](const std::string& name) {
177193
return ComputeDenseFeatures(name);
@@ -413,19 +429,24 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
413429
<< "FlexGlue requires the image name to read the image pixels.";
414430
const std::filesystem::path path =
415431
std::filesystem::path(options_.image_path) / name;
416-
Bitmap bitmap;
417-
THROW_CHECK(bitmap.Read(path, /*as_rgb=*/true))
418-
<< "Failed to read image: " << path;
419-
420432
auto dense = std::make_shared<DenseFeatures>();
421-
dense->width = bitmap.Width();
422-
dense->height = bitmap.Height();
423-
424-
std::vector<float> input =
425-
ResizeBilinearToCHW(bitmap, descriptor_width_, descriptor_height_);
433+
std::vector<float> input;
434+
{
435+
// Release the full-resolution decoded bitmap before acquiring an
436+
// inference slot, so workers waiting on the GPU hold only the resized
437+
// descriptor input tensor.
438+
Bitmap bitmap;
439+
THROW_CHECK(bitmap.Read(path, /*as_rgb=*/true))
440+
<< "Failed to read image: " << path;
441+
dense->width = bitmap.Width();
442+
dense->height = bitmap.Height();
443+
input =
444+
ResizeBilinearToCHW(bitmap, descriptor_width_, descriptor_height_);
445+
}
426446
const std::vector<int64_t> input_shape = {
427447
1, 3, descriptor_height_, descriptor_width_};
428448
std::vector<Ort::Value> inputs;
449+
inputs.reserve(1);
429450
inputs.push_back(CreateONNXTensor(input, input_shape));
430451
std::vector<Ort::Value> outputs =
431452
feature_memory_info_ ? descriptor_.Run(inputs, *feature_memory_info_)
@@ -437,6 +458,7 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
437458
}
438459

439460
const FlexGlueMatchingOptions options_;
461+
std::shared_ptr<ONNXInferenceLimiter> inference_limiter_;
440462
ONNXModel descriptor_;
441463
std::unique_ptr<ONNXModel> matcher_;
442464
int descriptor_height_ = 0;
@@ -468,6 +490,7 @@ bool FlexGlueMatchingOptions::Check() const {
468490
CHECK_OPTION(!descriptor_model_path.empty());
469491
CHECK_OPTION(!matcher_model_path.empty());
470492
CHECK_OPTION_GT(num_workers_per_gpu, 0);
493+
CHECK_OPTION_GT(num_inferences_per_gpu, 0);
471494
return true;
472495
}
473496

‎src/colmap/feature/flexglue.h‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,16 @@ struct FlexGlueMatchingOptions {
8383
// Number of matching worker threads per CUDA GPU that share one FlexGlue
8484
// matcher instance (including its ONNX sessions and GPU feature cache).
8585
// Multiple workers allow image decoding and preprocessing on the CPU to
86-
// overlap with model inference on the GPU without duplicating model weights
87-
// or cached features; lower this value (e.g., to 1 or 2) to reduce GPU
88-
// memory usage. Ignored when not running on a CUDA GPU.
89-
int num_workers_per_gpu = 3;
86+
// overlap with model inference on the GPU without duplicating cached
87+
// features. Ignored when not running on a CUDA GPU.
88+
int num_workers_per_gpu = 4;
89+
90+
// Maximum number of concurrent ONNX model inferences per CUDA GPU across the
91+
// shared worker threads. Setting this lower than num_workers_per_gpu (e.g.,
92+
// to 1 on GPUs with limited memory) allows multiple CPU threads to decode
93+
// and preprocess images in parallel while bounding peak GPU memory usage.
94+
// Ignored when not running on a CUDA GPU.
95+
int num_inferences_per_gpu = 2;
9096

9197
bool Check() const;
9298
};

‎src/colmap/feature/flexglue_test.cc‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,15 @@ TEST(FlexGlueMatchingOptions, Check) {
186186
// Without the float16 matcher, the float32 matcher is used.
187187
options.matcher_model_path_fp16.clear();
188188
EXPECT_TRUE(options.Check());
189-
EXPECT_EQ(options.num_workers_per_gpu, 3);
189+
EXPECT_EQ(options.num_workers_per_gpu, 4);
190190
options.num_workers_per_gpu = 0;
191191
EXPECT_FALSE(options.Check());
192-
options.num_workers_per_gpu = 3;
192+
options.num_workers_per_gpu = 4;
193+
EXPECT_TRUE(options.Check());
194+
EXPECT_EQ(options.num_inferences_per_gpu, 2);
195+
options.num_inferences_per_gpu = 0;
196+
EXPECT_FALSE(options.Check());
197+
options.num_inferences_per_gpu = 1;
193198
EXPECT_TRUE(options.Check());
194199
}
195200

0 commit comments

Comments
 (0)