Skip to content

Commit d0a411f

Browse files
authored
FlexGlue: cache the dense features on the GPU (colmap#4915)
When the FlexGlue models run on a CUDA GPU, this PR keeps the cached dense features on the GPU: - The descriptor writes its output directly to GPU memory through an ONNX Runtime I/O binding, via a new `ONNXModel::Run(inputs, output_memory_info)` overload. - The cache stores the resulting `Ort::Value`s. - The matcher reads the features in place, through non-owning tensor views. Previously, both 4 MB float32 feature maps were copied host→device for every pair.
1 parent 721e185 commit d0a411f

7 files changed

Lines changed: 96 additions & 19 deletions

File tree

‎doc/features.rst‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -212,11 +212,11 @@ default models with the default ``early_stop_decay 0.5`` and
212212
although they would pass ``min_covisibility`` after the last layer.
213213

214214
The dense features of the most recently used images are cached in memory
215-
(``--FlexGlueMatching.feature_cache_size``, ~4MB per image). On CUDA GPUs with
216-
compute capability 7.0 or newer, a variant of the matcher whose
217-
transformer layers run in float16 is used by default
218-
(``--FlexGlueMatching.use_fp16``), which is faster than the float32 matcher
219-
with practically identical matches; the float32 matcher is used on the CPU, on
215+
(``--FlexGlueMatching.feature_cache_size``, ~4MB per image), on the GPU if the
216+
models run on a CUDA GPU. On CUDA GPUs with compute capability 7.0 or newer, a
217+
variant of the matcher whose transformer layers run in float16 is used by
218+
default (``--FlexGlueMatching.use_fp16``), which is faster than the float32
219+
matcher with practically identical matches; the float32 matcher is used on the CPU, on
220220
older GPUs, or if the float16 variant fails to initialize. On the CPU, a
221221
single matching thread runs the models with ``--FeatureMatching.num_threads``
222222
threads, rather than one copy of the models per thread. The

‎src/colmap/feature/flexglue.cc‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <filesystem>
1919
#include <limits>
2020
#include <memory>
21+
#include <optional>
2122
#include <string>
2223
#include <string_view>
2324
#include <utility>
@@ -88,15 +89,27 @@ std::vector<float> ResizeBilinearToCHW(const Bitmap& bitmap,
8889

8990
// Dense features of an image, as output by the descriptor model.
9091
struct DenseFeatures {
91-
std::vector<float> data;
92-
std::vector<int64_t> shape;
92+
// Owned by ONNX Runtime, in host or GPU memory.
93+
Ort::Value value{nullptr};
9394
// Size of the image file.
9495
int width = 0;
9596
int height = 0;
9697
// Whether the image size was compared with the camera size.
9798
bool checked_camera_size = false;
9899
};
99100

101+
// Non-owning tensor referencing the memory of the given tensor.
102+
Ort::Value TensorView(const Ort::Value& value) {
103+
const Ort::TensorTypeAndShapeInfo info = value.GetTensorTypeAndShapeInfo();
104+
const std::vector<int64_t> shape = info.GetShape();
105+
return Ort::Value::CreateTensor(value.GetTensorMemoryInfo(),
106+
const_cast<void*>(value.GetTensorRawData()),
107+
info.GetElementCount() * sizeof(float),
108+
shape.data(),
109+
shape.size(),
110+
info.GetElementType());
111+
}
112+
100113
// Whether to load the matcher with float16 transformer layers instead of the
101114
// float32 matcher. This is decided up front, as for the bf16 LoMa models: the
102115
// float16 matcher only runs with the CUDA execution provider, on GPUs with
@@ -186,6 +199,17 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
186199
descriptor_.output_element_types()[0],
187200
ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT);
188201

202+
// Keep the dense features on the GPU, where the matcher reads them, which
203+
// avoids copying both feature maps to the GPU for every pair. The GPU
204+
// memory is bounded by feature_cache_size.
205+
if (matcher_->execution_provider() == ONNXExecutionProvider::CUDA &&
206+
descriptor_.execution_provider() == ONNXExecutionProvider::CUDA) {
207+
feature_memory_info_.emplace("Cuda",
208+
OrtArenaAllocator,
209+
SelectONNXCudaDevice(options.gpu_index),
210+
OrtMemTypeDefault);
211+
}
212+
189213
// Matcher: inputs are matched by name, see Match().
190214
const std::vector<std::string_view> expected_inputs = {"features0",
191215
"features1",
@@ -270,9 +294,9 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
270294
for (const char* name_cstr : matcher_->input_names()) {
271295
const std::string_view name(name_cstr);
272296
if (name == "features0") {
273-
inputs.push_back(CreateONNXTensor(dense1->data, dense1->shape));
297+
inputs.push_back(TensorView(dense1->value));
274298
} else if (name == "features1") {
275-
inputs.push_back(CreateONNXTensor(dense2->data, dense2->shape));
299+
inputs.push_back(TensorView(dense2->value));
276300
} else if (name == "kpts0") {
277301
inputs.push_back(CreateONNXTensor(kpts1, kpts1_shape));
278302
} else if (name == "kpts1") {
@@ -389,14 +413,12 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
389413
1, 3, descriptor_height_, descriptor_width_};
390414
std::vector<Ort::Value> inputs;
391415
inputs.push_back(CreateONNXTensor(input, input_shape));
392-
const std::vector<Ort::Value> outputs = descriptor_.Run(inputs);
416+
std::vector<Ort::Value> outputs =
417+
feature_memory_info_ ? descriptor_.Run(inputs, *feature_memory_info_)
418+
: descriptor_.Run(inputs);
393419
THROW_CHECK_EQ(outputs.size(), 1);
394-
395-
const auto info = outputs[0].GetTensorTypeAndShapeInfo();
396-
dense->shape = info.GetShape();
397-
THROW_CHECK_EQ(dense->shape.size(), 4);
398-
const float* features = outputs[0].GetTensorData<float>();
399-
dense->data.assign(features, features + info.GetElementCount());
420+
THROW_CHECK_EQ(outputs[0].GetTensorTypeAndShapeInfo().GetShape().size(), 4);
421+
dense->value = std::move(outputs[0]);
400422
return dense;
401423
}
402424

@@ -408,6 +430,8 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
408430
int matches0_idx_ = -1;
409431
int covisibility_score_idx_ = -1;
410432

433+
// Memory of the cached dense features, if not host memory.
434+
std::optional<Ort::MemoryInfo> feature_memory_info_;
411435
// Dense features keyed by image name. Unlike ImageFeatureCache, it retains
412436
// many images, since pairs of exhaustive or retrieval-based matching revisit
413437
// the same images far apart in time. Neither the cache nor its entries

‎src/colmap/feature/flexglue.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,9 @@ struct FlexGlueMatchingOptions {
5858
// min_covisibility = 0.
5959
double early_stop_decay = 0.5;
6060

61-
// Maximum number of images whose dense features are cached in memory.
62-
// Each entry takes ~4MB with the default model.
61+
// Maximum number of images whose dense features are cached in memory, on the
62+
// GPU if the models run on a CUDA GPU, which avoids copying them to the GPU
63+
// for every pair. Each entry takes ~4MB with the default model.
6364
int feature_cache_size = 128;
6465

6566
// Dense image descriptor: image [1, 3, S, S] -> features [1, C, S/8, S/8].

‎src/colmap/util/onnx.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,26 @@ std::vector<Ort::Value> ONNXModel::Run(
269269
}
270270
}
271271

272+
std::vector<Ort::Value> ONNXModel::Run(
273+
const std::vector<Ort::Value>& input_tensors,
274+
const Ort::MemoryInfo& output_memory_info) const {
275+
try {
276+
THROW_CHECK_EQ(input_tensors.size(), input_names_.size());
277+
Ort::IoBinding binding(*session_);
278+
for (size_t i = 0; i < input_tensors.size(); ++i) {
279+
binding.BindInput(input_names_[i], input_tensors[i]);
280+
}
281+
for (const char* name : output_names_) {
282+
binding.BindOutput(name, output_memory_info);
283+
}
284+
session_->Run(Ort::RunOptions(), binding);
285+
binding.SynchronizeOutputs();
286+
return binding.GetOutputValues();
287+
} catch (...) {
288+
RethrowONNXException();
289+
}
290+
}
291+
272292
#endif // COLMAP_ONNX_ENABLED
273293

274294
} // namespace colmap

‎src/colmap/util/onnx.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ class ONNXModel {
8181
std::vector<Ort::Value> Run(
8282
const std::vector<Ort::Value>& input_tensors) const;
8383

84+
// Same as above, but the outputs are allocated in the given memory, e.g., on
85+
// the GPU or in page-locked host memory, so that they can be passed to
86+
// another model without copies.
87+
std::vector<Ort::Value> Run(const std::vector<Ort::Value>& input_tensors,
88+
const Ort::MemoryInfo& output_memory_info) const;
89+
8490
const std::vector<std::vector<int64_t>>& input_shapes() const {
8591
return input_shapes_;
8692
}

‎src/colmap/util/onnx_test.cc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,31 @@ TEST(ONNXModelTest, MetadataAndInference) {
9797
EXPECT_FLOAT_EQ(output_data[1], input_data[1]);
9898
}
9999

100+
TEST(ONNXModelTest, RunWithOutputMemoryInfo) {
101+
ONNXModel model(WriteIdentityModel().string(),
102+
/*num_threads=*/1,
103+
/*use_gpu=*/false,
104+
/*gpu_index=*/"-1");
105+
106+
std::vector<float> input_data{1.25f, -2.5f};
107+
const std::vector<int64_t> input_shape{1, 2};
108+
std::vector<Ort::Value> inputs;
109+
inputs.push_back(MakeTensor(&input_data, input_shape));
110+
// The outputs are allocated in the given memory through an I/O binding, as
111+
// FlexGlue does on the GPU. Host memory gives the same values as Run.
112+
const std::vector<Ort::Value> outputs = model.Run(
113+
inputs, Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault));
114+
115+
ASSERT_EQ(outputs.size(), 1);
116+
EXPECT_EQ(outputs[0].GetTensorTypeAndShapeInfo().GetShape(), input_shape);
117+
const float* output_data = outputs[0].GetTensorData<float>();
118+
EXPECT_FLOAT_EQ(output_data[0], input_data[0]);
119+
EXPECT_FLOAT_EQ(output_data[1], input_data[1]);
120+
// The output owns its memory, independently of the input.
121+
input_data[0] = 0.f;
122+
EXPECT_FLOAT_EQ(output_data[0], 1.25f);
123+
}
124+
100125
TEST(ThrowCheckONNXElementTypeTest, AcceptsMatchingType) {
101126
EXPECT_NO_THROW(
102127
ThrowCheckONNXElementType("image",

‎src/pycolmap/feature/matching.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,8 @@ void BindFeatureMatching(py::module& m) {
201201
.def_readwrite("feature_cache_size",
202202
&FlexGlueMatchingOptions::feature_cache_size,
203203
"Maximum number of images whose dense features are "
204-
"cached in memory.")
204+
"cached in memory, on the GPU if the models run on "
205+
"a CUDA GPU (~4MB per image).")
205206
.def_readwrite("descriptor_model_path",
206207
&FlexGlueMatchingOptions::descriptor_model_path,
207208
"Path to the ONNX model of the dense descriptor.")

0 commit comments

Comments
 (0)