1818#include < filesystem>
1919#include < limits>
2020#include < memory>
21+ #include < optional>
2122#include < string>
2223#include < string_view>
2324#include < utility>
@@ -88,15 +89,27 @@ std::vector<float> ResizeBilinearToCHW(const Bitmap& bitmap,
8889
8990// Dense features of an image, as output by the descriptor model.
9091struct DenseFeatures {
91- std::vector< float > data;
92- std::vector< int64_t > shape ;
92+ // Owned by ONNX Runtime, in host or GPU memory.
93+ Ort::Value value{ nullptr } ;
9394 // Size of the image file.
9495 int width = 0 ;
9596 int height = 0 ;
9697 // Whether the image size was compared with the camera size.
9798 bool checked_camera_size = false ;
9899};
99100
101+ // Non-owning tensor referencing the memory of the given tensor.
102+ Ort::Value TensorView (const Ort::Value& value) {
103+ const Ort::TensorTypeAndShapeInfo info = value.GetTensorTypeAndShapeInfo ();
104+ const std::vector<int64_t > shape = info.GetShape ();
105+ return Ort::Value::CreateTensor (value.GetTensorMemoryInfo (),
106+ const_cast <void *>(value.GetTensorRawData ()),
107+ info.GetElementCount () * sizeof (float ),
108+ shape.data (),
109+ shape.size (),
110+ info.GetElementType ());
111+ }
112+
100113// Whether to load the matcher with float16 transformer layers instead of the
101114// float32 matcher. This is decided up front, as for the bf16 LoMa models: the
102115// float16 matcher only runs with the CUDA execution provider, on GPUs with
@@ -186,6 +199,17 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
186199 descriptor_.output_element_types ()[0 ],
187200 ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT );
188201
202+ // Keep the dense features on the GPU, where the matcher reads them, which
203+ // avoids copying both feature maps to the GPU for every pair. The GPU
204+ // memory is bounded by feature_cache_size.
205+ if (matcher_->execution_provider () == ONNXExecutionProvider::CUDA &&
206+ descriptor_.execution_provider () == ONNXExecutionProvider::CUDA ) {
207+ feature_memory_info_.emplace (" Cuda" ,
208+ OrtArenaAllocator,
209+ SelectONNXCudaDevice (options.gpu_index ),
210+ OrtMemTypeDefault);
211+ }
212+
189213 // Matcher: inputs are matched by name, see Match().
190214 const std::vector<std::string_view> expected_inputs = {" features0" ,
191215 " features1" ,
@@ -270,9 +294,9 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
270294 for (const char * name_cstr : matcher_->input_names ()) {
271295 const std::string_view name (name_cstr);
272296 if (name == " features0" ) {
273- inputs.push_back (CreateONNXTensor (dense1->data , dense1-> shape ));
297+ inputs.push_back (TensorView (dense1->value ));
274298 } else if (name == " features1" ) {
275- inputs.push_back (CreateONNXTensor (dense2->data , dense2-> shape ));
299+ inputs.push_back (TensorView (dense2->value ));
276300 } else if (name == " kpts0" ) {
277301 inputs.push_back (CreateONNXTensor (kpts1, kpts1_shape));
278302 } else if (name == " kpts1" ) {
@@ -389,14 +413,12 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
389413 1 , 3 , descriptor_height_, descriptor_width_};
390414 std::vector<Ort::Value> inputs;
391415 inputs.push_back (CreateONNXTensor (input, input_shape));
392- const std::vector<Ort::Value> outputs = descriptor_.Run (inputs);
416+ std::vector<Ort::Value> outputs =
417+ feature_memory_info_ ? descriptor_.Run (inputs, *feature_memory_info_)
418+ : descriptor_.Run (inputs);
393419 THROW_CHECK_EQ (outputs.size (), 1 );
394-
395- const auto info = outputs[0 ].GetTensorTypeAndShapeInfo ();
396- dense->shape = info.GetShape ();
397- THROW_CHECK_EQ (dense->shape .size (), 4 );
398- const float * features = outputs[0 ].GetTensorData <float >();
399- dense->data .assign (features, features + info.GetElementCount ());
420+ THROW_CHECK_EQ (outputs[0 ].GetTensorTypeAndShapeInfo ().GetShape ().size (), 4 );
421+ dense->value = std::move (outputs[0 ]);
400422 return dense;
401423 }
402424
@@ -408,6 +430,8 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
408430 int matches0_idx_ = -1 ;
409431 int covisibility_score_idx_ = -1 ;
410432
433+ // Memory of the cached dense features, if not host memory.
434+ std::optional<Ort::MemoryInfo> feature_memory_info_;
411435 // Dense features keyed by image name. Unlike ImageFeatureCache, it retains
412436 // many images, since pairs of exhaustive or retrieval-based matching revisit
413437 // the same images far apart in time. Neither the cache nor its entries
0 commit comments