@@ -139,19 +139,31 @@ bool UseFlexGlueFp16(const FeatureMatchingOptions& options) {
139139 return false ;
140140}
141141
142+ std::shared_ptr<ONNXInferenceLimiter> CreateInferenceLimiter (
143+ const FeatureMatchingOptions& options) {
144+ if (SelectONNXExecutionProvider (options.use_gpu ) !=
145+ ONNXExecutionProvider::CUDA ) {
146+ return nullptr ;
147+ }
148+ return std::make_shared<ONNXInferenceLimiter>(
149+ THROW_CHECK_NOTNULL (options.flexglue )->num_inferences_per_gpu );
150+ }
151+
142152// Matcher with float16 or float32 transformer layers (see UseFlexGlueFp16).
143153// The float16 matcher is loaded as a regular session, with a fallback to the
144154// float32 matcher if its initialization fails.
145155std::unique_ptr<ONNXModel> CreateMatcherModel (
146- const FeatureMatchingOptions& options) {
156+ const FeatureMatchingOptions& options,
157+ const std::shared_ptr<ONNXInferenceLimiter>& inference_limiter) {
147158 const FlexGlueMatchingOptions& flexglue_options = *options.flexglue ;
148159 if (UseFlexGlueFp16 (options)) {
149160 try {
150161 return std::make_unique<ONNXModel>(
151162 flexglue_options.matcher_model_path_fp16 ,
152163 options.num_threads ,
153164 options.use_gpu ,
154- options.gpu_index );
165+ options.gpu_index ,
166+ inference_limiter);
155167 } catch (const Ort::Exception& e) {
156168 LOG (WARNING ) << " Failed to initialize the fp16 FlexGlue matcher ("
157169 << e.what () << " ); falling back to fp32" ;
@@ -160,18 +172,22 @@ std::unique_ptr<ONNXModel> CreateMatcherModel(
160172 return std::make_unique<ONNXModel>(flexglue_options.matcher_model_path ,
161173 options.num_threads ,
162174 options.use_gpu ,
163- options.gpu_index );
175+ options.gpu_index ,
176+ inference_limiter);
164177}
165178
166179class FlexGlueFeatureMatcher : public FeatureMatcher {
167180 public:
168181 explicit FlexGlueFeatureMatcher (const FeatureMatchingOptions& options)
169182 : options_(*THROW_CHECK_NOTNULL (options.flexglue)),
183+ inference_limiter_(CreateInferenceLimiter(options)),
170184 descriptor_(options_.descriptor_model_path,
171185 options.num_threads,
172186 options.use_gpu,
173- options.gpu_index),
174- matcher_(CreateMatcherModel(options)),
187+ options.gpu_index,
188+ inference_limiter_,
189+ /* allow_session_replicas=*/ false ),
190+ matcher_(CreateMatcherModel(options, inference_limiter_)),
175191 dense_cache_(options_.feature_cache_size,
176192 [this ](const std::string& name) {
177193 return ComputeDenseFeatures (name);
@@ -413,19 +429,24 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
413429 << " FlexGlue requires the image name to read the image pixels." ;
414430 const std::filesystem::path path =
415431 std::filesystem::path (options_.image_path ) / name;
416- Bitmap bitmap;
417- THROW_CHECK (bitmap.Read (path, /* as_rgb=*/ true ))
418- << " Failed to read image: " << path;
419-
420432 auto dense = std::make_shared<DenseFeatures>();
421- dense->width = bitmap.Width ();
422- dense->height = bitmap.Height ();
423-
424- std::vector<float > input =
425- ResizeBilinearToCHW (bitmap, descriptor_width_, descriptor_height_);
433+ std::vector<float > input;
434+ {
435+ // Release the full-resolution decoded bitmap before acquiring an
436+ // inference slot, so workers waiting on the GPU hold only the resized
437+ // descriptor input tensor.
438+ Bitmap bitmap;
439+ THROW_CHECK (bitmap.Read (path, /* as_rgb=*/ true ))
440+ << " Failed to read image: " << path;
441+ dense->width = bitmap.Width ();
442+ dense->height = bitmap.Height ();
443+ input =
444+ ResizeBilinearToCHW (bitmap, descriptor_width_, descriptor_height_);
445+ }
426446 const std::vector<int64_t > input_shape = {
427447 1 , 3 , descriptor_height_, descriptor_width_};
428448 std::vector<Ort::Value> inputs;
449+ inputs.reserve (1 );
429450 inputs.push_back (CreateONNXTensor (input, input_shape));
430451 std::vector<Ort::Value> outputs =
431452 feature_memory_info_ ? descriptor_.Run (inputs, *feature_memory_info_)
@@ -437,6 +458,7 @@ class FlexGlueFeatureMatcher : public FeatureMatcher {
437458 }
438459
439460 const FlexGlueMatchingOptions options_;
461+ std::shared_ptr<ONNXInferenceLimiter> inference_limiter_;
440462 ONNXModel descriptor_;
441463 std::unique_ptr<ONNXModel> matcher_;
442464 int descriptor_height_ = 0 ;
@@ -468,6 +490,7 @@ bool FlexGlueMatchingOptions::Check() const {
468490 CHECK_OPTION (!descriptor_model_path.empty ());
469491 CHECK_OPTION (!matcher_model_path.empty ());
470492 CHECK_OPTION_GT (num_workers_per_gpu, 0 );
493+ CHECK_OPTION_GT (num_inferences_per_gpu, 0 );
471494 return true ;
472495}
473496
0 commit comments