Skip to content

Commit ab9141e

Browse files
skottmckaygithub-actions[bot]jywu-mysoft
authored
Cherry pick #24625 into rel-1.22.0 (#24626)
### Description <!-- Describe your changes. --> Add #24625 ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: George Wu <[email protected]>
1 parent d66cff1 commit ab9141e

19 files changed

Lines changed: 724 additions & 65 deletions

‎include/onnxruntime/core/session/onnxruntime_c_api.h‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,32 @@ typedef enum OrtExecutionProviderDevicePolicy {
425425
OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER,
426426
} OrtExecutionProviderDevicePolicy;
427427

428+
/** \brief Delegate to allow providing custom OrtEpDevice selection logic
429+
*
430+
* This delegate is called by the EP selection code to allow the user to provide custom device selection logic.
431+
* The user can use this to select OrtEpDevice instances from the list of available devices.
432+
*
433+
* \param ep_devices The list of available devices.
434+
* \param num_devices The number of available devices.
435+
* \param model_metadata The model metadata.
436+
* \param runtime_metadata The runtime metadata. May be nullptr.
437+
* \param selected Pre-allocated array to populate with selected OrtEpDevice pointers from ep_devices.
438+
* \param max_ep_devices The maximum number of devices that can be selected in the pre-allocated array.
439+
Currently the maximum is 8.
440+
* \param num_ep_devices The number of selected devices.
441+
*
442+
* \return OrtStatus* Selection status. Return nullptr on success.
443+
* Use CreateStatus to provide error info. Use ORT_FAIL as the error code.
444+
* ORT will release the OrtStatus* if not null.
445+
*/
446+
typedef OrtStatus* (*EpSelectionDelegate)(_In_ const OrtEpDevice** ep_devices,
447+
_In_ size_t num_devices,
448+
_In_ const OrtKeyValuePairs* model_metadata,
449+
_In_opt_ const OrtKeyValuePairs* runtime_metadata,
450+
_Inout_ const OrtEpDevice** selected,
451+
_In_ size_t max_selected,
452+
_Out_ size_t* num_selected);
453+
428454
/** \brief Algorithm to use for cuDNN Convolution Op
429455
*/
430456
typedef enum OrtCudnnConvAlgoSearch {
@@ -5073,7 +5099,8 @@ struct OrtApi {
50735099
ORT_API2_STATUS(GetEpDevices, _In_ const OrtEnv* env,
50745100
_Outptr_ const OrtEpDevice* const** ep_devices, _Out_ size_t* num_ep_devices);
50755101

5076-
/** \brief Append execution provider to the session options by name.
5102+
/** \brief Append the execution provider that is responsible for the selected OrtEpDevice instances
5103+
* to the session options.
50775104
*
50785105
* \param[in] session_options Session options to add execution provider to.
50795106
* \param[in] env Environment that execution providers were registered with.
@@ -5098,6 +5125,21 @@ struct OrtApi {
50985125
_In_reads_(num_op_options) const char* const* ep_option_vals,
50995126
size_t num_ep_options);
51005127

5128+
/** \brief Set the execution provider selection policy for the session.
5129+
*
5130+
* Allows users to specify a device selection policy for automatic execution provider (EP) selection,
5131+
* or provide a delegate callback for custom selection logic.
5132+
*
5133+
* \param[in] session_options The OrtSessionOptions instance.
5134+
* \param[in] policy The device selection policy to use (see OrtExecutionProviderDevicePolicy).
5135+
* \param[in] delegate Optional delegate callback for custom selection. Pass nullptr to use the built-in policy.
5136+
*
5137+
* \since Version 1.22
5138+
*/
5139+
ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicy, _In_ OrtSessionOptions* session_options,
5140+
_In_ OrtExecutionProviderDevicePolicy policy,
5141+
_In_opt_ EpSelectionDelegate* delegate);
5142+
51015143
/** \brief Get the hardware device type.
51025144
*
51035145
* \param[in] device The OrtHardwareDevice instance to query.

‎include/onnxruntime/core/session/onnxruntime_cxx_api.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1085,19 +1085,27 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
10851085
SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT
10861086
SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT
10871087
SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX
1088-
///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN
1088+
/// Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN
10891089
SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options);
1090-
///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl
1090+
/// Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl
10911091
SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options);
10921092
/// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK.
10931093
SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name,
10941094
const std::unordered_map<std::string, std::string>& provider_options = {});
10951095

1096+
/// Append EPs that have been registered previously with the OrtEnv.
1097+
/// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2
10961098
SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
10971099
const KeyValuePairs& ep_options);
1100+
/// Append EPs that have been registered previously with the OrtEnv.
1101+
/// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2
10981102
SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector<ConstEpDevice>& ep_devices,
10991103
const std::unordered_map<std::string, std::string>& ep_options);
11001104

1105+
/// Wraps OrtApi::SessionOptionsSetEpSelectionPolicy
1106+
SessionOptionsImpl& SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy,
1107+
EpSelectionDelegate* delegate = nullptr);
1108+
11011109
SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn
11021110
SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions
11031111
SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn

‎include/onnxruntime/core/session/onnxruntime_cxx_inline.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1149,6 +1149,13 @@ inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AppendExecutionProvider_V2(
11491149
return *this;
11501150
}
11511151

1152+
template <typename T>
1153+
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy,
1154+
EpSelectionDelegate* delegate) {
1155+
ThrowOnError(GetApi().SessionOptionsSetEpSelectionPolicy(this->p_, policy, delegate));
1156+
return *this;
1157+
}
1158+
11521159
template <typename T>
11531160
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) {
11541161
ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn));

‎onnxruntime/core/common/cpuid_info.cc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ void CPUIDInfo::X86Init() {
106106
GetCPUID(0, data);
107107

108108
vendor_ = GetX86Vendor(data);
109+
vendor_id_ = GetVendorId(vendor_);
109110

110111
int num_IDs = data[0];
111112
if (num_IDs >= 1) {
@@ -151,6 +152,14 @@ std::string CPUIDInfo::GetX86Vendor(int32_t* data) {
151152

152153
#endif // defined(CPUIDINFO_ARCH_X86)
153154

155+
uint32_t CPUIDInfo::GetVendorId(const std::string& vendor) {
156+
if (vendor == "GenuineIntel") return 0x8086;
157+
if (vendor == "GenuineAMD") return 0x1022;
158+
if (vendor.find("Qualcomm") == 0) return 'Q' << 24 | 'C' << 16 | 'O' << 8 | 'M';
159+
if (vendor.find("NV") == 0) return 0x10DE;
160+
return 0;
161+
}
162+
154163
#if defined(CPUIDINFO_ARCH_ARM)
155164

156165
#if defined(__linux__)
@@ -204,6 +213,7 @@ void CPUIDInfo::ArmLinuxInit() {
204213
void CPUIDInfo::ArmWindowsInit() {
205214
// Get the ARM vendor string from the registry
206215
vendor_ = GetArmWindowsVendor();
216+
vendor_id_ = GetVendorId(vendor_);
207217

208218
// Read MIDR and ID_AA64ISAR1_EL1 register values from Windows registry
209219
// There should be one per CPU

‎onnxruntime/core/common/cpuid_info.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ class CPUIDInfo {
1919
return vendor_;
2020
}
2121

22+
uint32_t GetCPUVendorId() const {
23+
return vendor_id_;
24+
}
25+
2226
bool HasAMX_BF16() const { return has_amx_bf16_; }
2327
bool HasAVX() const { return has_avx_; }
2428
bool HasAVX2() const { return has_avx2_; }
@@ -123,6 +127,9 @@ class CPUIDInfo {
123127
bool has_arm_neon_bf16_{false};
124128

125129
std::string vendor_;
130+
uint32_t vendor_id_;
131+
132+
uint32_t GetVendorId(const std::string& vendor);
126133

127134
#if defined(CPUIDINFO_ARCH_X86)
128135

‎onnxruntime/core/framework/graph_partitioner.cc‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,13 @@ static Status CreateEpContextModel(const ExecutionProviders& execution_providers
806806
ORT_RETURN_IF(ep_context_gen_options.error_if_no_compiled_nodes,
807807
"Compiled model does not contain any EPContext nodes. "
808808
"Check that the session EPs support compilation and can execute at least one model subgraph.");
809-
return Status::OK();
809+
810+
LOGS(logger, WARNING) << "Compiled model does not contain any EPContext nodes. "
811+
"Either the session EPs do not support compilation or "
812+
"no subgraphs were able to be compiled.";
813+
814+
// we continue on to generate the compiled model which may benefit from L1 optimizations even if there are not
815+
// EPContext nodes.
810816
}
811817

812818
auto get_ep_context_node = [&all_ep_context_nodes](const std::string& node_name) -> std::pair<bool, const Node*> {

‎onnxruntime/core/framework/session_options.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,15 @@ struct EpContextModelGenerationOptions {
9090
size_t output_external_initializer_size_threshold = 0;
9191
};
9292

93+
struct EpSelectionPolicy {
94+
// flag to detect that a policy was set by the user.
95+
// need to preserve current behavior of defaulting to CPU EP if no EPs are explicitly registered
96+
// and no selection policy was explicitly provided.
97+
bool enable{false};
98+
OrtExecutionProviderDevicePolicy policy = OrtExecutionProviderDevicePolicy_DEFAULT;
99+
EpSelectionDelegate* delegate{};
100+
};
101+
93102
/**
94103
* Configuration information for a session.
95104
*/
@@ -222,6 +231,11 @@ struct SessionOptions {
222231
// copied internally and the flag needs to be accessible across all copies.
223232
std::shared_ptr<std::atomic_bool> load_cancellation_flag = std::make_shared<std::atomic_bool>(false);
224233

234+
// Policy to guide Execution Provider selection
235+
EpSelectionPolicy ep_selection_policy = {false,
236+
OrtExecutionProviderDevicePolicy::OrtExecutionProviderDevicePolicy_DEFAULT,
237+
nullptr};
238+
225239
// Options for generating compile EPContext models were previously stored in session_option.configs as
226240
// string key/value pairs. To support more advanced options, such as setting input/output buffers, we
227241
// now have to store EPContext options in a struct of type EpContextModelGenerationOptions.
@@ -253,6 +267,7 @@ inline std::ostream& operator<<(std::ostream& os, const SessionOptions& session_
253267
<< " use_per_session_threads:" << session_options.use_per_session_threads
254268
<< " thread_pool_allow_spinning:" << session_options.thread_pool_allow_spinning
255269
<< " use_deterministic_compute:" << session_options.use_deterministic_compute
270+
<< " ep_selection_policy:" << session_options.ep_selection_policy.policy
256271
<< " config_options: { " << session_options.config_options << " }"
257272
//<< " initializers_to_share_map:" << session_options.initializers_to_share_map
258273
#if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_EXTERNAL_INITIALIZERS)

‎onnxruntime/core/platform/windows/device_discovery.cc‎

Lines changed: 50 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,13 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoSetupApi(const std::unorde
119119

120120
uint32_t vendor_id = get_id(buffer, L"VEN_");
121121
uint32_t device_id = get_id(buffer, L"DEV_");
122-
// won't always have a vendor id from an ACPI entry. need at least a device id to identify the hardware
122+
123+
// Processor ID should come from CPUID mapping.
124+
if (vendor_id == 0 && guid == GUID_DEVCLASS_PROCESSOR) {
125+
vendor_id = CPUIDInfo::GetCPUIDInfo().GetCPUVendorId();
126+
}
127+
128+
// Won't always have a vendor id from an ACPI entry. ACPI is not defined for this purpose.
123129
if (vendor_id == 0 && device_id == 0) {
124130
continue;
125131
}
@@ -138,8 +144,8 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoSetupApi(const std::unorde
138144
entry = &device_info[key];
139145
entry->vendor_id = vendor_id;
140146
entry->device_id = device_id;
141-
// put the first hardware id string in the metadata. ignore the other lines.
142-
entry->metadata.emplace(L"SPDRP_HARDWAREID", std::wstring(buffer, wcslen(buffer)));
147+
// put the first hardware id string in the metadata. ignore the other lines. not sure if this is of value.
148+
// entry->metadata.emplace(L"SPDRP_HARDWAREID", std::wstring(buffer, wcslen(buffer)));
143149
} else {
144150
// need valid ids
145151
continue;
@@ -156,22 +162,22 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoSetupApi(const std::unorde
156162
(PBYTE)buffer, sizeof(buffer), &size)) {
157163
std::wstring desc{buffer};
158164

159-
// Should we require the NPU to be found by DXCore or do we want to allow this vague matching?
160-
// Probably depends on whether we always attempt to run DXCore or not.
161-
const auto possible_npu = [](const std::wstring& desc) {
162-
return (desc.find(L"NPU") != std::wstring::npos ||
163-
desc.find(L"Neural") != std::wstring::npos ||
164-
desc.find(L"AI Engine") != std::wstring::npos ||
165-
desc.find(L"VPU") != std::wstring::npos);
166-
};
165+
// For now, require dxcore to identify an NPU.
166+
// If we want to try and infer it from the description this _may_ work but is untested.
167+
// const auto possible_npu = [](const std::wstring& desc) {
168+
// return (desc.find(L"NPU") != std::wstring::npos ||
169+
// desc.find(L"Neural") != std::wstring::npos ||
170+
// desc.find(L"AI Engine") != std::wstring::npos ||
171+
// desc.find(L"VPU") != std::wstring::npos);
172+
// };
167173

168174
// use description if no friendly name
169175
if (entry->description.empty()) {
170176
entry->description = desc;
171177
}
172178

173179
uint64_t npu_key = GetDeviceKey(*entry);
174-
bool is_npu = npus.count(npu_key) > 0 || possible_npu(desc);
180+
bool is_npu = npus.count(npu_key) > 0; // rely on dxcore to determine if something is an NPU
175181

176182
if (guid == GUID_DEVCLASS_DISPLAY) {
177183
entry->type = OrtHardwareDeviceType_GPU;
@@ -194,22 +200,17 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoSetupApi(const std::unorde
194200
continue;
195201
}
196202

197-
if (SetupDiGetDeviceRegistryPropertyW(devInfo, &devData, SPDRP_MFG, nullptr,
198-
(PBYTE)buffer, sizeof(buffer), &size)) {
199-
entry->vendor = std::wstring(buffer, wcslen(buffer));
203+
if (entry->type == OrtHardwareDeviceType_CPU) {
204+
// get 12 byte string from CPUID. easier for a user to match this if they are explicitly picking a device.
205+
std::string_view cpuid_vendor = CPUIDInfo::GetCPUIDInfo().GetCPUVendor();
206+
entry->vendor = std::wstring(cpuid_vendor.begin(), cpuid_vendor.end());
200207
}
201208

202-
// Add the UI number if GPU. Helpful if user has integrated and discrete GPUs
203-
if (entry->type == OrtHardwareDeviceType_GPU) {
204-
DWORD ui_number = 0;
205-
if (SetupDiGetDeviceRegistryPropertyW(devInfo, &devData, SPDRP_UI_NUMBER, nullptr,
206-
(PBYTE)&ui_number, sizeof(ui_number), &size)) {
207-
// use value read.
208-
} else {
209-
// infer it as 0 if not set.
209+
if (entry->vendor.empty()) {
210+
if (SetupDiGetDeviceRegistryPropertyW(devInfo, &devData, SPDRP_MFG, nullptr,
211+
(PBYTE)buffer, sizeof(buffer), &size)) {
212+
entry->vendor = std::wstring(buffer, wcslen(buffer));
210213
}
211-
212-
entry->metadata.emplace(L"SPDRP_UI_NUMBER", std::to_wstring(ui_number));
213214
}
214215
}
215216

@@ -252,9 +253,7 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoD3D12() {
252253
info.description = std::wstring(desc.Description);
253254

254255
info.metadata[L"DxgiAdapterNumber"] = std::to_wstring(i);
255-
info.metadata[L"VideoMemory"] = std::to_wstring(desc.DedicatedVideoMemory / (1024 * 1024)) + L" MB";
256-
info.metadata[L"SystemMemory"] = std::to_wstring(desc.DedicatedSystemMemory / (1024 * 1024)) + L" MB";
257-
info.metadata[L"SharedSystemMemory"] = std::to_wstring(desc.DedicatedSystemMemory / (1024 * 1024)) + L" MB";
256+
info.metadata[L"DxgiVideoMemory"] = std::to_wstring(desc.DedicatedVideoMemory / (1024 * 1024)) + L" MB";
258257
}
259258

260259
// iterate by high-performance GPU preference to add that info
@@ -272,7 +271,7 @@ std::unordered_map<uint64_t, DeviceInfo> GetDeviceInfoD3D12() {
272271
auto it = device_info.find(key);
273272
if (it != device_info.end()) {
274273
DeviceInfo& info = it->second;
275-
info.metadata[L"HighPerformanceIndex"] = std::to_wstring(i);
274+
info.metadata[L"DxgiHighPerformanceIndex"] = std::to_wstring(i);
276275
}
277276
}
278277

@@ -405,32 +404,48 @@ std::unordered_set<OrtHardwareDevice> DeviceDiscovery::DiscoverDevicesForPlatfor
405404
}
406405
}
407406

408-
std::wstring_convert<std::codecvt_utf8<wchar_t> > converter; // wstring to string
409-
const auto device_to_ortdevice = [&converter](
407+
// our log output to std::wclog breaks with UTF8 chars that are not supported by the current code page.
408+
// e.g. (TM) symbol. that stops ALL logging working on at least arm64.
409+
// safest way to avoid that is to keep it to single byte chars.
410+
// process the OrtHardwareDevice values this way so it can be safely logged.
411+
// only the 'description' metadata is likely to be affected and that is mainly for diagnostic purposes.
412+
const auto to_safe_string = [](const std::wstring& wstr) -> std::string {
413+
std::string str(wstr.size(), ' ');
414+
std::transform(wstr.begin(), wstr.end(), str.begin(), [](wchar_t wchar) {
415+
if (wchar >= 0 && wchar <= 127) {
416+
return static_cast<char>(wchar);
417+
}
418+
return ' ';
419+
});
420+
return str;
421+
};
422+
423+
const auto device_to_ortdevice = [&to_safe_string](
410424
DeviceInfo& device,
411425
std::unordered_map<std::wstring, std::wstring>* extra_metadata = nullptr) {
412-
OrtHardwareDevice ortdevice{device.type, device.vendor_id, device.device_id, converter.to_bytes(device.vendor)};
426+
OrtHardwareDevice ortdevice{device.type, device.vendor_id, device.device_id, to_safe_string(device.vendor)};
413427

414428
if (!device.description.empty()) {
415-
ortdevice.metadata.Add("Description", converter.to_bytes(device.description));
429+
ortdevice.metadata.Add("Description", to_safe_string(device.description));
416430
}
417431

418432
for (auto& [key, value] : device.metadata) {
419-
ortdevice.metadata.Add(converter.to_bytes(key), converter.to_bytes(value));
433+
ortdevice.metadata.Add(to_safe_string(key), to_safe_string(value));
420434
}
421435

422436
if (extra_metadata) {
423437
// add any extra metadata from the dxcore info
424438
for (auto& [key, value] : *extra_metadata) {
425439
if (device.metadata.find(key) == device.metadata.end()) {
426-
ortdevice.metadata.Add(converter.to_bytes(key), converter.to_bytes(value));
440+
ortdevice.metadata.Add(to_safe_string(key), to_safe_string(value));
427441
}
428442
}
429443
}
430444

431445
std::ostringstream oss;
432446
oss << "Adding OrtHardwareDevice {vendor_id:0x" << std::hex << ortdevice.vendor_id
433447
<< ", device_id:0x" << ortdevice.device_id
448+
<< ", vendor:" << ortdevice.vendor
434449
<< ", type:" << std::dec << static_cast<int>(ortdevice.type)
435450
<< ", metadata: [";
436451
for (auto& [key, value] : ortdevice.metadata.entries) {

‎onnxruntime/core/session/abi_key_value_pairs.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ struct OrtKeyValuePairs {
5757
keys.erase(key_iter);
5858
values.erase(values.begin() + idx);
5959
}
60+
61+
entries.erase(iter);
6062
}
6163
}
6264

0 commit comments

Comments
 (0)