@@ -31,20 +31,22 @@ template <bool is_channels_last, bool is_fused>
3131Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context) const {
3232 bool has_bias = context.InputCount () > 2 ;
3333 const auto * input = context.Input <Tensor>(0 );
34- const Tensor* kernel = nullptr ;
35- bool kernel_is_prepacked = false ;
36- if (transposed_kernel_) {
37- kernel = transposed_kernel_.get ();
38- kernel_is_prepacked = true ;
39- } else {
40- kernel = context.Input <Tensor>(1 );
41- }
34+ const Tensor* kernel = prepacked_kernel_ ? prepacked_kernel_.get () : context.Input <Tensor>(1 );
4235 const auto * bias = has_bias ? context.Input <Tensor>(2 ) : nullptr ;
4336 TensorShape input_shape = input->Shape ();
4437 ORT_ENFORCE (kernel != nullptr , " Conv kernel tensor is required." );
45- TensorShape kernel_shape = kernel_is_prepacked
46- ? TensorShape (TensorShapeVector{kernel->Shape ()[3 ], kernel->Shape ()[2 ], kernel->Shape ()[0 ], kernel->Shape ()[1 ]})
47- : kernel->Shape ();
38+ // Prepacked kernels are stored permuted; recover the logical OIHW shape.
39+ TensorShape kernel_shape = kernel->Shape ();
40+ switch (kernel_layout_) {
41+ case KernelLayout::OIHW :
42+ break ;
43+ case KernelLayout::HWIO :
44+ kernel_shape = TensorShape (TensorShapeVector{kernel_shape[3 ], kernel_shape[2 ], kernel_shape[0 ], kernel_shape[1 ]});
45+ break ;
46+ case KernelLayout::OHWI :
47+ kernel_shape = TensorShape (TensorShapeVector{kernel_shape[0 ], kernel_shape[3 ], kernel_shape[1 ], kernel_shape[2 ]});
48+ break ;
49+ }
4850 ConvAttributes::ConvPadVector local_pads (conv_attrs_.pads .begin (), conv_attrs_.pads .end ());
4951 TensorShapeVector local_dilations (conv_attrs_.dilations .begin (), conv_attrs_.dilations .end ());
5052 TensorShapeVector local_strides (conv_attrs_.strides .begin (), conv_attrs_.strides .end ());
@@ -168,20 +170,37 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
168170 kernel_shape,
169171 onnxruntime::narrow<uint32_t >(conv_attrs_.group ),
170172 kernel->DataType ())) {
173+ // A prepacked kernel must be OHWI here. If it were packed for another consumer, the
174+ // argument below would be null and ApplyIm2ColMatMulProgram would fall back to
175+ // transposing input 1 -- which PrePackInternal already had ORT release.
176+ ORT_ENFORCE (!prepacked_kernel_ || kernel_layout_ == KernelLayout::OHWI ,
177+ " Im2ColMatMul path reached with a kernel prepacked for a different layout." );
171178 return ApplyIm2ColMatMulProgram (context,
172179 is_channels_last,
173180 activation_,
174181 dilations,
175182 pads,
176183 strides,
184+ kernel_layout_ == KernelLayout::OHWI ? kernel : nullptr ,
177185 output);
178186 }
179187
188+ // The OHWI layout is only understood by the im2col path above. Reaching here with it
189+ // would mean PrePackInternal and ComputeInternal disagree on whether the im2col path
190+ // applies, and the branches below -- which expect either OIHW or HWIO -- would
191+ // silently misread the layout.
192+ ORT_ENFORCE (kernel_layout_ != KernelLayout::OHWI ,
193+ " Kernel was prepacked as OHWI but the Im2ColMatMul path was not taken." );
194+
195+ // Every remaining consumer wants HWIO, so the kernel has to be transposed unless
196+ // PrePackInternal already produced that layout.
197+ const bool kernel_needs_transpose = kernel_layout_ != KernelLayout::HWIO ;
198+
180199 if (conv_attrs_.group > 1 ) {
181200 Tensor transposed_kernel;
182201 if (is_channels_last) {
183202 const Tensor* grouped_kernel = kernel;
184- if (!kernel_is_prepacked ) {
203+ if (kernel_needs_transpose ) {
185204 ORT_RETURN_IF_ERROR (TransposeKernel (context, kernel, kernel_shape, &transposed_kernel, perm));
186205 grouped_kernel = &transposed_kernel;
187206 }
@@ -218,7 +237,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
218237 if (is_channels_last) {
219238 // Transpose weights
220239 const Tensor* matmul_kernel = kernel;
221- if (!kernel_is_prepacked ) {
240+ if (kernel_needs_transpose ) {
222241 ORT_RETURN_IF_ERROR (TransposeKernel (context, kernel, kernel_shape, &transposed_kernel, perm));
223242 matmul_kernel = &transposed_kernel;
224243 }
@@ -240,7 +259,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
240259 matmul_inputs.push_back (input);
241260 }
242261 const bool matmul_b_is_constant =
243- is_channels_last && transposed_kernel_ != nullptr && matmul_inputs[1 ] == transposed_kernel_ .get ();
262+ is_channels_last && prepacked_kernel_ != nullptr && matmul_inputs[1 ] == prepacked_kernel_ .get ();
244263 Tensor matmul_a = CreateTensorView (*matmul_inputs[0 ], matmul_a_shape);
245264 Tensor matmul_b = CreateTensorView (*matmul_inputs[1 ], matmul_b_shape);
246265 matmul_inputs[0 ] = &matmul_a;
@@ -254,7 +273,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
254273 // Transpose weights when necessary
255274 Tensor transposed_kernel;
256275 const Tensor* conv_kernel = kernel;
257- if (!kernel_is_prepacked ) {
276+ if (kernel_needs_transpose ) {
258277 ORT_RETURN_IF_ERROR (TransposeKernel (context, kernel, kernel_shape, &transposed_kernel, perm));
259278 conv_kernel = &transposed_kernel;
260279 }
@@ -289,6 +308,30 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
289308 return Status::OK ();
290309 }
291310
311+ // Im2ColMatMul path: transpose OIHW -> OHWI once here instead of on every inference.
312+ //
313+ // Placed before the auto_pad check below on purpose:
314+ // - Safe: CanApplyIm2ColMatMulProgram() only looks at the adapter, dtype, layout,
315+ // fusion, group and kernel H/W -- never at pads -- and ComputeInternal tests it
316+ // before every pads-dependent branch. So a true here means the im2col path is
317+ // taken at runtime no matter what the padding turns out to be.
318+ // - Necessary: otherwise every auto_pad != NOTSET model would bail out below and
319+ // keep paying for the transpose on every inference.
320+ //
321+ // This call and the one in ComputeInternal must stay in agreement: only the im2col
322+ // path can read the OHWI layout, so a decision made here that ComputeInternal later
323+ // reverses would corrupt the weights. If CanApplyIm2ColMatMulProgram() ever gains a
324+ // condition that is not known at prepack time (pads, strides, input shape), this
325+ // shortcut must go away. ComputeInternal ORT_ENFORCEs the invariant.
326+ if (CanApplyIm2ColMatMulProgram (context, is_channels_last, activation_,
327+ kernel_shape, onnxruntime::narrow<uint32_t >(conv_attrs_.group ),
328+ tensor.DataType ())) {
329+ ORT_RETURN_IF_ERROR (PrePackIm2ColMatMulWeight (context, tensor, alloc, prepacked_kernel_));
330+ kernel_layout_ = KernelLayout::OHWI ;
331+ is_packed = true ; // set this flag to true so that ORT will release the initializer tensor
332+ return Status::OK ();
333+ }
334+
292335 // Grouped convolution (group > 1):
293336 // - Only transposes when is_channels_last
294337 // - channels_first: no transpose
@@ -308,18 +351,9 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
308351 return Status::OK ();
309352 }
310353
311- // Im2ColMatMul path uses a different transpose (OIHW -> OHWI) and reads
312- // kernel directly from context.Input(1), ignoring prepacked weights.
313- // Skip prepacking when this path will be used at runtime.
314- if (CanApplyIm2ColMatMulProgram (context, is_channels_last, activation_,
315- kernel_shape, onnxruntime::narrow<uint32_t >(conv_attrs_.group ),
316- tensor.DataType ())) {
317- return Status::OK ();
318- }
319-
320354 // Analyze execution paths in ComputeInternal to determine if kernel transpose is needed:
321355 //
322- // 1. Im2ColMatMul path: handled above (skip prepacking )
356+ // 1. Im2ColMatMul path: handled above (prepacked as OHWI )
323357 // 2. Grouped conv (group > 1): handled above (skip if !is_channels_last)
324358 // 3. MatMul optimization (same_size || is_1x1_conv):
325359 // - is_channels_last: transposes
@@ -369,11 +403,12 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
369403
370404 // Create the transposed kernel tensor using the prepack allocator.
371405 // This allocator creates GPU buffers without mapping, suitable for GPU-based operations.
372- transposed_kernel_ = std::make_unique<Tensor>(tensor.DataType (), transposed_kernel_shape, alloc);
406+ prepacked_kernel_ = std::make_unique<Tensor>(tensor.DataType (), transposed_kernel_shape, alloc);
373407
374408 // Perform GPU-based transpose directly from the input GPU tensor
375- ORT_RETURN_IF_ERROR (Transpose::DoTranspose (context, perm, tensor, *transposed_kernel_ ));
409+ ORT_RETURN_IF_ERROR (Transpose::DoTranspose (context, perm, tensor, *prepacked_kernel_ ));
376410
411+ kernel_layout_ = KernelLayout::HWIO ;
377412 is_packed = true ; // set this flag to true so that ORT will release the initializer tensor
378413
379414 return Status::OK ();
0 commit comments