Skip to content

Commit b652e59

Browse files
authored
[WebGPU] Prepack Conv weights for the im2col-matmul path (#32420)
### Description <!-- Describe your changes. --> The im2col-matmul Conv path consumes weights in OHWI layout, but the Conv kernel receives them as OIHW, so `ApplyIm2ColMatMulProgram` transposed them on every inference. This PR moves that transpose into `Conv::PrePackInternal` so it runs once at session initialization. ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> The weight transpose was marked `// TODO: Use prepack` and was re-dispatched on every Conv execution even though the weights are constant. This removes a redundant GPU dispatch and a transient buffer allocation per Conv per inference on the im2col-matmul path. ### Testing Verified on Intel Panther Lake (xe-3lpg) - `ConvFp16Test.Conv2D_1` - `ConvFp16Test.Conv2D_KleidiAiImatmulEligibleNoBias` - `ConvFp16Test.Conv2D_KleidiAiImatmulEligibleBiasAndDisabledFallback` - `ConvFp16Test.Conv2D_Bias_1` - `ConvFp16Test.Conv2D_Bias_2` - `ConvFp16Test.Conv2D_AutoPad1` - `ConvFp16Test.Conv2D_AutoPad2` - `ConvFp16Test.Conv_AutoPad_with_non_default_strides` The `AutoPad` cases specifically cover prepacking ahead of the `auto_pad != NOTSET` early return, and the bias/no-bias pairs cover both input-count variants.
1 parent 0f0f29f commit b652e59

4 files changed

Lines changed: 125 additions & 39 deletions

File tree

‎onnxruntime/core/providers/webgpu/nn/conv.cc‎

Lines changed: 62 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,22 @@ template <bool is_channels_last, bool is_fused>
3131
Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context) const {
3232
bool has_bias = context.InputCount() > 2;
3333
const auto* input = context.Input<Tensor>(0);
34-
const Tensor* kernel = nullptr;
35-
bool kernel_is_prepacked = false;
36-
if (transposed_kernel_) {
37-
kernel = transposed_kernel_.get();
38-
kernel_is_prepacked = true;
39-
} else {
40-
kernel = context.Input<Tensor>(1);
41-
}
34+
const Tensor* kernel = prepacked_kernel_ ? prepacked_kernel_.get() : context.Input<Tensor>(1);
4235
const auto* bias = has_bias ? context.Input<Tensor>(2) : nullptr;
4336
TensorShape input_shape = input->Shape();
4437
ORT_ENFORCE(kernel != nullptr, "Conv kernel tensor is required.");
45-
TensorShape kernel_shape = kernel_is_prepacked
46-
? TensorShape(TensorShapeVector{kernel->Shape()[3], kernel->Shape()[2], kernel->Shape()[0], kernel->Shape()[1]})
47-
: kernel->Shape();
38+
// Prepacked kernels are stored permuted; recover the logical OIHW shape.
39+
TensorShape kernel_shape = kernel->Shape();
40+
switch (kernel_layout_) {
41+
case KernelLayout::OIHW:
42+
break;
43+
case KernelLayout::HWIO:
44+
kernel_shape = TensorShape(TensorShapeVector{kernel_shape[3], kernel_shape[2], kernel_shape[0], kernel_shape[1]});
45+
break;
46+
case KernelLayout::OHWI:
47+
kernel_shape = TensorShape(TensorShapeVector{kernel_shape[0], kernel_shape[3], kernel_shape[1], kernel_shape[2]});
48+
break;
49+
}
4850
ConvAttributes::ConvPadVector local_pads(conv_attrs_.pads.begin(), conv_attrs_.pads.end());
4951
TensorShapeVector local_dilations(conv_attrs_.dilations.begin(), conv_attrs_.dilations.end());
5052
TensorShapeVector local_strides(conv_attrs_.strides.begin(), conv_attrs_.strides.end());
@@ -168,20 +170,37 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
168170
kernel_shape,
169171
onnxruntime::narrow<uint32_t>(conv_attrs_.group),
170172
kernel->DataType())) {
173+
// A prepacked kernel must be OHWI here. If it were packed for another consumer, the
174+
// argument below would be null and ApplyIm2ColMatMulProgram would fall back to
175+
// transposing input 1 -- which PrePackInternal already had ORT release.
176+
ORT_ENFORCE(!prepacked_kernel_ || kernel_layout_ == KernelLayout::OHWI,
177+
"Im2ColMatMul path reached with a kernel prepacked for a different layout.");
171178
return ApplyIm2ColMatMulProgram(context,
172179
is_channels_last,
173180
activation_,
174181
dilations,
175182
pads,
176183
strides,
184+
kernel_layout_ == KernelLayout::OHWI ? kernel : nullptr,
177185
output);
178186
}
179187

188+
// The OHWI layout is only understood by the im2col path above. Reaching here with it
189+
// would mean PrePackInternal and ComputeInternal disagree on whether the im2col path
190+
// applies, and the branches below -- which expect either OIHW or HWIO -- would
191+
// silently misread the layout.
192+
ORT_ENFORCE(kernel_layout_ != KernelLayout::OHWI,
193+
"Kernel was prepacked as OHWI but the Im2ColMatMul path was not taken.");
194+
195+
// Every remaining consumer wants HWIO, so the kernel has to be transposed unless
196+
// PrePackInternal already produced that layout.
197+
const bool kernel_needs_transpose = kernel_layout_ != KernelLayout::HWIO;
198+
180199
if (conv_attrs_.group > 1) {
181200
Tensor transposed_kernel;
182201
if (is_channels_last) {
183202
const Tensor* grouped_kernel = kernel;
184-
if (!kernel_is_prepacked) {
203+
if (kernel_needs_transpose) {
185204
ORT_RETURN_IF_ERROR(TransposeKernel(context, kernel, kernel_shape, &transposed_kernel, perm));
186205
grouped_kernel = &transposed_kernel;
187206
}
@@ -218,7 +237,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
218237
if (is_channels_last) {
219238
// Transpose weights
220239
const Tensor* matmul_kernel = kernel;
221-
if (!kernel_is_prepacked) {
240+
if (kernel_needs_transpose) {
222241
ORT_RETURN_IF_ERROR(TransposeKernel(context, kernel, kernel_shape, &transposed_kernel, perm));
223242
matmul_kernel = &transposed_kernel;
224243
}
@@ -240,7 +259,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
240259
matmul_inputs.push_back(input);
241260
}
242261
const bool matmul_b_is_constant =
243-
is_channels_last && transposed_kernel_ != nullptr && matmul_inputs[1] == transposed_kernel_.get();
262+
is_channels_last && prepacked_kernel_ != nullptr && matmul_inputs[1] == prepacked_kernel_.get();
244263
Tensor matmul_a = CreateTensorView(*matmul_inputs[0], matmul_a_shape);
245264
Tensor matmul_b = CreateTensorView(*matmul_inputs[1], matmul_b_shape);
246265
matmul_inputs[0] = &matmul_a;
@@ -254,7 +273,7 @@ Status Conv<is_channels_last, is_fused>::ComputeInternal(ComputeContext& context
254273
// Transpose weights when necessary
255274
Tensor transposed_kernel;
256275
const Tensor* conv_kernel = kernel;
257-
if (!kernel_is_prepacked) {
276+
if (kernel_needs_transpose) {
258277
ORT_RETURN_IF_ERROR(TransposeKernel(context, kernel, kernel_shape, &transposed_kernel, perm));
259278
conv_kernel = &transposed_kernel;
260279
}
@@ -289,6 +308,30 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
289308
return Status::OK();
290309
}
291310

311+
// Im2ColMatMul path: transpose OIHW -> OHWI once here instead of on every inference.
312+
//
313+
// Placed before the auto_pad check below on purpose:
314+
// - Safe: CanApplyIm2ColMatMulProgram() only looks at the adapter, dtype, layout,
315+
// fusion, group and kernel H/W -- never at pads -- and ComputeInternal tests it
316+
// before every pads-dependent branch. So a true here means the im2col path is
317+
// taken at runtime no matter what the padding turns out to be.
318+
// - Necessary: otherwise every auto_pad != NOTSET model would bail out below and
319+
// keep paying for the transpose on every inference.
320+
//
321+
// This call and the one in ComputeInternal must stay in agreement: only the im2col
322+
// path can read the OHWI layout, so a decision made here that ComputeInternal later
323+
// reverses would corrupt the weights. If CanApplyIm2ColMatMulProgram() ever gains a
324+
// condition that is not known at prepack time (pads, strides, input shape), this
325+
// shortcut must go away. ComputeInternal ORT_ENFORCEs the invariant.
326+
if (CanApplyIm2ColMatMulProgram(context, is_channels_last, activation_,
327+
kernel_shape, onnxruntime::narrow<uint32_t>(conv_attrs_.group),
328+
tensor.DataType())) {
329+
ORT_RETURN_IF_ERROR(PrePackIm2ColMatMulWeight(context, tensor, alloc, prepacked_kernel_));
330+
kernel_layout_ = KernelLayout::OHWI;
331+
is_packed = true; // set this flag to true so that ORT will release the initializer tensor
332+
return Status::OK();
333+
}
334+
292335
// Grouped convolution (group > 1):
293336
// - Only transposes when is_channels_last
294337
// - channels_first: no transpose
@@ -308,18 +351,9 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
308351
return Status::OK();
309352
}
310353

311-
// Im2ColMatMul path uses a different transpose (OIHW -> OHWI) and reads
312-
// kernel directly from context.Input(1), ignoring prepacked weights.
313-
// Skip prepacking when this path will be used at runtime.
314-
if (CanApplyIm2ColMatMulProgram(context, is_channels_last, activation_,
315-
kernel_shape, onnxruntime::narrow<uint32_t>(conv_attrs_.group),
316-
tensor.DataType())) {
317-
return Status::OK();
318-
}
319-
320354
// Analyze execution paths in ComputeInternal to determine if kernel transpose is needed:
321355
//
322-
// 1. Im2ColMatMul path: handled above (skip prepacking)
356+
// 1. Im2ColMatMul path: handled above (prepacked as OHWI)
323357
// 2. Grouped conv (group > 1): handled above (skip if !is_channels_last)
324358
// 3. MatMul optimization (same_size || is_1x1_conv):
325359
// - is_channels_last: transposes
@@ -369,11 +403,12 @@ Status Conv<is_channels_last, is_fused>::PrePackInternal(ComputeContextBase& con
369403

370404
// Create the transposed kernel tensor using the prepack allocator.
371405
// This allocator creates GPU buffers without mapping, suitable for GPU-based operations.
372-
transposed_kernel_ = std::make_unique<Tensor>(tensor.DataType(), transposed_kernel_shape, alloc);
406+
prepacked_kernel_ = std::make_unique<Tensor>(tensor.DataType(), transposed_kernel_shape, alloc);
373407

374408
// Perform GPU-based transpose directly from the input GPU tensor
375-
ORT_RETURN_IF_ERROR(Transpose::DoTranspose(context, perm, tensor, *transposed_kernel_));
409+
ORT_RETURN_IF_ERROR(Transpose::DoTranspose(context, perm, tensor, *prepacked_kernel_));
376410

411+
kernel_layout_ = KernelLayout::HWIO;
377412
is_packed = true; // set this flag to true so that ORT will release the initializer tensor
378413

379414
return Status::OK();

‎onnxruntime/core/providers/webgpu/nn/conv.h‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,16 @@
1414
namespace onnxruntime {
1515
namespace webgpu {
1616

17+
// Layout of the kernel tensor that ComputeInternal consumes. `OIHW` is the layout the
18+
// Conv operator is defined with; the others are produced by PrePackInternal and are each
19+
// understood by exactly one consumer, so the layout has to be tracked explicitly rather
20+
// than inferred from which prepacked tensor happens to be present.
21+
enum class KernelLayout {
22+
OIHW, // No prepacked tensor -- the kernel is read straight from input 1.
23+
HWIO, // Consumed by grouped conv, the 1x1/same_size MatMul path and Conv2dMM.
24+
OHWI, // Consumed by the Im2ColMatMul path only.
25+
};
26+
1727
template <bool is_channels_last, bool is_fused>
1828
class Conv : public WebGpuKernel {
1929
public:
@@ -33,8 +43,13 @@ class Conv : public WebGpuKernel {
3343
protected:
3444
ConvAttributes conv_attrs_;
3545
Activation activation_;
46+
// Set by PrePackInternal; null when the kernel could not be prepacked (e.g. the weight
47+
// is not a constant initializer), in which case ComputeInternal reads input 1 instead.
48+
std::unique_ptr<Tensor> prepacked_kernel_;
49+
// Layout of the tensor ComputeInternal ends up consuming -- `prepacked_kernel_` when it
50+
// is set, otherwise input 1. Stays `OIHW` while `prepacked_kernel_` is null.
51+
KernelLayout kernel_layout_{KernelLayout::OIHW};
3652
mutable MatMulOptImplCache matmul_compute_cache_;
37-
std::unique_ptr<Tensor> transposed_kernel_; // should only have value when `is_initializer` AND `is_4D` AND `is_NHWC`
3853
};
3954

4055
Status TransposeKernel(ComputeContext& context, const Tensor* kernel, const TensorShape& kernel_shape, Tensor* transposed_kernel, const InlinedVector<size_t>& perm);

‎onnxruntime/core/providers/webgpu/nn/im2col_matmul.cc‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "core/providers/webgpu/nn/im2col_matmul.h"
99
#include "core/providers/webgpu/nn/conv.h"
1010
#include "core/providers/webgpu/nn/activation_util.h"
11+
#include "core/providers/webgpu/tensor/transpose.h"
1112

1213
namespace onnxruntime {
1314
namespace webgpu {
@@ -70,8 +71,26 @@ bool IsActivationSupported(const Activation& activation) {
7071
}
7172
}
7273

74+
// The weight layout consumed by Im2ColMatMulProgram: OIHW -> OHWI.
75+
const InlinedVector<size_t>& OihwToOhwiPerm() {
76+
static const InlinedVector<size_t> perm = {0, 2, 3, 1};
77+
return perm;
78+
}
79+
7380
} // namespace
7481

82+
Status PrePackIm2ColMatMulWeight(ComputeContextBase& context,
83+
const Tensor& weight,
84+
AllocatorPtr alloc,
85+
std::unique_ptr<Tensor>& packed_weight) {
86+
const TensorShape& weight_shape = weight.Shape();
87+
ORT_RETURN_IF_NOT(weight_shape.NumDimensions() == 4, "Im2ColMatMul weight must be 4D (OIHW).");
88+
89+
TensorShape ohwi_shape({weight_shape[0], weight_shape[2], weight_shape[3], weight_shape[1]});
90+
packed_weight = std::make_unique<Tensor>(weight.DataType(), ohwi_shape, alloc);
91+
return Transpose::DoTranspose(context, OihwToOhwiPerm(), weight, *packed_weight);
92+
}
93+
7594
// The template dispatches on the numeric enum values.
7695
static_assert(static_cast<int>(ActivationKind::None) == 0, "im2col_matmul.wgsl.template mirrors ActivationKind");
7796
static_assert(static_cast<int>(ActivationKind::Relu) == 1, "im2col_matmul.wgsl.template mirrors ActivationKind");
@@ -111,22 +130,27 @@ Status ApplyIm2ColMatMulProgram(ComputeContext& context,
111130
const std::vector<uint32_t>& dilations,
112131
const std::vector<uint32_t>& pads,
113132
const std::vector<uint32_t>& strides,
133+
const Tensor* packed_weight,
114134
Tensor* output) {
115135
const auto* src = context.Input<Tensor>(0);
116-
const auto* weight = context.Input<Tensor>(1);
117136
const bool has_bias = context.InputCount() > 2;
118137
const auto* bias = has_bias ? context.Input<Tensor>(2) : nullptr;
119138

120-
TensorShape weight_shape = weight->Shape();
121-
const uint32_t channel_output = onnxruntime::narrow<uint32_t>(weight_shape[0]);
122-
const uint32_t channel_input = onnxruntime::narrow<uint32_t>(weight_shape[1]);
123-
const uint32_t kernel_height = onnxruntime::narrow<uint32_t>(weight_shape[2]);
124-
const uint32_t kernel_width = onnxruntime::narrow<uint32_t>(weight_shape[3]);
139+
// The weight is expected in OHWI layout. Prefer the prepacked one; otherwise
140+
// transpose OIHW -> OHWI on the fly (e.g. when the weight is not an initializer).
141+
Tensor transposed_weight;
142+
const Tensor* ohwi_weight = packed_weight;
143+
if (ohwi_weight == nullptr) {
144+
const auto* weight = context.Input<Tensor>(1);
145+
ORT_RETURN_IF_ERROR(TransposeKernel(context, weight, weight->Shape(), &transposed_weight, OihwToOhwiPerm()));
146+
ohwi_weight = &transposed_weight;
147+
}
125148

126-
// Transpose OIHW Weight to OHWI
127-
// TODO: Use prepack
128-
Tensor ohwi_weight;
129-
ORT_RETURN_IF_ERROR(TransposeKernel(context, weight, weight->Shape(), &ohwi_weight, {0, 2, 3, 1}));
149+
const TensorShape& ohwi_shape = ohwi_weight->Shape();
150+
const uint32_t channel_output = onnxruntime::narrow<uint32_t>(ohwi_shape[0]);
151+
const uint32_t kernel_height = onnxruntime::narrow<uint32_t>(ohwi_shape[1]);
152+
const uint32_t kernel_width = onnxruntime::narrow<uint32_t>(ohwi_shape[2]);
153+
const uint32_t channel_input = onnxruntime::narrow<uint32_t>(ohwi_shape[3]);
130154

131155
// im2col-matmul
132156
const TensorShape src_shape = src->Shape();
@@ -163,7 +187,7 @@ Status ApplyIm2ColMatMulProgram(ComputeContext& context,
163187
im2col_mm_program.AddInput({src,
164188
ProgramTensorMetadataDependency::TypeAndRank,
165189
static_cast<int>(vec_size)});
166-
im2col_mm_program.AddInput({&ohwi_weight,
190+
im2col_mm_program.AddInput({ohwi_weight,
167191
ProgramTensorMetadataDependency::TypeAndRank,
168192
static_cast<int>(vec_size)});
169193
if (has_bias) {

‎onnxruntime/core/providers/webgpu/nn/im2col_matmul.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#pragma once
55

6+
#include <memory>
67
#include <vector>
78

89
#include "core/framework/tensor_shape.h"
@@ -72,12 +73,23 @@ bool CanApplyIm2ColMatMulProgram(ComputeContextBase& context,
7273
const uint32_t group,
7374
const MLDataType data_type);
7475

76+
// Transposes the OIHW weight into the OHWI layout expected by Im2ColMatMulProgram.
77+
// Called from Conv::PrePackInternal so the transpose runs once at session
78+
// initialization instead of on every inference.
79+
Status PrePackIm2ColMatMulWeight(ComputeContextBase& context,
80+
const Tensor& weight,
81+
AllocatorPtr alloc,
82+
/*out*/ std::unique_ptr<Tensor>& packed_weight);
83+
84+
// `packed_weight` is the OHWI weight produced by PrePackIm2ColMatMulWeight. When it
85+
// is nullptr, the OIHW weight is read from input 1 and transposed on the fly.
7586
Status ApplyIm2ColMatMulProgram(ComputeContext& context,
7687
const bool is_channels_last,
7788
const Activation& activation,
7889
const std::vector<uint32_t>& dilations,
7990
const std::vector<uint32_t>& pads,
8091
const std::vector<uint32_t>& strides,
92+
const Tensor* packed_weight,
8193
Tensor* output);
8294

8395
} // namespace webgpu

0 commit comments

Comments
 (0)