Skip to content

Commit 86b4bea

Browse files
authored
Use fp32 accumulation in portable/optimized norms (#16727)
Summary: When running in fp16, our layer norm implementation diverges from ATen, especially for larger variances. This is because ATen internally accumulates mean and variance in fp32 for fp16/bf16, whereas we accumulate in the input/output dtype. To resolve this, I've matched the ATen behavior by using acc_t (similar to OpMathType from ATen) in optimized. In portable, I've just removed the cast back to CTYPE, as the accumulation logic was already in float. There was also a straight up bug where the element count was being cast to CTYPE in group norm, which tends to overflow or lose precision. I've added an additional test case with high variance. In combination with the existing coverage, this should show that the behavior is unchanged for existing cases and no longer overflows or diverges for larger inputs. Thanks Jason Zhu for spotting this! Differential Revision: D91096147
1 parent 5b3d9fc commit 86b4bea

8 files changed

Lines changed: 83 additions & 14 deletions

File tree

‎kernels/optimized/cpu/op_native_layer_norm.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,18 +76,18 @@ void layer_norm(
7676
const CTYPE* src_ptr = input_data + i * N;
7777
CTYPE* dst_ptr = out_data + i * N;
7878

79-
CTYPE mean_val;
80-
CTYPE rstd_val;
79+
acc_t<CTYPE> mean_val;
80+
acc_t<CTYPE> rstd_val;
8181
std::tie(mean_val, rstd_val) = RowwiseMoments(src_ptr, N);
8282
rstd_val = CTYPE(1) / std::sqrt(rstd_val + eps);
8383

84-
const CTYPE scale = rstd_val;
85-
const CTYPE offset = -rstd_val * mean_val;
84+
const acc_t<CTYPE> scale = rstd_val;
85+
const acc_t<CTYPE> offset = -rstd_val * mean_val;
8686

8787
if (gamma_null || beta_null) {
8888
for (size_t j = 0; j < N; ++j) {
89-
const CTYPE gamma_v = gamma_null ? CTYPE(1) : gamma_data[j];
90-
const CTYPE beta_v = beta_null ? CTYPE(0) : beta_data[j];
89+
const acc_t<CTYPE> gamma_v = gamma_null ? CTYPE(1) : gamma_data[j];
90+
const acc_t<CTYPE> beta_v = beta_null ? CTYPE(0) : beta_data[j];
9191
dst_ptr[j] = (src_ptr[j] * scale + offset) * gamma_v + beta_v;
9292
}
9393
} else {

‎kernels/optimized/test/targets.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def _lib_test_bin(name, extra_deps = [], in_cpu = False):
2525
deps = [
2626
"//executorch/test/utils:utils",
2727
"//executorch/kernels/optimized{}:{}".format(cpu_path, lib_root),
28+
"//executorch/runtime/core/portable_type:scalar_type",
2829
] + extra_deps,
2930
preprocessor_flags = get_vec_preprocessor_flags() + get_vec_cxx_preprocessor_flags(),
3031
)

‎kernels/optimized/utils/math_utils.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include <cstdint>
1212

1313
#include <executorch/kernels/optimized/utils/llvmMathExtras.h>
14+
#include <executorch/runtime/core/portable_type/bfloat16.h>
15+
#include <executorch/runtime/core/portable_type/half.h>
1416

1517
namespace executorch {
1618
namespace utils {
@@ -37,6 +39,15 @@ template <>
3739
struct ComputeDTypeTraits<int8_t> {
3840
using type = int32_t;
3941
};
42+
// For 16 bit float types, ops should perform internal math in float32.
43+
template <>
44+
struct ComputeDTypeTraits<c10::BFloat16> {
45+
using type = float;
46+
};
47+
template <>
48+
struct ComputeDTypeTraits<c10::Half> {
49+
using type = float;
50+
};
4051

4152
template <typename T>
4253
using compute_dtype = typename ComputeDTypeTraits<T>::type;

‎kernels/portable/cpu/op_native_group_norm.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ void group_norm(
7777
const CTYPE* x = input_data + i * inner_size;
7878

7979
// compute E[X] and Var[x] = E[x^2] - E[x]^2
80-
CTYPE sum = reduce_add(x, static_cast<CTYPE>(inner_size));
81-
CTYPE sq_sum = vec_powerf(x, static_cast<CTYPE>(inner_size));
80+
float sum = reduce_add(x, inner_size);
81+
float sq_sum = vec_powerf(x, inner_size);
8282
double mean_value =
8383
static_cast<double>(sum) / static_cast<double>(inner_size);
8484
double variance =

‎kernels/portable/cpu/op_native_layer_norm.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,11 @@ void layer_norm(
7373
CTYPE* y = out_data + i * normalized;
7474

7575
// compute E[X] and Var[x] = E[x^2] - E[x]^2
76-
CTYPE sum = reduce_add(x, ct_normalized);
77-
CTYPE sq_sum = vec_powerf(x, ct_normalized);
78-
CTYPE mean_value = sum / ct_normalized;
79-
CTYPE variance = sq_sum / ct_normalized - mean_value * mean_value;
80-
CTYPE std = std::sqrt(variance + eps);
76+
float sum = reduce_add(x, ct_normalized);
77+
float sq_sum = vec_powerf(x, ct_normalized);
78+
float mean_value = sum / ct_normalized;
79+
float variance = sq_sum / ct_normalized - mean_value * mean_value;
80+
float std = std::sqrt(variance + eps);
8181

8282
// Calculate the elements of output
8383
for (const auto j : c10::irange(normalized)) {

‎kernels/portable/cpu/vec_ops.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ template <typename T>
179179
inline float vec_powerf(const T* x, size_t size) {
180180
float sum = 0;
181181
for (const auto i : c10::irange(size)) {
182-
sum += x[i] * x[i];
182+
sum += static_cast<float>(x[i]) * x[i];
183183
}
184184
return sum;
185185
}

‎kernels/test/op_native_group_norm_test.cpp‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,42 @@ class OpNativeGroupNormTest : public OperatorTest {
243243
0.38038814,
244244
0.75809801}, // expected_rstd_data
245245
},
246+
{
247+
{1, 4, 3}, // sizes
248+
{0.0,
249+
1000.0,
250+
2000.0,
251+
3000.0,
252+
4000.0,
253+
5000.0,
254+
6000.0,
255+
7000.0,
256+
8000.0,
257+
9000.0,
258+
10000.0,
259+
11000.0}, // input_data
260+
{1.0, 1.0, 1.0, 1.0}, // weight_data
261+
{0.0, 0.0, 0.0, 0.0}, // bias_data
262+
1, // N
263+
4, // C
264+
3, // HxW
265+
2, // group
266+
1e-5, // eps
267+
{-1.46385,
268+
-0.87831,
269+
-0.29277,
270+
0.29277,
271+
0.87831,
272+
1.46385,
273+
-1.46385,
274+
-0.87831,
275+
-0.29277,
276+
0.29277,
277+
0.87831,
278+
1.46385}, // expected_data
279+
{2500.0, 8500.0}, // expected_mean_data
280+
{0.00058554, 0.00058554}, // expected_rstd_data
281+
},
246282
};
247283

248284
run_test_cases(test_cases);

‎kernels/test/op_native_layer_norm_test.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,12 @@ class OpNativeLayerNormTest : public OperatorTest {
101101
expected,
102102
1e-2,
103103
executorch::runtime::testing::internal::kDefaultBFloat16Atol);
104+
} else if constexpr (DTYPE == ScalarType::Half) {
105+
EXPECT_TENSOR_CLOSE_WITH_TOL(
106+
out0,
107+
expected,
108+
1e-3,
109+
executorch::runtime::testing::internal::kDefaultHalfAtol);
104110
} else {
105111
EXPECT_TENSOR_CLOSE(out0, expected);
106112
}
@@ -235,6 +241,21 @@ class OpNativeLayerNormTest : public OperatorTest {
235241
1.38873,
236242
-0.46291}, // expected_data
237243
},
244+
{
245+
std::string(__func__) + ": Large variance",
246+
{1, 2, 3}, // sizes
247+
{0.0, 1000.0, 2000.0, 3000.0, 4000.0, 5000.0}, // input_data
248+
{1, 2, 3}, // normalized shape
249+
{1.0, 1.0, 1.0, 1.0, 1.0, 1.0}, // weights
250+
{0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, // bias
251+
1.0e-5, // eps
252+
{-1.46385,
253+
-0.87831,
254+
-0.29277,
255+
0.29277,
256+
0.87831,
257+
1.46385}, // expected_data
258+
},
238259
};
239260

240261
run_test_cases(test_cases);

0 commit comments

Comments
 (0)