Skip to content

Commit 3120a57

Browse files
Backport PR #71572 on branch 3.1.x (BLD: don't include moments_simd.hpp in moments_scalar.cpp) (#71863)
Co-authored-by: Álvaro Kothe <[email protected]>
1 parent 378620e commit 3120a57

6 files changed

Lines changed: 199 additions & 143 deletions

File tree

‎doc/source/whatsnew/v3.1.1.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ Bug fixes
2424

2525
- Fixed ``AttributeError`` when using dateutil timezones such as ``"dateutil/US/Pacific"`` with python-dateutil 3.0 (:issue:`71459`)
2626

27+
.. ---------------------------------------------------------------------------
28+
.. _whatsnew_311.packaging.build:
29+
30+
Build
31+
^^^^^
32+
- Fixed build error on unsupported ``xsimd`` architectures (:issue:`71548`)
33+
34+
2735
.. ---------------------------------------------------------------------------
2836
.. _whatsnew_311.contributors:
2937

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
Copyright (c) 2026, PyData Development Team
3+
All rights reserved.
4+
5+
Distributed under the terms of the BSD Simplified License.
6+
7+
The full license is in the LICENSE file, distributed with this software.
8+
*/
9+
10+
#pragma once
11+
12+
#include "pandas/moments.h"
13+
#include <algorithm>
14+
#include <cassert>
15+
#include <cmath>
16+
#include <optional>
17+
#include <span>
18+
19+
/// Common functions used for simd and scalar versions.
20+
21+
namespace pandas::moments::detail {
22+
23+
template <typename T, typename U> struct MeanAcc {
24+
T sum{};
25+
U count{};
26+
};
27+
28+
template <typename T> struct CentralDiffs {
29+
T m1{}, m2{}, m3{}, m4{};
30+
};
31+
32+
/// Compute Moments struct using two-pass
33+
/// with correction presented in Pébay (2016).
34+
///
35+
/// References:
36+
/// Pébay, P., Terriberry, T. B., Kolla, H., & Bennett, J. (2016).
37+
/// Numerically stable, scalable formulas for parallel
38+
/// and online computation of higher-order multivariate central moments with
39+
/// arbitrary weights. Comput. Stat., 31(4), 1305–1325.
40+
/// https://doi.org/10.1007/s00180-015-0637-z
41+
static inline Moments
42+
compute_moments_with_correction(MeanAcc<double, std::size_t> total_acc,
43+
CentralDiffs<double> central_diffs,
44+
int max_moment) {
45+
const auto count_double = static_cast<double>(total_acc.count);
46+
const double trial_mean = total_acc.sum / count_double;
47+
const double correction_term = central_diffs.m1 / count_double;
48+
const double correction_term2 = correction_term * correction_term;
49+
const double term1 = central_diffs.m1 * correction_term;
50+
51+
Moments result{};
52+
53+
result.n = total_acc.count;
54+
result.mean = trial_mean + correction_term;
55+
result.m2 = central_diffs.m2 - term1;
56+
result.m2 = std::max(result.m2, 0.0);
57+
58+
if (max_moment >= 3) {
59+
result.m3 = central_diffs.m3 - (3.0 * central_diffs.m2 * correction_term) +
60+
(2.0 * correction_term * term1);
61+
}
62+
if (max_moment >= 4) {
63+
result.m4 = central_diffs.m4 - (4.0 * central_diffs.m3 * correction_term) +
64+
(6.0 * central_diffs.m2 * correction_term2) -
65+
(3.0 * term1 * correction_term2);
66+
result.m4 = std::max(result.m4, 0.0);
67+
}
68+
69+
return result;
70+
}
71+
72+
static inline std::optional<MeanAcc<double, std::size_t>>
73+
accumulate_mean_scalar_direct(
74+
const std::span<const double> values,
75+
const std::optional<std::span<const uint8_t>> mask, const bool skipna) {
76+
double sum = 0.0;
77+
std::size_t count = 0;
78+
for (std::size_t i = 0; i < values.size(); ++i) {
79+
const double val = values[i];
80+
const bool isna = mask.has_value() ? (*mask)[i] != 0 : std::isnan(val);
81+
if (isna) {
82+
if (!skipna) {
83+
return std::nullopt;
84+
}
85+
continue;
86+
}
87+
sum += val;
88+
count++;
89+
}
90+
// Explicit template arguments: CTAD for aggregates (P1816) is unavailable
91+
// in Apple clang < 17 (Xcode < 16).
92+
return MeanAcc<double, std::size_t>{.sum = sum, .count = count};
93+
}
94+
95+
static inline CentralDiffs<double> accumulate_central_diffs_scalar_direct(
96+
std::span<const double> values,
97+
std::optional<std::span<const uint8_t>> mask, double mean, int max_moment) {
98+
assert(!mask.has_value() || mask->size() == values.size());
99+
100+
CentralDiffs<double> acc{};
101+
for (std::size_t i = 0; i < values.size(); i++) {
102+
const auto isna = mask ? ((*mask)[i] != 0) : std::isnan(values[i]);
103+
104+
if (isna) {
105+
continue;
106+
}
107+
108+
const auto diff = values[i] - mean;
109+
const auto diff2 = diff * diff;
110+
111+
acc.m1 += diff;
112+
acc.m2 += diff2;
113+
if (max_moment >= 3) {
114+
acc.m3 += diff2 * diff;
115+
}
116+
if (max_moment >= 4) {
117+
acc.m4 += diff2 * diff2;
118+
}
119+
}
120+
121+
return acc;
122+
}
123+
124+
/// accumulate_{mean,central_diffs} are forward declared for
125+
/// accumulate_moments_impl. It has a simd implementation in moments_simd.hpp
126+
/// and a scalar version in moments_scalar.cpp.
127+
128+
template <class Arch>
129+
std::optional<MeanAcc<double, size_t>>
130+
accumulate_mean(std::span<const double> values,
131+
std::optional<std::span<const uint8_t>> mask, bool skipna);
132+
133+
template <class Arch>
134+
CentralDiffs<double>
135+
accumulate_central_diffs(std::span<const double> values,
136+
std::optional<std::span<const uint8_t>> mask,
137+
double mean, int max_moment);
138+
139+
template <class Arch>
140+
Moments accumulate_moments_impl(std::span<const double> values,
141+
std::optional<std::span<const uint8_t>> mask,
142+
bool skipna, int max_moment) {
143+
const auto total_acc = accumulate_mean<Arch>(values, mask, skipna);
144+
145+
if (!total_acc.has_value()) {
146+
return {.mean = NAN, .m2 = NAN, .m3 = NAN, .m4 = NAN, .n = values.size()};
147+
}
148+
149+
if (total_acc->count == 0) {
150+
return {};
151+
}
152+
153+
const auto count_double = static_cast<double>(total_acc->count);
154+
double trial_mean = total_acc->sum / count_double;
155+
156+
const auto central_diffs =
157+
accumulate_central_diffs<Arch>(values, mask, trial_mean, max_moment);
158+
159+
return compute_moments_with_correction(*total_acc, central_diffs, max_moment);
160+
}
161+
162+
} // namespace pandas::moments::detail

‎pandas/_libs/include/pandas/simd/moments_simd.hpp‎

Lines changed: 3 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -10,90 +10,17 @@ The full license is in the LICENSE file, distributed with this software.
1010
#pragma once
1111

1212
#include "pandas/moments.h"
13+
#include "pandas/simd/moments_common.hpp"
1314
#include "xsimd/xsimd.hpp" // IWYU pragma: keep
14-
#include <algorithm>
1515
#include <array>
1616
#include <cassert>
17-
#include <cmath>
1817
#include <optional>
1918
#include <span>
2019

2120
namespace pandas::moments {
2221

2322
namespace detail {
2423

25-
template <typename T, typename U> struct MeanAcc {
26-
T sum{};
27-
U count{};
28-
};
29-
30-
template <typename T> struct CentralDiffs {
31-
T m1{}, m2{}, m3{}, m4{};
32-
};
33-
34-
/// Compute Moments struct using two-pass
35-
/// with correction presented in Pébay (2016).
36-
///
37-
/// References:
38-
/// Pébay, P., Terriberry, T. B., Kolla, H., & Bennett, J. (2016).
39-
/// Numerically stable, scalable formulas for parallel
40-
/// and online computation of higher-order multivariate central moments with
41-
/// arbitrary weights. Comput. Stat., 31(4), 1305–1325.
42-
/// https://doi.org/10.1007/s00180-015-0637-z
43-
static inline Moments
44-
compute_moments_with_correction(MeanAcc<double, std::size_t> total_acc,
45-
CentralDiffs<double> central_diffs,
46-
int max_moment) {
47-
const auto count_double = static_cast<double>(total_acc.count);
48-
const double trial_mean = total_acc.sum / count_double;
49-
const double correction_term = central_diffs.m1 / count_double;
50-
const double correction_term2 = correction_term * correction_term;
51-
const double term1 = central_diffs.m1 * correction_term;
52-
53-
Moments result{};
54-
55-
result.n = total_acc.count;
56-
result.mean = trial_mean + correction_term;
57-
result.m2 = central_diffs.m2 - term1;
58-
result.m2 = std::max(result.m2, 0.0);
59-
60-
if (max_moment >= 3) {
61-
result.m3 = central_diffs.m3 - (3.0 * central_diffs.m2 * correction_term) +
62-
(2.0 * correction_term * term1);
63-
}
64-
if (max_moment >= 4) {
65-
result.m4 = central_diffs.m4 - (4.0 * central_diffs.m3 * correction_term) +
66-
(6.0 * central_diffs.m2 * correction_term2) -
67-
(3.0 * term1 * correction_term2);
68-
result.m4 = std::max(result.m4, 0.0);
69-
}
70-
71-
return result;
72-
}
73-
74-
static inline std::optional<detail::MeanAcc<double, std::size_t>>
75-
accumulate_mean_scalar_direct(
76-
const std::span<const double> values,
77-
const std::optional<std::span<const uint8_t>> mask, const bool skipna) {
78-
double sum = 0.0;
79-
std::size_t count = 0;
80-
for (std::size_t i = 0; i < values.size(); ++i) {
81-
const double val = values[i];
82-
const bool isna = mask.has_value() ? (*mask)[i] != 0 : std::isnan(val);
83-
if (isna) {
84-
if (!skipna) {
85-
return std::nullopt;
86-
}
87-
continue;
88-
}
89-
sum += val;
90-
count++;
91-
}
92-
// Explicit template arguments: CTAD for aggregates (P1816) is unavailable
93-
// in Apple clang < 17 (Xcode < 16).
94-
return MeanAcc<double, std::size_t>{.sum = sum, .count = count};
95-
}
96-
9724
template <class Arch>
9825
std::optional<
9926
MeanAcc<xsimd::batch<double, Arch>, xsimd::batch<std::uint64_t, Arch>>>
@@ -185,35 +112,6 @@ accumulate_mean_direct(std::span<const double> values,
185112
.sum = sum[0], .count = count};
186113
}
187114

188-
static inline CentralDiffs<double> accumulate_central_diffs_scalar_direct(
189-
std::span<const double> values,
190-
std::optional<std::span<const uint8_t>> mask, double mean, int max_moment) {
191-
assert(!mask.has_value() || mask->size() == values.size());
192-
193-
CentralDiffs<double> acc{};
194-
for (std::size_t i = 0; i < values.size(); i++) {
195-
const auto isna = mask ? ((*mask)[i] != 0) : std::isnan(values[i]);
196-
197-
if (isna) {
198-
continue;
199-
}
200-
201-
const auto diff = values[i] - mean;
202-
const auto diff2 = diff * diff;
203-
204-
acc.m1 += diff;
205-
acc.m2 += diff2;
206-
if (max_moment >= 3) {
207-
acc.m3 += diff2 * diff;
208-
}
209-
if (max_moment >= 4) {
210-
acc.m4 += diff2 * diff2;
211-
}
212-
}
213-
214-
return acc;
215-
}
216-
217115
template <class Arch>
218116
CentralDiffs<xsimd::batch<double, Arch>>
219117
accumulate_central_diffs_direct(std::span<const double> values,
@@ -446,30 +344,6 @@ accumulate_central_diffs(std::span<const double> values,
446344
};
447345
}
448346

449-
template <class Arch>
450-
Moments
451-
accumulate_moments_simd_impl(std::span<const double> values,
452-
std::optional<std::span<const uint8_t>> mask,
453-
bool skipna, int max_moment) {
454-
const auto total_acc = accumulate_mean<Arch>(values, mask, skipna);
455-
456-
if (!total_acc.has_value()) {
457-
return {.mean = NAN, .m2 = NAN, .m3 = NAN, .m4 = NAN, .n = values.size()};
458-
}
459-
460-
if (total_acc->count == 0) {
461-
return {};
462-
}
463-
464-
const auto count_double = static_cast<double>(total_acc->count);
465-
double trial_mean = total_acc->sum / count_double;
466-
467-
const auto central_diffs =
468-
accumulate_central_diffs<Arch>(values, mask, trial_mean, max_moment);
469-
470-
return compute_moments_with_correction(*total_acc, central_diffs, max_moment);
471-
}
472-
473347
} // namespace detail
474348

475349
struct accumulate_moments_simd {
@@ -483,8 +357,8 @@ template <class Arch>
483357
Moments accumulate_moments_simd::operator()(
484358
Arch /*arch*/, std::span<const double> values, bool skipna,
485359
std::optional<std::span<const uint8_t>> mask, int max_moment) noexcept {
486-
return detail::accumulate_moments_simd_impl<Arch>(values, mask, skipna,
487-
max_moment);
360+
return detail::accumulate_moments_impl<Arch>(values, mask, skipna,
361+
max_moment);
488362
}
489363

490364
} // namespace pandas::moments

‎pandas/_libs/meson.build‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ endif
6767
m_dep = cc.find_library('m', required: false)
6868
fast_float = subproject('fast_float')
6969
fast_float_dep = fast_float.get_variable('fast_float_dep')
70-
xsimd_dep = dependency('xsimd', version: '>=14.2')
7170

7271
subdir('simd')
7372

‎pandas/_libs/simd/meson.build‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ else
1212
simd_config.set('PANDAS_HAVE_SCALAR', 1)
1313
endif
1414

15+
if simd_target != 'scalar'
16+
xsimd_dep = dependency('xsimd', version: '>=14.2')
17+
moments_deps = [xsimd_dep]
18+
else
19+
moments_deps = []
20+
endif
21+
1522
configure_file(
1623
output: 'pandas_simd_config.h',
1724
configuration: simd_config,
@@ -23,7 +30,7 @@ moments_lib = static_library(
2330
'moments_@0@'.format(simd_target),
2431
'moments_@[email protected]'.format(simd_target),
2532
include_directories: [inc_pd],
26-
dependencies: [xsimd_dep],
33+
dependencies: moments_deps,
2734
cpp_args: strict_c_args,
2835
)
2936

0 commit comments

Comments
 (0)