Skip to content

Commit f70538c

Browse files
RalfJungRalfJung
committed
make simd_reduce_{mul,add}_unordered use only the 'reassoc' flag, not all fast-math flags
1 parent 361fd7e commit f70538c

File tree

7 files changed

+25
-23
lines changed

7 files changed

+25
-23
lines changed

‎compiler/rustc_codegen_gcc/src/builder.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1727,7 +1727,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
17271727
self.vector_reduce(src, |a, b, context| context.new_binary_op(None, op, a.get_type(), a, b))
17281728
}
17291729

1730-
pub fn vector_reduce_fadd_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1730+
pub fn vector_reduce_fadd_reassoc(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
17311731
unimplemented!();
17321732
}
17331733

@@ -1747,7 +1747,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
17471747
unimplemented!();
17481748
}
17491749

1750-
pub fn vector_reduce_fmul_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1750+
pub fn vector_reduce_fmul_reassoc(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
17511751
unimplemented!();
17521752
}
17531753

‎compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -989,14 +989,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
989989

990990
arith_red!(
991991
simd_reduce_add_unordered: BinaryOp::Plus,
992-
vector_reduce_fadd_fast,
992+
vector_reduce_fadd_reassoc,
993993
false,
994994
add,
995995
0.0 // TODO: Use this argument.
996996
);
997997
arith_red!(
998998
simd_reduce_mul_unordered: BinaryOp::Mult,
999-
vector_reduce_fmul_fast,
999+
vector_reduce_fmul_reassoc,
10001000
false,
10011001
mul,
10021002
1.0

‎compiler/rustc_codegen_llvm/src/builder.rs

+4-4
Original file line numberDiff line numberDiff line change
@@ -1327,17 +1327,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
13271327
pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13281328
unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
13291329
}
1330-
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1330+
pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13311331
unsafe {
13321332
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1333-
llvm::LLVMRustSetFastMath(instr);
1333+
llvm::LLVMRustSetAllowReassoc(instr);
13341334
instr
13351335
}
13361336
}
1337-
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1337+
pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13381338
unsafe {
13391339
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1340-
llvm::LLVMRustSetFastMath(instr);
1340+
llvm::LLVMRustSetAllowReassoc(instr);
13411341
instr
13421342
}
13431343
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1880,14 +1880,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18801880
arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
18811881
arith_red!(
18821882
simd_reduce_add_unordered: vector_reduce_add,
1883-
vector_reduce_fadd_fast,
1883+
vector_reduce_fadd_reassoc,
18841884
false,
18851885
add,
18861886
0.0
18871887
);
18881888
arith_red!(
18891889
simd_reduce_mul_unordered: vector_reduce_mul,
1890-
vector_reduce_fmul_fast,
1890+
vector_reduce_fmul_reassoc,
18911891
false,
18921892
mul,
18931893
1.0

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1618,6 +1618,7 @@ extern "C" {
16181618
) -> &'a Value;
16191619

16201620
pub fn LLVMRustSetFastMath(Instr: &Value);
1621+
pub fn LLVMRustSetAllowReassoc(Instr: &Value);
16211622

16221623
// Miscellaneous instructions
16231624
pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

+10-1
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
418418
}
419419
}
420420

421-
// Enable a fast-math flag
421+
// Enable all fast-math flags
422422
//
423423
// https://llvm.org/docs/LangRef.html#fast-math-flags
424424
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
@@ -427,6 +427,15 @@ extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
427427
}
428428
}
429429

430+
// Enable the reassoc fast-math flag
431+
//
432+
// https://llvm.org/docs/LangRef.html#fast-math-flags
433+
extern "C" void LLVMRustSetAllowReassoc(LLVMValueRef V) {
434+
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
435+
I->setHasAllowReassoc(true);
436+
}
437+
}
438+
430439
extern "C" LLVMValueRef
431440
LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source,
432441
const char *Name, LLVMAtomicOrdering Order) {

‎library/core/src/intrinsics/simd.rs

+4-12
Original file line numberDiff line numberDiff line change
@@ -333,16 +333,12 @@ extern "platform-intrinsic" {
333333
/// Starting with the value `y`, add the elements of `x` and accumulate.
334334
pub fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
335335

336-
/// Add elements within a vector in arbitrary order, and without regard
337-
/// for signed zeros.
336+
/// Add elements within a vector in arbitrary order. May also be re-associated with
337+
/// unordered additions on the inputs/outputs.
338338
///
339339
/// `T` must be a vector of integer or floating-point primitive types.
340340
///
341341
/// `U` must be the element type of `T`.
342-
///
343-
/// # Safety
344-
///
345-
/// All input elements must be finite (i.e., not NAN and not +/- INF).
346342
pub fn simd_reduce_add_unordered<T, U>(x: T) -> U;
347343

348344
/// Multiply elements within a vector from left to right.
@@ -354,16 +350,12 @@ extern "platform-intrinsic" {
354350
/// Starting with the value `y`, multiply the elements of `x` and accumulate.
355351
pub fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
356352

357-
/// Multiply elements within a vector in arbitrary order, and without regard
358-
/// for signed zeros.
353+
/// Add elements within a vector in arbitrary order. May also be re-associated with
354+
/// unordered additions on the inputs/outputs.
359355
///
360356
/// `T` must be a vector of integer or floating-point primitive types.
361357
///
362358
/// `U` must be the element type of `T`.
363-
///
364-
/// # Safety
365-
///
366-
/// All input elements must be finite (i.e., not NAN and not +/- INF).
367359
pub fn simd_reduce_mul_unordered<T, U>(x: T) -> U;
368360

369361
/// Check if all mask values are true.

0 commit comments

Comments
 (0)