Skip to content

Commit dfa88b3

Browse files
borsbors
committed
Auto merge of #120500 - oli-obk:intrinsics2.0, r=WaffleLapkin
Implement intrinsics with fallback bodies fixes #93145 (though we can port many more intrinsics) cc #63585 The way this works is that the backend logic for generating custom code for intrinsics has been made fallible. The only failure path is "this intrinsic is unknown". The `Instance` (that was `InstanceDef::Intrinsic`) then gets converted to `InstanceDef::Item`, which represents the fallback body. A regular function call to that body is then codegenned. This is currently implemented for * codegen_ssa (so llvm and gcc) * codegen_cranelift other backends will need to adjust, but they can just keep doing what they were doing if they prefer (though adding new intrinsics to the compiler will then require them to implement them, instead of getting the fallback body). cc `@scottmcm` `@WaffleLapkin` ### todo * [ ] miri support * [x] default intrinsic name to name of function instead of requiring it to be specified in attribute * [x] make sure that the bodies are always available (must be collected for metadata)
2 parents 1be4688 + 164b9c3 commit dfa88b3

File tree

49 files changed

+620
-451
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+620
-451
lines changed

‎compiler/rustc_borrowck/src/type_check/mod.rs

+3-10
Original file line numberDiff line numberDiff line change
@@ -1666,16 +1666,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
16661666

16671667
let func_ty = func.ty(body, self.infcx.tcx);
16681668
if let ty::FnDef(def_id, _) = *func_ty.kind() {
1669-
if self.tcx().is_intrinsic(def_id) {
1670-
match self.tcx().item_name(def_id) {
1671-
sym::simd_shuffle => {
1672-
if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
1673-
self.tcx()
1674-
.dcx()
1675-
.emit_err(SimdShuffleLastConst { span: term.source_info.span });
1676-
}
1677-
}
1678-
_ => {}
1669+
if let Some(sym::simd_shuffle) = self.tcx().intrinsic(def_id) {
1670+
if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
1671+
self.tcx().dcx().emit_err(SimdShuffleLastConst { span: term.source_info.span });
16791672
}
16801673
}
16811674
}

‎compiler/rustc_codegen_cranelift/src/abi/mod.rs

+5-3
Original file line numberDiff line numberDiff line change
@@ -387,15 +387,17 @@ pub(crate) fn codegen_terminator_call<'tcx>(
387387

388388
match instance.def {
389389
InstanceDef::Intrinsic(_) => {
390-
crate::intrinsics::codegen_intrinsic_call(
390+
match crate::intrinsics::codegen_intrinsic_call(
391391
fx,
392392
instance,
393393
args,
394394
ret_place,
395395
target,
396396
source_info,
397-
);
398-
return;
397+
) {
398+
Ok(()) => return,
399+
Err(instance) => Some(instance),
400+
}
399401
}
400402
InstanceDef::DropGlue(_, None) => {
401403
// empty drop glue - a nop.

‎compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

+27-45
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
268268
destination: CPlace<'tcx>,
269269
target: Option<BasicBlock>,
270270
source_info: mir::SourceInfo,
271-
) {
271+
) -> Result<(), Instance<'tcx>> {
272272
let intrinsic = fx.tcx.item_name(instance.def_id());
273273
let instance_args = instance.args;
274274

@@ -295,8 +295,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
295295
destination,
296296
target,
297297
source_info,
298-
);
298+
)?;
299299
}
300+
Ok(())
300301
}
301302

302303
fn codegen_float_intrinsic_call<'tcx>(
@@ -430,25 +431,20 @@ fn codegen_regular_intrinsic_call<'tcx>(
430431
ret: CPlace<'tcx>,
431432
destination: Option<BasicBlock>,
432433
source_info: mir::SourceInfo,
433-
) {
434+
) -> Result<(), Instance<'tcx>> {
435+
assert_eq!(generic_args, instance.args);
434436
let usize_layout = fx.layout_of(fx.tcx.types.usize);
435437

436438
match intrinsic {
437439
sym::abort => {
438440
fx.bcx.ins().trap(TrapCode::User(0));
439-
return;
441+
return Ok(());
440442
}
441443
sym::likely | sym::unlikely => {
442444
intrinsic_args!(fx, args => (a); intrinsic);
443445

444446
ret.write_cvalue(fx, a);
445447
}
446-
sym::is_val_statically_known => {
447-
intrinsic_args!(fx, args => (_a); intrinsic);
448-
449-
let res = fx.bcx.ins().iconst(types::I8, 0);
450-
ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
451-
}
452448
sym::breakpoint => {
453449
intrinsic_args!(fx, args => (); intrinsic);
454450

@@ -697,7 +693,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
697693
})
698694
});
699695
crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span));
700-
return;
696+
return Ok(());
701697
}
702698
}
703699
}
@@ -792,7 +788,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
792788
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
793789
// special case for compiler-builtins to avoid having to patch it
794790
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
795-
return;
791+
return Ok(());
796792
} else {
797793
fx.tcx
798794
.dcx()
@@ -802,7 +798,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
802798
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
803799
_ => {
804800
report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
805-
return;
801+
return Ok(());
806802
}
807803
}
808804
let clif_ty = fx.clif_type(ty).unwrap();
@@ -823,7 +819,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
823819
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
824820
// special case for compiler-builtins to avoid having to patch it
825821
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
826-
return;
822+
return Ok(());
827823
} else {
828824
fx.tcx
829825
.dcx()
@@ -833,7 +829,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
833829
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
834830
_ => {
835831
report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
836-
return;
832+
return Ok(());
837833
}
838834
}
839835

@@ -850,7 +846,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
850846
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
851847
_ => {
852848
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
853-
return;
849+
return Ok(());
854850
}
855851
}
856852
let ty = fx.clif_type(layout.ty).unwrap();
@@ -872,7 +868,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
872868
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
873869
_ => {
874870
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
875-
return;
871+
return Ok(());
876872
}
877873
}
878874

@@ -895,7 +891,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
895891
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
896892
_ => {
897893
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
898-
return;
894+
return Ok(());
899895
}
900896
}
901897
let ty = fx.clif_type(layout.ty).unwrap();
@@ -917,7 +913,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
917913
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
918914
_ => {
919915
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
920-
return;
916+
return Ok(());
921917
}
922918
}
923919
let ty = fx.clif_type(layout.ty).unwrap();
@@ -939,7 +935,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
939935
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
940936
_ => {
941937
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
942-
return;
938+
return Ok(());
943939
}
944940
}
945941
let ty = fx.clif_type(layout.ty).unwrap();
@@ -960,7 +956,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
960956
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
961957
_ => {
962958
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
963-
return;
959+
return Ok(());
964960
}
965961
}
966962
let ty = fx.clif_type(layout.ty).unwrap();
@@ -981,7 +977,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
981977
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
982978
_ => {
983979
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
984-
return;
980+
return Ok(());
985981
}
986982
}
987983
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1002,7 +998,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
1002998
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1003999
_ => {
10041000
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1005-
return;
1001+
return Ok(());
10061002
}
10071003
}
10081004
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1023,7 +1019,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
10231019
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
10241020
_ => {
10251021
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1026-
return;
1022+
return Ok(());
10271023
}
10281024
}
10291025
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1044,7 +1040,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
10441040
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
10451041
_ => {
10461042
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1047-
return;
1043+
return Ok(());
10481044
}
10491045
}
10501046
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1065,7 +1061,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
10651061
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
10661062
_ => {
10671063
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1068-
return;
1064+
return Ok(());
10691065
}
10701066
}
10711067
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1086,7 +1082,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
10861082
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
10871083
_ => {
10881084
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1089-
return;
1085+
return Ok(());
10901086
}
10911087
}
10921088
let ty = fx.clif_type(layout.ty).unwrap();
@@ -1233,19 +1229,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
12331229
ret.write_cvalue(fx, CValue::by_val(cmp, ret.layout()));
12341230
}
12351231

1236-
sym::const_allocate => {
1237-
intrinsic_args!(fx, args => (_size, _align); intrinsic);
1238-
1239-
// returns a null pointer at runtime.
1240-
let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
1241-
ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
1242-
}
1243-
1244-
sym::const_deallocate => {
1245-
intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
1246-
// nop at runtime.
1247-
}
1248-
12491232
sym::black_box => {
12501233
intrinsic_args!(fx, args => (a); intrinsic);
12511234

@@ -1261,13 +1244,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
12611244
);
12621245
}
12631246

1264-
_ => {
1265-
fx.tcx
1266-
.dcx()
1267-
.span_fatal(source_info.span, format!("unsupported intrinsic {}", intrinsic));
1268-
}
1247+
// Unimplemented intrinsics must have a fallback body. The fallback body is obtained
1248+
// by converting the `InstanceDef::Intrinsic` to an `InstanceDef::Item`.
1249+
_ => return Err(Instance::new(instance.def_id(), instance.args)),
12691250
}
12701251

12711252
let ret_block = fx.get_block(destination.unwrap());
12721253
fx.bcx.ins().jump(ret_block, &[]);
1254+
Ok(())
12731255
}

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

+10-8
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ fn get_simple_intrinsic<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, name: Symbol) ->
9090
}
9191

9292
impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
93-
fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) {
93+
fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) -> Result<(), Instance<'tcx>> {
9494
let tcx = self.tcx;
9595
let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
9696

@@ -137,7 +137,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
137137
args[2].immediate(),
138138
llresult,
139139
);
140-
return;
140+
return Ok(());
141141
}
142142
sym::breakpoint => {
143143
unimplemented!();
@@ -166,12 +166,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
166166
sym::volatile_store => {
167167
let dst = args[0].deref(self.cx());
168168
args[1].val.volatile_store(self, dst);
169-
return;
169+
return Ok(());
170170
}
171171
sym::unaligned_volatile_store => {
172172
let dst = args[0].deref(self.cx());
173173
args[1].val.unaligned_volatile_store(self, dst);
174-
return;
174+
return Ok(());
175175
}
176176
sym::prefetch_read_data
177177
| sym::prefetch_write_data
@@ -269,7 +269,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
269269
},
270270
None => {
271271
tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
272-
return;
272+
return Ok(());
273273
}
274274
}
275275
}
@@ -339,7 +339,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
339339
extended_asm.set_volatile_flag(true);
340340

341341
// We have copied the value to `result` already.
342-
return;
342+
return Ok(());
343343
}
344344

345345
sym::ptr_mask => {
@@ -357,11 +357,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
357357
_ if name_str.starts_with("simd_") => {
358358
match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
359359
Ok(llval) => llval,
360-
Err(()) => return,
360+
Err(()) => return Ok(()),
361361
}
362362
}
363363

364-
_ => bug!("unknown intrinsic '{}'", name),
364+
// Fall back to default body
365+
_ => return Err(Instance::new(instance.def_id(), instance.args)),
365366
};
366367

367368
if !fn_abi.ret.is_ignore() {
@@ -376,6 +377,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
376377
.store(self, result);
377378
}
378379
}
380+
Ok(())
379381
}
380382

381383
fn abort(&mut self) {

0 commit comments

Comments
 (0)