Skip to content

Commit 9272ac8

Browse files
rust-timerrust-timer
authored
Unrolled build for rust-lang#123840
Rollup merge of rust-lang#123840 - scottmcm:aggregate-kind-rawptr, r=cjgillot Add an intrinsic for `ptr::from_raw_parts(_mut)` Fixes rust-lang#123174 cc `@CAD97` `@saethlin` r? `@cjgillot` As suggested in rust-lang#123190 (comment), this adds a new `AggregateKind::RawPtr` for creating a pointer from its data pointer and its metadata. That means that `slice::from_raw_parts` and friends no longer need to hard-code pointer layout into `libcore`, and because it no longer does union hacks the MIR is shorter and more amenable to optimizations.
2 parents 1b3fba0 + 5e785b1 commit 9272ac8

38 files changed

+949
-16
lines changed

‎compiler/rustc_borrowck/src/lib.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1315,7 +1315,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
13151315
}
13161316
AggregateKind::Adt(..)
13171317
| AggregateKind::Array(..)
1318-
| AggregateKind::Tuple { .. } => (),
1318+
| AggregateKind::Tuple { .. }
1319+
| AggregateKind::RawPtr(..) => (),
13191320
}
13201321

13211322
for operand in operands {

‎compiler/rustc_borrowck/src/type_check/mod.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -1921,7 +1921,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
19211921
}
19221922
}
19231923
AggregateKind::Array(ty) => Ok(ty),
1924-
AggregateKind::Tuple => {
1924+
AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
19251925
unreachable!("This should have been covered in check_rvalues");
19261926
}
19271927
}
@@ -2518,6 +2518,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
25182518
AggregateKind::Closure(_, _) => None,
25192519
AggregateKind::Coroutine(_, _) => None,
25202520
AggregateKind::CoroutineClosure(_, _) => None,
2521+
AggregateKind::RawPtr(_, _) => None,
25212522
},
25222523
}
25232524
}
@@ -2539,6 +2540,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
25392540
return;
25402541
}
25412542

2543+
if let AggregateKind::RawPtr(..) = aggregate_kind {
2544+
bug!("RawPtr should only be in runtime MIR");
2545+
}
2546+
25422547
for (i, operand) in operands.iter_enumerated() {
25432548
let field_ty = match self.aggregate_field_ty(aggregate_kind, i, location) {
25442549
Ok(field_ty) => field_ty,
@@ -2757,7 +2762,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
27572762
),
27582763
),
27592764

2760-
AggregateKind::Array(_) | AggregateKind::Tuple => {
2765+
AggregateKind::Array(_) | AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
27612766
(CRATE_DEF_ID.to_def_id(), ty::InstantiatedPredicates::empty())
27622767
}
27632768
};

‎compiler/rustc_codegen_cranelift/src/base.rs

+13
Original file line numberDiff line numberDiff line change
@@ -813,6 +813,19 @@ fn codegen_stmt<'tcx>(
813813
);
814814
lval.write_cvalue(fx, val);
815815
}
816+
Rvalue::Aggregate(ref kind, ref operands)
817+
if matches!(**kind, AggregateKind::RawPtr(..)) =>
818+
{
819+
let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx);
820+
let layout = fx.layout_of(fx.monomorphize(ty));
821+
let [data, meta] = &*operands.raw else {
822+
bug!("RawPtr fields: {operands:?}");
823+
};
824+
let data = codegen_operand(fx, data);
825+
let meta = codegen_operand(fx, meta);
826+
let ptr_val = CValue::pointer_from_data_and_meta(data, meta, layout);
827+
lval.write_cvalue(fx, ptr_val);
828+
}
816829
Rvalue::Aggregate(ref kind, ref operands) => {
817830
let (variant_index, variant_dest, active_field_index) = match **kind {
818831
mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {

‎compiler/rustc_codegen_cranelift/src/value_and_place.rs

+17
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,23 @@ impl<'tcx> CValue<'tcx> {
9494
CValue(CValueInner::ByValPair(value, extra), layout)
9595
}
9696

97+
/// For `AggregateKind::RawPtr`, create a pointer from its parts.
98+
///
99+
/// Panics if the `layout` is not a raw pointer.
100+
pub(crate) fn pointer_from_data_and_meta(
101+
data: CValue<'tcx>,
102+
meta: CValue<'tcx>,
103+
layout: TyAndLayout<'tcx>,
104+
) -> CValue<'tcx> {
105+
assert!(layout.ty.is_unsafe_ptr());
106+
let inner = match (data.0, meta.0) {
107+
(CValueInner::ByVal(p), CValueInner::ByVal(m)) => CValueInner::ByValPair(p, m),
108+
(p @ CValueInner::ByVal(_), CValueInner::ByRef(..)) if meta.1.is_zst() => p,
109+
_ => bug!("RawPtr operands {data:?} {meta:?}"),
110+
};
111+
CValue(inner, layout)
112+
}
113+
97114
pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
98115
self.1
99116
}

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+21-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::MemFlags;
99

1010
use rustc_hir as hir;
1111
use rustc_middle::mir;
12-
use rustc_middle::mir::Operand;
12+
use rustc_middle::mir::{AggregateKind, Operand};
1313
use rustc_middle::ty::cast::{CastTy, IntTy};
1414
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
1515
use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt};
@@ -720,6 +720,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
720720
OperandRef { val: OperandValue::Immediate(static_), layout }
721721
}
722722
mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand),
723+
mir::Rvalue::Aggregate(box mir::AggregateKind::RawPtr(..), ref fields) => {
724+
let ty = rvalue.ty(self.mir, self.cx.tcx());
725+
let layout = self.cx.layout_of(self.monomorphize(ty));
726+
let [data, meta] = &*fields.raw else {
727+
bug!("RawPtr fields: {fields:?}");
728+
};
729+
let data = self.codegen_operand(bx, data);
730+
let meta = self.codegen_operand(bx, meta);
731+
match (data.val, meta.val) {
732+
(p @ OperandValue::Immediate(_), OperandValue::ZeroSized) => {
733+
OperandRef { val: p, layout }
734+
}
735+
(OperandValue::Immediate(p), OperandValue::Immediate(m)) => {
736+
OperandRef { val: OperandValue::Pair(p, m), layout }
737+
}
738+
_ => bug!("RawPtr operands {data:?} {meta:?}"),
739+
}
740+
}
723741
mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
724742
// According to `rvalue_creates_operand`, only ZST
725743
// aggregate rvalues are allowed to be operands.
@@ -1032,6 +1050,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
10321050
mir::Rvalue::ThreadLocalRef(_) |
10331051
mir::Rvalue::Use(..) => // (*)
10341052
true,
1053+
// This always produces a `ty::RawPtr`, so will be Immediate or Pair
1054+
mir::Rvalue::Aggregate(box AggregateKind::RawPtr(..), ..) => true,
10351055
mir::Rvalue::Repeat(..) |
10361056
mir::Rvalue::Aggregate(..) => {
10371057
let ty = rvalue.ty(self.mir, self.cx.tcx());

‎compiler/rustc_const_eval/src/interpret/step.rs

+24-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ use rustc_middle::mir;
99
use rustc_middle::ty::layout::LayoutOf;
1010
use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
1111

12-
use super::{ImmTy, InterpCx, InterpResult, Machine, PlaceTy, Projectable, Scalar};
12+
use super::{
13+
ImmTy, Immediate, InterpCx, InterpResult, Machine, MemPlaceMeta, PlaceTy, Projectable, Scalar,
14+
};
1315
use crate::util;
1416

1517
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@@ -303,6 +305,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
303305
let variant_dest = self.project_downcast(dest, variant_index)?;
304306
(variant_index, variant_dest, active_field_index)
305307
}
308+
mir::AggregateKind::RawPtr(..) => {
309+
// Pointers don't have "fields" in the normal sense, so the
310+
// projection-based code below would either fail in projection
311+
// or in type mismatches. Instead, build an `Immediate` from
312+
// the parts and write that to the destination.
313+
let [data, meta] = &operands.raw else {
314+
bug!("{kind:?} should have 2 operands, had {operands:?}");
315+
};
316+
let data = self.eval_operand(data, None)?;
317+
let data = self.read_pointer(&data)?;
318+
let meta = self.eval_operand(meta, None)?;
319+
let meta = if meta.layout.is_zst() {
320+
MemPlaceMeta::None
321+
} else {
322+
MemPlaceMeta::Meta(self.read_scalar(&meta)?)
323+
};
324+
let ptr_imm = Immediate::new_pointer_with_meta(data, meta, self);
325+
let ptr = ImmTy::from_immediate(ptr_imm, dest.layout);
326+
self.copy_op(&ptr, dest)?;
327+
return Ok(());
328+
}
306329
_ => (FIRST_VARIANT, dest.clone(), None),
307330
};
308331
if active_field_index.is_some() {

‎compiler/rustc_const_eval/src/transform/validate.rs

+41
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,47 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
923923
}
924924
}
925925
}
926+
AggregateKind::RawPtr(pointee_ty, mutability) => {
927+
if !matches!(self.mir_phase, MirPhase::Runtime(_)) {
928+
// It would probably be fine to support this in earlier phases,
929+
// but at the time of writing it's only ever introduced from intrinsic lowering,
930+
// so earlier things just `bug!` on it.
931+
self.fail(location, "RawPtr should be in runtime MIR only");
932+
}
933+
934+
if fields.len() != 2 {
935+
self.fail(location, "raw pointer aggregate must have 2 fields");
936+
} else {
937+
let data_ptr_ty = fields.raw[0].ty(self.body, self.tcx);
938+
let metadata_ty = fields.raw[1].ty(self.body, self.tcx);
939+
if let ty::RawPtr(in_pointee, in_mut) = data_ptr_ty.kind() {
940+
if *in_mut != mutability {
941+
self.fail(location, "input and output mutability must match");
942+
}
943+
944+
// FIXME: check `Thin` instead of `Sized`
945+
if !in_pointee.is_sized(self.tcx, self.param_env) {
946+
self.fail(location, "input pointer must be thin");
947+
}
948+
} else {
949+
self.fail(
950+
location,
951+
"first operand to raw pointer aggregate must be a raw pointer",
952+
);
953+
}
954+
955+
// FIXME: Check metadata more generally
956+
if pointee_ty.is_slice() {
957+
if !self.mir_assign_valid_types(metadata_ty, self.tcx.types.usize) {
958+
self.fail(location, "slice metadata must be usize");
959+
}
960+
} else if pointee_ty.is_sized(self.tcx, self.param_env) {
961+
if metadata_ty != self.tcx.types.unit {
962+
self.fail(location, "metadata for pointer-to-thin must be unit");
963+
}
964+
}
965+
}
966+
}
926967
},
927968
Rvalue::Ref(_, BorrowKind::Fake, _) => {
928969
if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

+5
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
128128
| sym::variant_count
129129
| sym::is_val_statically_known
130130
| sym::ptr_mask
131+
| sym::aggregate_raw_ptr
131132
| sym::ub_checks
132133
| sym::fadd_algebraic
133134
| sym::fsub_algebraic
@@ -574,6 +575,10 @@ pub fn check_intrinsic_type(
574575
(0, 0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
575576
}
576577

578+
// This type check is not particularly useful, but the `where` bounds
579+
// on the definition in `core` do the heavy lifting for checking it.
580+
sym::aggregate_raw_ptr => (3, 1, vec![param(1), param(2)], param(0)),
581+
577582
sym::ub_checks => (0, 1, Vec::new(), tcx.types.bool),
578583

579584
sym::simd_eq

‎compiler/rustc_middle/src/mir/pretty.rs

+9
Original file line numberDiff line numberDiff line change
@@ -1094,6 +1094,15 @@ impl<'tcx> Debug for Rvalue<'tcx> {
10941094

10951095
struct_fmt.finish()
10961096
}),
1097+
1098+
AggregateKind::RawPtr(pointee_ty, mutability) => {
1099+
let kind_str = match mutability {
1100+
Mutability::Mut => "mut",
1101+
Mutability::Not => "const",
1102+
};
1103+
with_no_trimmed_paths!(write!(fmt, "*{kind_str} {pointee_ty} from "))?;
1104+
fmt_tuple(fmt, "")
1105+
}
10971106
}
10981107
}
10991108

‎compiler/rustc_middle/src/mir/syntax.rs

+15
Original file line numberDiff line numberDiff line change
@@ -1351,6 +1351,21 @@ pub enum AggregateKind<'tcx> {
13511351
Closure(DefId, GenericArgsRef<'tcx>),
13521352
Coroutine(DefId, GenericArgsRef<'tcx>),
13531353
CoroutineClosure(DefId, GenericArgsRef<'tcx>),
1354+
1355+
/// Construct a raw pointer from the data pointer and metadata.
1356+
///
1357+
/// The `Ty` here is the type of the *pointee*, not the pointer itself.
1358+
/// The `Mutability` indicates whether this produces a `*const` or `*mut`.
1359+
///
1360+
/// The [`Rvalue::Aggregate`] operands for thus must be
1361+
///
1362+
/// 0. A raw pointer of matching mutability with any [`core::ptr::Thin`] pointee
1363+
/// 1. A value of the appropriate [`core::ptr::Pointee::Metadata`] type
1364+
///
1365+
/// *Both* operands must always be included, even the unit value if this is
1366+
/// creating a thin pointer. If you're just converting between thin pointers,
1367+
/// you may want an [`Rvalue::Cast`] with [`CastKind::PtrToPtr`] instead.
1368+
RawPtr(Ty<'tcx>, Mutability),
13541369
}
13551370

13561371
#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]

‎compiler/rustc_middle/src/mir/tcx.rs

+1
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ impl<'tcx> Rvalue<'tcx> {
206206
AggregateKind::CoroutineClosure(did, args) => {
207207
Ty::new_coroutine_closure(tcx, did, args)
208208
}
209+
AggregateKind::RawPtr(ty, mutability) => Ty::new_ptr(tcx, ty, mutability),
209210
},
210211
Rvalue::ShallowInitBox(_, ty) => Ty::new_box(tcx, ty),
211212
Rvalue::CopyForDeref(ref place) => place.ty(local_decls, tcx).ty,

‎compiler/rustc_middle/src/mir/visit.rs

+3
Original file line numberDiff line numberDiff line change
@@ -751,6 +751,9 @@ macro_rules! make_mir_visitor {
751751
) => {
752752
self.visit_args(coroutine_closure_args, location);
753753
}
754+
AggregateKind::RawPtr(ty, _) => {
755+
self.visit_ty($(& $mutability)? *ty, TyContext::Location(location));
756+
}
754757
}
755758

756759
for operand in operands {

‎compiler/rustc_mir_transform/src/gvn.rs

+3
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
885885
AggregateKind::Adt(did, ..) => tcx.def_kind(did) != DefKind::Enum,
886886
// Coroutines are never ZST, as they at least contain the implicit states.
887887
AggregateKind::Coroutine(..) => false,
888+
AggregateKind::RawPtr(..) => bug!("MIR for RawPtr aggregate must have 2 fields"),
888889
};
889890

890891
if is_zst {
@@ -910,6 +911,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
910911
}
911912
// Do not track unions.
912913
AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
914+
// FIXME: Do the extra work to GVN `from_raw_parts`
915+
AggregateKind::RawPtr(..) => return None,
913916
};
914917

915918
let fields: Option<Vec<_>> = fields

‎compiler/rustc_mir_transform/src/instsimplify.rs

+37-3
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ impl<'tcx> MirPass<'tcx> for InstSimplify {
3636
ctx.simplify_bool_cmp(&statement.source_info, rvalue);
3737
ctx.simplify_ref_deref(&statement.source_info, rvalue);
3838
ctx.simplify_len(&statement.source_info, rvalue);
39+
ctx.simplify_ptr_aggregate(&statement.source_info, rvalue);
3940
ctx.simplify_cast(rvalue);
4041
}
4142
_ => {}
@@ -58,8 +59,17 @@ struct InstSimplifyContext<'tcx, 'a> {
5859

5960
impl<'tcx> InstSimplifyContext<'tcx, '_> {
6061
fn should_simplify(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool {
62+
self.should_simplify_custom(source_info, "Rvalue", rvalue)
63+
}
64+
65+
fn should_simplify_custom(
66+
&self,
67+
source_info: &SourceInfo,
68+
label: &str,
69+
value: impl std::fmt::Debug,
70+
) -> bool {
6171
self.tcx.consider_optimizing(|| {
62-
format!("InstSimplify - Rvalue: {rvalue:?} SourceInfo: {source_info:?}")
72+
format!("InstSimplify - {label}: {value:?} SourceInfo: {source_info:?}")
6373
})
6474
}
6575

@@ -111,7 +121,7 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
111121
if a.const_.ty().is_bool() { a.const_.try_to_bool() } else { None }
112122
}
113123

114-
/// Transform "&(*a)" ==> "a".
124+
/// Transform `&(*a)` ==> `a`.
115125
fn simplify_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
116126
if let Rvalue::Ref(_, _, place) = rvalue {
117127
if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() {
@@ -131,7 +141,7 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
131141
}
132142
}
133143

134-
/// Transform "Len([_; N])" ==> "N".
144+
/// Transform `Len([_; N])` ==> `N`.
135145
fn simplify_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
136146
if let Rvalue::Len(ref place) = *rvalue {
137147
let place_ty = place.ty(self.local_decls, self.tcx).ty;
@@ -147,6 +157,30 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
147157
}
148158
}
149159

160+
/// Transform `Aggregate(RawPtr, [p, ()])` ==> `Cast(PtrToPtr, p)`.
161+
fn simplify_ptr_aggregate(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
162+
if let Rvalue::Aggregate(box AggregateKind::RawPtr(pointee_ty, mutability), fields) = rvalue
163+
{
164+
let meta_ty = fields.raw[1].ty(self.local_decls, self.tcx);
165+
if meta_ty.is_unit() {
166+
// The mutable borrows we're holding prevent printing `rvalue` here
167+
if !self.should_simplify_custom(
168+
source_info,
169+
"Aggregate::RawPtr",
170+
(&pointee_ty, *mutability, &fields),
171+
) {
172+
return;
173+
}
174+
175+
let mut fields = std::mem::take(fields);
176+
let _meta = fields.pop().unwrap();
177+
let data = fields.pop().unwrap();
178+
let ptr_ty = Ty::new_ptr(self.tcx, *pointee_ty, *mutability);
179+
*rvalue = Rvalue::Cast(CastKind::PtrToPtr, data, ptr_ty);
180+
}
181+
}
182+
}
183+
150184
fn simplify_ub_check(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
151185
if let Rvalue::NullaryOp(NullOp::UbChecks, _) = *rvalue {
152186
let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks());

‎compiler/rustc_mir_transform/src/known_panics_lint.rs

+1
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
603603
AggregateKind::Adt(_, variant, _, _, _) => variant,
604604
AggregateKind::Array(_)
605605
| AggregateKind::Tuple
606+
| AggregateKind::RawPtr(_, _)
606607
| AggregateKind::Closure(_, _)
607608
| AggregateKind::Coroutine(_, _)
608609
| AggregateKind::CoroutineClosure(_, _) => VariantIdx::ZERO,

0 commit comments

Comments
 (0)