Skip to content

Commit 42bd138

Browse files
borsbors
committed
Auto merge of #98145 - ouz-a:some_branch, r=oli-obk
Pull Derefer before ElaborateDrops _Follow up work to #97025 #96549 #96116 #95887 #95649_ This moves `Derefer` before `ElaborateDrops` and creates a new `Rvalue` called `VirtualRef` that allows us to bypass many constraints for `DerefTemp`. r? `@oli-obk`
2 parents ca4e394 + b4c3a2a commit 42bd138

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+401
-227
lines changed

‎compiler/rustc_borrowck/src/invalidation.rs

+4
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,10 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
289289
| Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => {
290290
self.consume_operand(location, operand)
291291
}
292+
Rvalue::CopyForDeref(ref place) => {
293+
let op = &Operand::Copy(*place);
294+
self.consume_operand(location, op);
295+
}
292296

293297
Rvalue::Len(place) | Rvalue::Discriminant(place) => {
294298
let af = match *rvalue {

‎compiler/rustc_borrowck/src/lib.rs

+17
Original file line numberDiff line numberDiff line change
@@ -1236,6 +1236,23 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
12361236
| Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => {
12371237
self.consume_operand(location, (operand, span), flow_state)
12381238
}
1239+
Rvalue::CopyForDeref(place) => {
1240+
self.access_place(
1241+
location,
1242+
(place, span),
1243+
(Deep, Read(ReadKind::Copy)),
1244+
LocalMutationIsAllowed::No,
1245+
flow_state,
1246+
);
1247+
1248+
// Finally, check if path was already moved.
1249+
self.check_if_path_or_subpath_is_moved(
1250+
location,
1251+
InitializationRequiringAction::Use,
1252+
(place.as_ref(), span),
1253+
flow_state,
1254+
);
1255+
}
12391256

12401257
Rvalue::Len(place) | Rvalue::Discriminant(place) => {
12411258
let af = match *rvalue {

‎compiler/rustc_borrowck/src/type_check/mod.rs

+5
Original file line numberDiff line numberDiff line change
@@ -2269,6 +2269,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
22692269
Rvalue::Use(operand) | Rvalue::UnaryOp(_, operand) => {
22702270
self.check_operand(operand, location);
22712271
}
2272+
Rvalue::CopyForDeref(place) => {
2273+
let op = &Operand::Copy(*place);
2274+
self.check_operand(op, location);
2275+
}
22722276

22732277
Rvalue::BinaryOp(_, box (left, right))
22742278
| Rvalue::CheckedBinaryOp(_, box (left, right)) => {
@@ -2299,6 +2303,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
22992303
| Rvalue::BinaryOp(..)
23002304
| Rvalue::CheckedBinaryOp(..)
23012305
| Rvalue::NullaryOp(..)
2306+
| Rvalue::CopyForDeref(..)
23022307
| Rvalue::UnaryOp(..)
23032308
| Rvalue::Discriminant(..) => None,
23042309

‎compiler/rustc_codegen_cranelift/src/base.rs

+5
Original file line numberDiff line numberDiff line change
@@ -503,6 +503,11 @@ fn codegen_stmt<'tcx>(
503503
let val = codegen_operand(fx, operand);
504504
lval.write_cvalue(fx, val);
505505
}
506+
Rvalue::CopyForDeref(place) => {
507+
let cplace = codegen_place(fx, place);
508+
let val = cplace.to_cvalue(fx);
509+
lval.write_cvalue(fx, val)
510+
}
506511
Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
507512
let place = codegen_place(fx, place);
508513
let ref_ = place.place_ref(fx, lval.layout());

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use crate::traits::*;
88
use crate::MemFlags;
99

1010
use rustc_middle::mir;
11+
use rustc_middle::mir::Operand;
1112
use rustc_middle::ty::cast::{CastTy, IntTy};
1213
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
1314
use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
@@ -344,6 +345,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
344345
self.codegen_place_to_pointer(bx, place, mk_ref)
345346
}
346347

348+
mir::Rvalue::CopyForDeref(place) => {
349+
let operand = self.codegen_operand(&mut bx, &Operand::Copy(place));
350+
(bx, operand)
351+
}
347352
mir::Rvalue::AddressOf(mutability, place) => {
348353
let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
349354
tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability })
@@ -698,6 +703,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
698703
pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
699704
match *rvalue {
700705
mir::Rvalue::Ref(..) |
706+
mir::Rvalue::CopyForDeref(..) |
701707
mir::Rvalue::AddressOf(..) |
702708
mir::Rvalue::Len(..) |
703709
mir::Rvalue::Cast(..) | // (*)

‎compiler/rustc_const_eval/src/interpret/step.rs

+5
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
172172
self.copy_op(&op, &dest, /*allow_transmute*/ false)?;
173173
}
174174

175+
CopyForDeref(ref place) => {
176+
let op = self.eval_place_to_op(*place, Some(dest.layout))?;
177+
self.copy_op(&op, &dest, /* allow_transmute*/ false)?;
178+
}
179+
175180
BinaryOp(bin_op, box (ref left, ref right)) => {
176181
let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
177182
let left = self.read_immediate(&self.eval_operand(left, layout)?)?;

‎compiler/rustc_const_eval/src/transform/check_consts/check.rs

+1
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
446446
Rvalue::ThreadLocalRef(_) => self.check_op(ops::ThreadLocalAccess),
447447

448448
Rvalue::Use(_)
449+
| Rvalue::CopyForDeref(..)
449450
| Rvalue::Repeat(..)
450451
| Rvalue::Discriminant(..)
451452
| Rvalue::Len(_)

‎compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs

+2
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,8 @@ where
260260
in_place::<Q, _>(cx, in_local, place.as_ref())
261261
}
262262

263+
Rvalue::CopyForDeref(place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
264+
263265
Rvalue::Use(operand)
264266
| Rvalue::Repeat(operand, _)
265267
| Rvalue::UnaryOp(_, operand)

‎compiler/rustc_const_eval/src/transform/check_consts/resolver.rs

+1
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ where
199199
mir::Rvalue::Cast(..)
200200
| mir::Rvalue::ShallowInitBox(..)
201201
| mir::Rvalue::Use(..)
202+
| mir::Rvalue::CopyForDeref(..)
202203
| mir::Rvalue::ThreadLocalRef(..)
203204
| mir::Rvalue::Repeat(..)
204205
| mir::Rvalue::Len(..)

‎compiler/rustc_const_eval/src/transform/promote_consts.rs

+4
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,10 @@ impl<'tcx> Validator<'_, 'tcx> {
494494
Rvalue::Use(operand) | Rvalue::Repeat(operand, _) => {
495495
self.validate_operand(operand)?;
496496
}
497+
Rvalue::CopyForDeref(place) => {
498+
let op = &Operand::Copy(*place);
499+
self.validate_operand(op)?
500+
}
497501

498502
Rvalue::Discriminant(place) | Rvalue::Len(place) => {
499503
self.validate_place(place.as_ref())?

‎compiler/rustc_const_eval/src/transform/validate.rs

+10-1
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
382382
};
383383
}
384384
match rvalue {
385-
Rvalue::Use(_) => {}
385+
Rvalue::Use(_) | Rvalue::CopyForDeref(_) => {}
386386
Rvalue::Aggregate(agg_kind, _) => {
387387
let disallowed = match **agg_kind {
388388
AggregateKind::Array(..) => false,
@@ -592,6 +592,15 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
592592
),
593593
);
594594
}
595+
if let Rvalue::CopyForDeref(place) = rvalue {
596+
if !place.ty(&self.body.local_decls, self.tcx).ty.builtin_deref(true).is_some()
597+
{
598+
self.fail(
599+
location,
600+
"`CopyForDeref` should only be used for dereferenceable types",
601+
)
602+
}
603+
}
595604
// FIXME(JakobDegen): Check this for all rvalues, not just this one.
596605
if let Rvalue::Use(Operand::Copy(src) | Operand::Move(src)) = rvalue {
597606
// The sides of an assignment must not alias. Currently this just checks whether

‎compiler/rustc_middle/src/mir/mod.rs

+13-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ mod syntax;
6161
pub use syntax::*;
6262
mod switch_sources;
6363
pub mod tcx;
64-
mod terminator;
64+
pub mod terminator;
6565
pub use terminator::*;
6666

6767
pub mod traversal;
@@ -925,6 +925,15 @@ impl<'tcx> LocalDecl<'tcx> {
925925
}
926926
}
927927

928+
/// Returns `true` if this is a DerefTemp
929+
pub fn is_deref_temp(&self) -> bool {
930+
match self.local_info {
931+
Some(box LocalInfo::DerefTemp) => return true,
932+
_ => (),
933+
}
934+
return false;
935+
}
936+
928937
/// Returns `true` is the local is from a compiler desugaring, e.g.,
929938
/// `__next` from a `for` loop.
930939
#[inline]
@@ -1795,6 +1804,7 @@ impl<'tcx> Rvalue<'tcx> {
17951804
Rvalue::Cast(CastKind::PointerExposeAddress, _, _) => false,
17961805

17971806
Rvalue::Use(_)
1807+
| Rvalue::CopyForDeref(_)
17981808
| Rvalue::Repeat(_, _)
17991809
| Rvalue::Ref(_, _, _)
18001810
| Rvalue::ThreadLocalRef(_)
@@ -1889,6 +1899,8 @@ impl<'tcx> Debug for Rvalue<'tcx> {
18891899
write!(fmt, "&{}{}{:?}", region, kind_str, place)
18901900
}
18911901

1902+
CopyForDeref(ref place) => write!(fmt, "deref_copy {:#?}", place),
1903+
18921904
AddressOf(mutability, ref place) => {
18931905
let kind_str = match mutability {
18941906
Mutability::Mut => "mut",

‎compiler/rustc_middle/src/mir/syntax.rs

+13-3
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ pub enum MirPhase {
5252
/// of the `mir_promoted` query), these promoted elements are available in the `promoted_mir`
5353
/// query.
5454
ConstsPromoted = 2,
55+
/// After this projections may only contain deref projections as the first element.
56+
Derefered = 3,
5557
/// Beginning with this phase, the following variants are disallowed:
5658
/// * [`TerminatorKind::DropAndReplace`]
5759
/// * [`TerminatorKind::FalseUnwind`]
@@ -66,9 +68,7 @@ pub enum MirPhase {
6668
/// Furthermore, `Drop` now uses explicit drop flags visible in the MIR and reaching a `Drop`
6769
/// terminator means that the auto-generated drop glue will be invoked. Also, `Copy` operands
6870
/// are allowed for non-`Copy` types.
69-
DropsLowered = 3,
70-
/// After this projections may only contain deref projections as the first element.
71-
Derefered = 4,
71+
DropsLowered = 4,
7272
/// Beginning with this phase, the following variant is disallowed:
7373
/// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
7474
///
@@ -1051,6 +1051,16 @@ pub enum Rvalue<'tcx> {
10511051
/// initialized but its content as uninitialized. Like other pointer casts, this in general
10521052
/// affects alias analysis.
10531053
ShallowInitBox(Operand<'tcx>, Ty<'tcx>),
1054+
1055+
/// A CopyForDeref is equivalent to a read from a place at the
1056+
/// codegen level, but is treated specially by drop elaboration. When such a read happens, it
1057+
/// is guaranteed (via nature of the mir_opt `Derefer` in rustc_mir_transform/src/deref_separator)
1058+
/// that the only use of the returned value is a deref operation, immediately
1059+
/// followed by one or more projections. Drop elaboration treats this rvalue as if the
1060+
/// read never happened and just projects further. This allows simplifying various MIR
1061+
/// optimizations and codegen backends that previously had to handle deref operations anywhere
1062+
/// in a place.
1063+
CopyForDeref(Place<'tcx>),
10541064
}
10551065

10561066
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]

‎compiler/rustc_middle/src/mir/tcx.rs

+1
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ impl<'tcx> Rvalue<'tcx> {
211211
}
212212
},
213213
Rvalue::ShallowInitBox(_, ty) => tcx.mk_box(ty),
214+
Rvalue::CopyForDeref(ref place) => place.ty(local_decls, tcx).ty,
214215
}
215216
}
216217

‎compiler/rustc_middle/src/mir/type_foldable.rs

+1
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
122122
Ref(region, bk, place) => {
123123
Ref(region.try_fold_with(folder)?, bk, place.try_fold_with(folder)?)
124124
}
125+
CopyForDeref(place) => CopyForDeref(place.try_fold_with(folder)?),
125126
AddressOf(mutability, place) => AddressOf(mutability, place.try_fold_with(folder)?),
126127
Len(place) => Len(place.try_fold_with(folder)?),
127128
Cast(kind, op, ty) => Cast(kind, op.try_fold_with(folder)?, ty.try_fold_with(folder)?),

‎compiler/rustc_middle/src/mir/type_visitable.rs

+4
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ impl<'tcx> TypeVisitable<'tcx> for Rvalue<'tcx> {
7878
use crate::mir::Rvalue::*;
7979
match *self {
8080
Use(ref op) => op.visit_with(visitor),
81+
CopyForDeref(ref place) => {
82+
let op = &Operand::Copy(*place);
83+
op.visit_with(visitor)
84+
}
8185
Repeat(ref op, _) => op.visit_with(visitor),
8286
ThreadLocalRef(did) => did.visit_with(visitor),
8387
Ref(region, _, ref place) => {

‎compiler/rustc_middle/src/mir/visit.rs

+7
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,13 @@ macro_rules! make_mir_visitor {
711711
};
712712
self.visit_place(path, ctx, location);
713713
}
714+
Rvalue::CopyForDeref(place) => {
715+
self.visit_place(
716+
place,
717+
PlaceContext::NonMutatingUse(NonMutatingUseContext::Inspect),
718+
location
719+
);
720+
}
714721

715722
Rvalue::AddressOf(m, path) => {
716723
let ctx = match m {

‎compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,8 @@ where
102102
| mir::Rvalue::NullaryOp(..)
103103
| mir::Rvalue::UnaryOp(..)
104104
| mir::Rvalue::Discriminant(..)
105-
| mir::Rvalue::Aggregate(..) => {}
105+
| mir::Rvalue::Aggregate(..)
106+
| mir::Rvalue::CopyForDeref(..) => {}
106107
}
107108
}
108109

‎compiler/rustc_mir_dataflow/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ pub mod impls;
3838
pub mod move_paths;
3939
pub mod rustc_peek;
4040
pub mod storage;
41+
pub mod un_derefer;
4142

4243
pub(crate) mod indexes {
4344
pub(crate) use super::move_paths::MovePathIndex;

‎compiler/rustc_mir_dataflow/src/move_paths/builder.rs

+29-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::un_derefer::UnDerefer;
12
use rustc_index::vec::IndexVec;
23
use rustc_middle::mir::tcx::RvalueInitializationState;
34
use rustc_middle::mir::*;
@@ -19,6 +20,7 @@ struct MoveDataBuilder<'a, 'tcx> {
1920
param_env: ty::ParamEnv<'tcx>,
2021
data: MoveData<'tcx>,
2122
errors: Vec<(Place<'tcx>, MoveError<'tcx>)>,
23+
un_derefer: UnDerefer<'tcx>,
2224
}
2325

2426
impl<'a, 'tcx> MoveDataBuilder<'a, 'tcx> {
@@ -32,6 +34,7 @@ impl<'a, 'tcx> MoveDataBuilder<'a, 'tcx> {
3234
tcx,
3335
param_env,
3436
errors: Vec::new(),
37+
un_derefer: UnDerefer { tcx: tcx, derefer_sidetable: Default::default() },
3538
data: MoveData {
3639
moves: IndexVec::new(),
3740
loc_map: LocationMap::new(body),
@@ -94,6 +97,11 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
9497
///
9598
/// Maybe we should have separate "borrowck" and "moveck" modes.
9699
fn move_path_for(&mut self, place: Place<'tcx>) -> Result<MovePathIndex, MoveError<'tcx>> {
100+
if let Some(new_place) = self.builder.un_derefer.derefer(place.as_ref(), self.builder.body)
101+
{
102+
return self.move_path_for(new_place);
103+
}
104+
97105
debug!("lookup({:?})", place);
98106
let mut base = self.builder.data.rev_lookup.locals[place.local];
99107

@@ -276,6 +284,12 @@ struct Gatherer<'b, 'a, 'tcx> {
276284
impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
277285
fn gather_statement(&mut self, stmt: &Statement<'tcx>) {
278286
match &stmt.kind {
287+
StatementKind::Assign(box (place, Rvalue::CopyForDeref(reffed))) => {
288+
assert!(place.projection.is_empty());
289+
if self.builder.body.local_decls[place.local].is_deref_temp() {
290+
self.builder.un_derefer.derefer_sidetable.insert(place.local, *reffed);
291+
}
292+
}
279293
StatementKind::Assign(box (place, rval)) => {
280294
self.create_move_path(*place);
281295
if let RvalueInitializationState::Shallow = rval.initialization_state() {
@@ -294,7 +308,10 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
294308
}
295309
StatementKind::StorageLive(_) => {}
296310
StatementKind::StorageDead(local) => {
297-
self.gather_move(Place::from(*local));
311+
// DerefTemp locals (results of CopyForDeref) don't actually move anything.
312+
if !self.builder.un_derefer.derefer_sidetable.contains_key(&local) {
313+
self.gather_move(Place::from(*local));
314+
}
298315
}
299316
StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) => {
300317
span_bug!(
@@ -328,6 +345,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
328345
self.gather_operand(operand);
329346
}
330347
}
348+
Rvalue::CopyForDeref(..) => unreachable!(),
331349
Rvalue::Ref(..)
332350
| Rvalue::AddressOf(..)
333351
| Rvalue::Discriminant(..)
@@ -439,6 +457,11 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
439457

440458
fn gather_move(&mut self, place: Place<'tcx>) {
441459
debug!("gather_move({:?}, {:?})", self.loc, place);
460+
if let Some(new_place) = self.builder.un_derefer.derefer(place.as_ref(), self.builder.body)
461+
{
462+
self.gather_move(new_place);
463+
return;
464+
}
442465

443466
if let [ref base @ .., ProjectionElem::Subslice { from, to, from_end: false }] =
444467
**place.projection
@@ -494,6 +517,11 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
494517
fn gather_init(&mut self, place: PlaceRef<'tcx>, kind: InitKind) {
495518
debug!("gather_init({:?}, {:?})", self.loc, place);
496519

520+
if let Some(new_place) = self.builder.un_derefer.derefer(place, self.builder.body) {
521+
self.gather_init(new_place.as_ref(), kind);
522+
return;
523+
}
524+
497525
let mut place = place;
498526

499527
// Check if we are assigning into a field of a union, if so, lookup the place

0 commit comments

Comments
 (0)