Set metadata for vtable-related loads

Aatch · Aatch · commit 7af3406a49eb · 2017-02-21T21:08:06.000+13:00
Give LLVM much more information about vtable pointers. Without the extra information, LLVM has to be rather pessimistic about vtables, preventing a number of obvious optimisations. * Makes the vtable pointer argument noalias and readonly. * Marks loads of the vtable pointer as nonnull. * Marks load from the vtable with `!invariant.load` metadata. Fixes #39992
diff --git a/src/librustc_trans/abi.rs b/src/librustc_trans/abi.rs
@@ -506,7 +506,11 @@ impl FnType {
                 if let Some(inner) = rust_ptr_attrs(ty, &mut data) {
                     data.attrs.set(ArgAttribute::NonNull);
                     if ccx.tcx().struct_tail(inner).is_trait() {
+                        // vtables can be safely marked non-null, readonly
+                        // and noalias.
                         info.attrs.set(ArgAttribute::NonNull);
+                        info.attrs.set(ArgAttribute::ReadOnly);
+                        info.attrs.set(ArgAttribute::NoAlias);
                     }
                 }
                 args.push(data);
diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs
@@ -472,8 +472,15 @@ pub fn load_fat_ptr<'a, 'tcx>(
         b.load(ptr, alignment.to_align())
     };
 
-    // FIXME: emit metadata on `meta`.
-    let meta = b.load(get_meta(b, src), alignment.to_align());
+    let meta = get_meta(b, src);
+    let meta_ty = val_ty(meta);
+    // If the 'meta' field is a pointer, it's a vtable, so use load_nonnull
+    // instead
+    let meta = if meta_ty.element_type().kind() == llvm::TypeKind::Pointer {
+        b.load_nonnull(meta, None)
+    } else {
+        b.load(meta, None)
+    };
 
     (ptr, meta)
 }
diff --git a/src/librustc_trans/builder.rs b/src/librustc_trans/builder.rs
@@ -1149,6 +1149,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
+    pub fn set_invariant_load(&self, load: ValueRef) {
+        unsafe {
+            llvm::LLVMSetMetadata(load, llvm::MD_invariant_load as c_uint,
+                                  llvm::LLVMMDNodeInContext(self.ccx.llcx(), ptr::null(), 0));
+        }
+    }
+
     /// Returns the ptr value that should be used for storing `val`.
     fn check_store<'b>(&self,
                        val: ValueRef,
diff --git a/src/librustc_trans/glue.rs b/src/librustc_trans/glue.rs
@@ -386,7 +386,15 @@ pub fn size_and_align_of_dst<'a, 'tcx>(bcx: &Builder<'a, 'tcx>, t: Ty<'tcx>, inf
             let info = bcx.pointercast(info, Type::int(bcx.ccx).ptr_to());
             let size_ptr = bcx.gepi(info, &[1]);
             let align_ptr = bcx.gepi(info, &[2]);
-            (bcx.load(size_ptr, None), bcx.load(align_ptr, None))
+
+            let size = bcx.load(size_ptr, None);
+            let align = bcx.load(align_ptr, None);
+
+            // Vtable loads are invariant
+            bcx.set_invariant_load(size);
+            bcx.set_invariant_load(align);
+
+            (size, align)
         }
         ty::TySlice(_) | ty::TyStr => {
             let unit_ty = t.sequence_element_type(bcx.tcx());
diff --git a/src/librustc_trans/meth.rs b/src/librustc_trans/meth.rs
@@ -30,13 +30,15 @@ const VTABLE_OFFSET: usize = 3;
 /// Extracts a method from a trait object's vtable, at the specified index.
 pub fn get_virtual_method<'a, 'tcx>(bcx: &Builder<'a, 'tcx>,
                                     llvtable: ValueRef,
-                                    vtable_index: usize)
-                                    -> ValueRef {
+                                    vtable_index: usize) -> ValueRef {
     // Load the data pointer from the object.
     debug!("get_virtual_method(vtable_index={}, llvtable={:?})",
            vtable_index, Value(llvtable));
 
-    bcx.load(bcx.gepi(llvtable, &[vtable_index + VTABLE_OFFSET]), None)
+    let ptr = bcx.load_nonnull(bcx.gepi(llvtable, &[vtable_index + VTABLE_OFFSET]), None);
+    // Vtable loads are invariant
+    bcx.set_invariant_load(ptr);
+    ptr
 }
 
 /// Generate a shim function that allows an object type like `SomeTrait` to

Original file line number	Diff line number	Diff line change
`@@ -506,7 +506,11 @@ impl FnType {`
`506`	`506`	`if let Some(inner) = rust_ptr_attrs(ty, &mut data) {`
`507`	`507`	`data.attrs.set(ArgAttribute::NonNull);`
`508`	`508`	`if ccx.tcx().struct_tail(inner).is_trait() {`
	`509`	`+ // vtables can be safely marked non-null, readonly`
	`510`	`+ // and noalias.`
`509`	`511`	`info.attrs.set(ArgAttribute::NonNull);`
	`512`	`+ info.attrs.set(ArgAttribute::ReadOnly);`
	`513`	`+ info.attrs.set(ArgAttribute::NoAlias);`
`510`	`514`	`}`
`511`	`515`	`}`
`512`	`516`	`args.push(data);`
Original file line number	Diff line number	Diff line change
`@@ -1149,6 +1149,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {`
`1149`	`1149`	`}`
`1150`	`1150`	`}`
`1151`	`1151`
	`1152`	`+ pub fn set_invariant_load(&self, load: ValueRef) {`
	`1153`	`+ unsafe {`
	`1154`	`+ llvm::LLVMSetMetadata(load, llvm::MD_invariant_load as c_uint,`
	`1155`	`+ llvm::LLVMMDNodeInContext(self.ccx.llcx(), ptr::null(), 0));`
	`1156`	`+ }`
	`1157`	`+ }`
	`1158`	`+`
`1152`	`1159`	/// Returns the ptr value that should be used for storing `val`.
`1153`	`1160`	`fn check_store<'b>(&self,`
`1154`	`1161`	`val: ValueRef,`