Introduce TypedMetadata wrapper to coerce metadata

compiler/rustc_codegen_ssa/src/base.rs

@@ -37,6 +37,7 @@ use rustc_span::symbol::sym;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
 use rustc_target::abi::{Align, VariantIdx};
 
+use std::assert_matches::assert_matches;
 use std::collections::BTreeSet;
 use std::convert::TryFrom;
 use std::ops::{Deref, DerefMut};
@@ -197,21 +198,21 @@ pub fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     src_ty: Ty<'tcx>,
     dst_ty: Ty<'tcx>,
     old_info: Option<Bx::Value>,
-) -> (Bx::Value, Bx::Value) {
+) -> OperandValue<Bx::Value> {
     debug!("unsize_ptr: {:?} => {:?}", src_ty, dst_ty);
+    let src_layout = bx.cx().layout_of(src_ty);
+    let dst_layout = bx.cx().layout_of(dst_ty);
     match (src_ty.kind(), dst_ty.kind()) {
         (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(ty::TypeAndMut { ty: b, .. }))
         | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => {
             assert_eq!(bx.cx().type_is_sized(a), old_info.is_none());
             let ptr_ty = bx.cx().type_ptr_to(bx.cx().backend_type(bx.cx().layout_of(b)));
-            (bx.pointercast(src, ptr_ty), unsized_info(bx, a, b, old_info))
+            OperandValue::Pair(bx.pointercast(src, ptr_ty), unsized_info(bx, a, b, old_info))
         }
-        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
+        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if bx.cx().is_backend_scalar_pair(dst_layout) => {
             assert_eq!(def_a, def_b);
-            let src_layout = bx.cx().layout_of(src_ty);
-            let dst_layout = bx.cx().layout_of(dst_ty);
             if src_ty == dst_ty {
-                return (src, old_info.unwrap());
+                return OperandValue::Pair(src, old_info.unwrap());
             }
             let mut result = None;
             for i in 0..src_layout.fields.count() {
@@ -226,16 +227,29 @@ pub fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
                 let dst_f = dst_layout.field(bx.cx(), i);
                 assert_ne!(src_f.ty, dst_f.ty);
-                assert_eq!(result, None);
+                assert_matches!(result, None);
                 result = Some(unsize_ptr(bx, src, src_f.ty, dst_f.ty, old_info));
             }
-            let (lldata, llextra) = result.unwrap();
+            let OperandValue::Pair(lldata, llextra) = result.unwrap()
+            else { bug!() };
             let lldata_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 0, true);
             let llextra_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 1, true);
             // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
-            (bx.bitcast(lldata, lldata_ty), bx.bitcast(llextra, llextra_ty))
+            OperandValue::Pair(bx.bitcast(lldata, lldata_ty), bx.bitcast(llextra, llextra_ty))
+        }
+        (&ty::Adt(def_a, substs_a), &ty::Adt(def_b, substs_b)) => {
+            assert_eq!(def_a, def_b);
+            let typed_metadata = bx.tcx().require_lang_item(LangItem::TypedMetadata, None);
+            assert_eq!(def_a.did(), typed_metadata);
+            if src_ty == dst_ty {
+                return OperandValue::Immediate(src);
+            }
+            let a = substs_a.type_at(0);
+            let b = substs_b.type_at(0);
+            assert_eq!(bx.cx().type_is_sized(a), old_info.is_none());
+            OperandValue::Immediate(unsized_info(bx, a, b, old_info))
         }
-        _ => bug!("unsize_ptr: called on bad types"),
+        _ => bug!("unsize_ptr: called on bad types {:?} -> {:?}", src_ty, dst_ty),
     }
 }
 
@@ -250,12 +264,12 @@ pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let dst_ty = dst.layout.ty;
     match (src_ty.kind(), dst_ty.kind()) {
         (&ty::Ref(..), &ty::Ref(..) | &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => {
-            let (base, info) = match bx.load_operand(src).val {
+            let val = match bx.load_operand(src).val {
                 OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)),
                 OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None),
                 OperandValue::Ref(..) => bug!(),
             };
-            OperandValue::Pair(base, info).store(bx, dst);
+            val.store(bx, dst);
         }
 
         (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {

compiler/rustc_codegen_ssa/src/lib.rs

@@ -1,4 +1,5 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(try_blocks)]
 #![feature(let_else)]

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -225,7 +225,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         operand.val
                     }
                     mir::CastKind::Pointer(PointerCast::Unsize) => {
-                        assert!(bx.cx().is_backend_scalar_pair(cast));
                         let (lldata, llextra) = match operand.val {
                             OperandValue::Pair(lldata, llextra) => {
                                 // unsize from a fat pointer -- this is a
@@ -240,9 +239,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand);
                             }
                         };
-                        let (lldata, llextra) =
-                            base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra);
-                        OperandValue::Pair(lldata, llextra)
+                        base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra)
                     }
                     mir::CastKind::Pointer(PointerCast::MutToConstPointer)
                     | mir::CastKind::Misc

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -3,7 +3,7 @@ use std::convert::TryFrom;
 
 use rustc_apfloat::ieee::{Double, Single};
 use rustc_apfloat::{Float, FloatConvert};
-use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar};
+use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar, ScalarMaybeUninit};
 use rustc_middle::mir::CastKind;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::{IntegerExt, LayoutOf, TyAndLayout};
@@ -305,22 +305,37 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         source_ty: Ty<'tcx>,
         cast_ty: Ty<'tcx>,
     ) -> InterpResult<'tcx> {
+        // We *could* forward `data` without even checking that it is initialized, but for now,
+        // let's only allow casting properly initialized pointers.
+        let (data, old_meta) = match *self.read_immediate(src)? {
+            // If the input ptr is thin, use `Uninit` for the old metadata.
+            // `unsize_just_metadata` knows how to handle that.
+            Immediate::Scalar(data) => (data.check_init()?, ScalarMaybeUninit::Uninit),
+            Immediate::ScalarPair(data, meta) => (data.check_init()?, meta),
+        };
+
+        let new_meta = self.unsize_just_metadata(old_meta, source_ty, cast_ty)?;
+        self.write_immediate(Immediate::ScalarPair(data.into(), new_meta.into()), dest)
+    }
+
+    fn unsize_just_metadata(
+        &mut self,
+        src_meta: ScalarMaybeUninit<M::PointerTag>,
+        // The pointee types
+        source_ty: Ty<'tcx>,
+        cast_ty: Ty<'tcx>,
+    ) -> InterpResult<'tcx, Scalar<M::PointerTag>> {
         // A<Struct> -> A<Trait> conversion
         let (src_pointee_ty, dest_pointee_ty) =
             self.tcx.struct_lockstep_tails_erasing_lifetimes(source_ty, cast_ty, self.param_env);
 
-        match (&src_pointee_ty.kind(), &dest_pointee_ty.kind()) {
+        Ok(match (&src_pointee_ty.kind(), &dest_pointee_ty.kind()) {
             (&ty::Array(_, length), &ty::Slice(_)) => {
-                let ptr = self.read_immediate(src)?.to_scalar()?;
-                // u64 cast is from usize to u64, which is always good
-                let val =
-                    Immediate::new_slice(ptr, length.eval_usize(*self.tcx, self.param_env), self);
-                self.write_immediate(val, dest)
+                Scalar::from_machine_usize(length.eval_usize(*self.tcx, self.param_env), self)
             }
             (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => {
-                let val = self.read_immediate(src)?;
                 if data_a.principal_def_id() == data_b.principal_def_id() {
-                    return self.write_immediate(*val, dest);
+                    return src_meta.check_init();
                 }
                 // trait upcasting coercion
                 let vptr_entry_idx = self.tcx.vtable_trait_upcasting_coercion_new_vptr_slot((
@@ -330,27 +345,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
                 if let Some(entry_idx) = vptr_entry_idx {
                     let entry_idx = u64::try_from(entry_idx).unwrap();
-                    let (old_data, old_vptr) = val.to_scalar_pair()?;
+                    let old_vptr = src_meta.check_init()?;
                     let old_vptr = self.scalar_to_ptr(old_vptr)?;
                     let new_vptr = self
                         .read_new_vtable_after_trait_upcasting_from_vtable(old_vptr, entry_idx)?;
-                    self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest)
+                    Scalar::from_maybe_pointer(new_vptr, self)
                 } else {
-                    self.write_immediate(*val, dest)
+                    src_meta.check_init()?
                 }
             }
             (_, &ty::Dynamic(ref data, _)) => {
                 // Initial cast from sized to dyn trait
                 let vtable = self.get_vtable(src_pointee_ty, data.principal())?;
-                let ptr = self.read_immediate(src)?.to_scalar()?;
-                let val = Immediate::new_dyn_trait(ptr, vtable, &*self.tcx);
-                self.write_immediate(val, dest)
+                Scalar::from_maybe_pointer(vtable, &*self.tcx)
             }
 
             _ => {
-                span_bug!(self.cur_span(), "invalid unsizing {:?} -> {:?}", src.layout.ty, cast_ty)
+                span_bug!(self.cur_span(), "invalid unsizing {:?} -> {:?}", source_ty, cast_ty)
             }
-        }
+        })
     }
 
     fn unsize_into(
@@ -360,16 +373,40 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         dest: &PlaceTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx> {
         trace!("Unsizing {:?} of type {} into {:?}", *src, src.layout.ty, cast_ty.ty);
-        match (&src.layout.ty.kind(), &cast_ty.ty.kind()) {
-            (&ty::Ref(_, s, _), &ty::Ref(_, c, _) | &ty::RawPtr(TypeAndMut { ty: c, .. }))
-            | (&ty::RawPtr(TypeAndMut { ty: s, .. }), &ty::RawPtr(TypeAndMut { ty: c, .. })) => {
+        let typed_metadata = self.tcx.lang_items().typed_metadata();
+        match (src.layout.ty.kind(), cast_ty.ty.kind()) {
+            (ty::Ref(_, s, _), ty::Ref(_, c, _) | ty::RawPtr(TypeAndMut { ty: c, .. }))
+            | (ty::RawPtr(TypeAndMut { ty: s, .. }), ty::RawPtr(TypeAndMut { ty: c, .. })) => {
                 self.unsize_into_ptr(src, dest, *s, *c)
             }
-            (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
+            (ty::Adt(def_a, _), ty::Adt(def_b, _)) if def_a.is_box() || def_b.is_box() => {
                 assert_eq!(def_a, def_b);
-
+                if !def_a.is_box() || !def_b.is_box() {
+                    span_bug!(
+                        self.cur_span(),
+                        "invalid unsizing between {:?} -> {:?}",
+                        src.layout.ty,
+                        cast_ty.ty
+                    );
+                }
+                self.unsize_into_ptr(src, dest, src.layout.ty.boxed_ty(), cast_ty.ty.boxed_ty())
+            }
+            (ty::Adt(def_a, substs_a), ty::Adt(def_b, substs_b))
+                if def_a == def_b && Some(def_a.did()) == typed_metadata =>
+            {
+                // unsizing of TypedMetadata container
+                // Example: `TypedMetadata<T>` -> `TypedMetadata<dyn Trait>`
+                let a_pointee = substs_a.type_at(0);
+                let b_pointee = substs_b.type_at(0);
+                let src_field = self.operand_field(src, 0)?;
+                let src = self.read_immediate(&src_field)?.to_scalar_or_uninit();
+                let dst_field = self.place_field(dest, 0)?;
+                let new_meta = self.unsize_just_metadata(src, a_pointee, b_pointee)?;
+                self.write_scalar(new_meta, &dst_field)
+            }
+            (ty::Adt(def_a, _), ty::Adt(def_b, _)) if def_a == def_b => {
                 // unsizing of generic struct with pointer fields
-                // Example: `Arc<T>` -> `Arc<Trait>`
+                // Example: `Arc<T>` -> `Arc<dyn Trait>`
                 // here we need to increase the size of every &T thin ptr field to a fat ptr
                 for i in 0..src.layout.fields.count() {
                     let cast_ty_field = cast_ty.field(self, i);

compiler/rustc_hir/src/lang_items.rs

@@ -182,6 +182,7 @@ language_item_table! {
     PointeeTrait,            sym::pointee_trait,       pointee_trait,              Target::Trait,          GenericRequirement::None;
     Metadata,                sym::metadata_type,       metadata_type,              Target::AssocTy,        GenericRequirement::None;
     DynMetadata,             sym::dyn_metadata,        dyn_metadata,               Target::Struct,         GenericRequirement::None;
+    TypedMetadata,           sym::typed_metadata,      typed_metadata,             Target::Struct,         GenericRequirement::Exact(1);
 
     Freeze,                  sym::freeze,              freeze_trait,               Target::Trait,          GenericRequirement::Exact(0);
 

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -215,7 +215,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
     impl_polarity => { table_direct }
     impl_defaultness => { table_direct }
     constness => { table_direct }
-    coerce_unsized_info => { table }
+    coerce_unsized_kind => { table }
     mir_const_qualif => { table }
     rendered_const => { table }
     asyncness => { table_direct }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1482,9 +1482,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     // if this is an impl of `CoerceUnsized`, create its
                     // "unsized info", else just store None
                     if Some(trait_ref.def_id) == self.tcx.lang_items().coerce_unsized_trait() {
-                        let coerce_unsized_info =
-                            self.tcx.at(item.span).coerce_unsized_info(def_id);
-                        record!(self.tables.coerce_unsized_info[def_id] <- coerce_unsized_info);
+                        let coerce_unsized_kind =
+                            self.tcx.at(item.span).coerce_unsized_kind(def_id);
+                        record!(self.tables.coerce_unsized_kind[def_id] <- coerce_unsized_kind);
                     }
                 }
 

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -368,7 +368,7 @@ define_tables! {
     is_intrinsic: Table<DefIndex, ()>,
     impl_defaultness: Table<DefIndex, hir::Defaultness>,
     // FIXME(eddyb) perhaps compute this on the fly if cheap enough?
-    coerce_unsized_info: Table<DefIndex, LazyValue<ty::adjustment::CoerceUnsizedInfo>>,
+    coerce_unsized_kind: Table<DefIndex, LazyValue<ty::adjustment::CoerceUnsizedKind>>,
     mir_const_qualif: Table<DefIndex, LazyValue<mir::ConstQualifs>>,
     rendered_const: Table<DefIndex, LazyValue<String>>,
     asyncness: Table<DefIndex, hir::IsAsync>,

compiler/rustc_middle/src/query/mod.rs

@@ -835,8 +835,8 @@ rustc_queries! {
     }
 
     /// Caches `CoerceUnsized` kinds for impls on custom types.
-    query coerce_unsized_info(key: DefId) -> ty::adjustment::CoerceUnsizedInfo {
-        desc { |tcx| "computing CoerceUnsized info for `{}`", tcx.def_path_str(key) }
+    query coerce_unsized_kind(key: DefId) -> ty::adjustment::CoerceUnsizedKind {
+        desc { |tcx| "computing CoerceUnsized kind for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }

compiler/rustc_middle/src/ty/adjustment.rs

@@ -177,20 +177,15 @@ pub enum AutoBorrow<'tcx> {
 /// Information for `CoerceUnsized` impls, storing information we
 /// have computed about the coercion.
 ///
-/// This struct can be obtained via the `coerce_impl_info` query.
+/// This enum can be obtained via the `coerce_unsized_kind` query.
 /// Demanding this struct also has the side-effect of reporting errors
 /// for inappropriate impls.
-#[derive(Clone, Copy, TyEncodable, TyDecodable, Debug, HashStable)]
-pub struct CoerceUnsizedInfo {
-    /// If this is a "custom coerce" impl, then what kind of custom
-    /// coercion is it? This applies to impls of `CoerceUnsized` for
-    /// structs, primarily, where we store a bit of info about which
-    /// fields need to be coerced.
-    pub custom_kind: Option<CustomCoerceUnsized>,
-}
-
-#[derive(Clone, Copy, TyEncodable, TyDecodable, Debug, HashStable)]
-pub enum CustomCoerceUnsized {
-    /// Records the index of the field being coerced.
+#[derive(Clone, Copy, PartialEq, TyEncodable, TyDecodable, Debug, HashStable)]
+pub enum CoerceUnsizedKind {
+    /// Coercion of a raw pointer or ref.
+    Ptr,
+    /// Coercion of a struct. Records the index of the field being coerced.
     Struct(usize),
+    /// Coercion of the pointee metadata directly.
+    TypedMetadata,
 }

compiler/rustc_middle/src/ty/parameterized.rs

@@ -59,7 +59,7 @@ trivially_parameterized_over_tcx! {
     ty::ReprOptions,
     ty::TraitDef,
     ty::Visibility,
-    ty::adjustment::CoerceUnsizedInfo,
+    ty::adjustment::CoerceUnsizedKind,
     ty::fast_reject::SimplifiedTypeGen<DefId>,
     rustc_ast::Attribute,
     rustc_ast::MacArgs,