Auto merge of #154144 - Human9000-bit:const-prop-transmute-async-fix, r=cjgillot

Fix malformed transmute handling during mir build



Running dataflow const prop optimization on  `std::mem::transmute` of mismatched sizes failed the assertion in interpreter and caused ICE.

So it's better to not let those transmutations into the interpreter at all

Fixes #149920

Author: bors

compiler/rustc_hir_typeck/src/intrinsicck.rs

@@ -8,6 +8,7 @@ use rustc_index::Idx;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutError, SizeSkeleton};
 use rustc_middle::ty::{self, Ty, TyCtxt, Unnormalized};
+use rustc_span::ErrorGuaranteed;
 use rustc_span::def_id::LocalDefId;
 use tracing::trace;
 
@@ -72,7 +73,7 @@ fn check_transmute<'tcx>(
     from: Ty<'tcx>,
     to: Ty<'tcx>,
     hir_id: HirId,
-) {
+) -> Result<(), ErrorGuaranteed> {
     let span = tcx.hir_span(hir_id);
     let normalize = |ty| {
         if let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, Unnormalized::new_wip(ty)) {
@@ -92,7 +93,7 @@ fn check_transmute<'tcx>(
 
     // Transmutes that are only changing lifetimes are always ok.
     if from == to {
-        return;
+        return Ok(());
     }
 
     let sk_from = SizeSkeleton::compute(from, tcx, typing_env, span);
@@ -104,7 +105,7 @@ fn check_transmute<'tcx>(
         && let Ok(sk_to) = sk_to
     {
         if sk_from.same_size(sk_to) {
-            return;
+            return Ok(());
         }
 
         // Special-case transmuting from `typeof(function)` and
@@ -119,7 +120,7 @@ fn check_transmute<'tcx>(
                 .with_note(format!("target type: {to}"))
                 .with_help("cast with `as` to a pointer instead")
                 .emit();
-            return;
+            return Ok(());
         }
     }
 
@@ -131,21 +132,25 @@ fn check_transmute<'tcx>(
     );
     if from == to {
         err.note(format!("`{from}` does not have a fixed size"));
-        err.emit();
+        Err(err.emit())
     } else {
         err.note(format!("source type: `{}` ({})", from, skeleton_string(from, sk_from)));
         err.note(format!("target type: `{}` ({})", to, skeleton_string(to, sk_to)));
-        err.emit();
+        Err(err.emit())
     }
 }
 
-pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) {
+pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) -> Result<(), ErrorGuaranteed> {
     assert!(!tcx.is_typeck_child(owner.to_def_id()));
     let typeck_results = tcx.typeck(owner);
-    let None = typeck_results.tainted_by_errors else { return };
+    if let Some(e) = typeck_results.tainted_by_errors {
+        return Err(e);
+    };
 
     let typing_env = ty::TypingEnv::post_analysis(tcx, owner);
+    let mut result = Ok(());
     for &(from, to, hir_id) in &typeck_results.transmutes_to_check {
-        check_transmute(tcx, typing_env, from, to, hir_id);
+        result = result.and(check_transmute(tcx, typing_env, from, to, hir_id));
     }
+    result
 }

compiler/rustc_middle/src/queries.rs

@@ -1118,7 +1118,7 @@ rustc_queries! {
     }
 
     /// Unsafety-check this `LocalDefId`.
-    query check_transmutes(key: LocalDefId) {
+    query check_transmutes(key: LocalDefId) -> Result<(), ErrorGuaranteed> {
         desc { "check transmute calls inside `{}`", tcx.def_path_str(key) }
     }
 

compiler/rustc_mir_transform/src/lib.rs

@@ -544,11 +544,15 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
         body.tainted_by_errors = Some(error_reported);
     }
 
+    let root = tcx.typeck_root_def_id_local(def);
+    if let Err(e) = tcx.check_transmutes(root) {
+        body.tainted_by_errors = Some(e);
+    }
+
     // Also taint the body if it's within a top-level item that is not well formed.
     //
     // We do this check here and not during `mir_promoted` because that may result
     // in borrowck cycles if WF requires looking into an opaque hidden type.
-    let root = tcx.typeck_root_def_id_local(def);
     match tcx.def_kind(root) {
         DefKind::Fn
         | DefKind::AssocFn

tests/rustdoc-ui/issues/issue-79494.rs

@@ -1,4 +1,4 @@
 //@ only-64bit
 
 pub const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-//~^ ERROR transmuting from 8-byte type to 16-byte type
+//~^ ERROR: cannot transmute between types of different sizes, or dependently-sized types [E0512]

tests/rustdoc-ui/issues/issue-79494.stderr

@@ -1,9 +1,12 @@
-error[E0080]: transmuting from 8-byte type to 16-byte type: `usize` -> `&[u8]`
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/issue-79494.rs:3:33
    |
 LL | pub const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-   |                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `ZST` failed here
+   |                                 ^^^^^^^^^^^^^^^^^^^
+   |
+   = note: source type: `usize` (64 bits)
+   = note: target type: `&[u8]` (128 bits)
 
 error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0512`.

tests/ui/consts/transmute-size-mismatch-before-typeck.rs

@@ -14,5 +14,4 @@ fn main() {
 }
 
 const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-//~^ ERROR transmuting from
-//~| ERROR cannot transmute between types of different sizes
+//~^ ERROR cannot transmute between types of different sizes

tests/ui/consts/transmute-size-mismatch-before-typeck.stderr

@@ -1,9 +1,3 @@
-error[E0080]: transmuting from word size type to 2 * word size type: `usize` -> `&[u8]`
-  --> $DIR/transmute-size-mismatch-before-typeck.rs:16:29
-   |
-LL | const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-   |                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `ZST` failed here
-
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/transmute-size-mismatch-before-typeck.rs:16:29
    |
@@ -13,7 +7,6 @@ LL | const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
    = note: source type: `usize` (word size)
    = note: target type: `&[u8]` (2 * word size)
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
-Some errors have detailed explanations: E0080, E0512.
-For more information about an error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0512`.

tests/ui/layout/base-layout-is-sized-ice-123078.rs

@@ -8,8 +8,7 @@ struct S {
 }
 
 const C: S = unsafe { std::mem::transmute(()) };
-//~^ ERROR the type `S` has an unknown layout
-//~| ERROR cannot transmute between types of different sizes, or dependently-sized types
+//~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
 
 const _: [(); {
     C;

tests/ui/layout/base-layout-is-sized-ice-123078.stderr

@@ -16,12 +16,6 @@ help: the `Box` type always has a statically known size and allocates its conten
 LL |     a: Box<[u8]>,
    |        ++++    +
 
-error[E0080]: the type `S` has an unknown layout
-  --> $DIR/base-layout-is-sized-ice-123078.rs:10:1
-   |
-LL | const C: S = unsafe { std::mem::transmute(()) };
-   | ^^^^^^^^^^ evaluation of `C` failed here
-
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/base-layout-is-sized-ice-123078.rs:10:23
    |
@@ -31,7 +25,7 @@ LL | const C: S = unsafe { std::mem::transmute(()) };
    = note: source type: `()` (0 bits)
    = note: target type: `S` (the type `S` has an unknown layout)
 
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 
-Some errors have detailed explanations: E0080, E0277, E0512.
-For more information about an error, try `rustc --explain E0080`.
+Some errors have detailed explanations: E0277, E0512.
+For more information about an error, try `rustc --explain E0277`.

tests/ui/transmute/transmute-in-async-fn.rs

@@ -0,0 +1,12 @@
+//@compile-flags: -Zmir-enable-passes=+DataflowConstProp --crate-type lib
+//@ edition:2021
+pub async fn a() -> u32 {
+    unsafe { std::mem::transmute(1u64) }
+    //~^error: cannot transmute between types of different sizes, or dependently-sized types
+}
+
+pub async fn b() -> u32 {
+    let closure = || unsafe { std::mem::transmute(1u64) };
+    //~^ ERROR: cannot transmute between types of different sizes, or dependently-sized types [E0512]
+    closure()
+}