ICE for generators involving Never type

[RalfJung]

Building the following code ICEs with current master (not yet on nightly):

#![feature(never_type)]

use std::future::Future;

// See if we can run a basic `async fn`
pub async fn foo(x: &u32, y: u32) -> u32 {
    let y = &y;
    let z = 9;
    let z = &z;
    let y = async { *y + *z }.await;
    let a = 10;
    let a = &a;
    *x + y + *a
}

async fn add(x: u32, y: u32) -> u32 {
    let a = async { x + y };
    a.await
}

async fn build_aggregate(a: u32, b: u32, c: u32, d: u32) -> u32 {
    let x = (add(a, b).await, add(c, d).await);
    x.0 + x.1
}

enum Never {}
fn never() -> Never {
    panic!()
}

async fn includes_never(crash: bool, x: u32) -> u32 {
    let mut result = async { x * x }.await;
    if !crash {
        return result;
    }
    #[allow(unused)]
    let bad = never();
    result *= async { x + x }.await;
    drop(bad);
    result
}

async fn partial_init(x: u32) -> u32 {
    #[allow(unreachable_code)]
    let _x: (String, !) = (String::new(), return async { x + x }.await);
}

async fn read_exact(_from: &mut &[u8], _to: &mut [u8]) -> Option<()> {
    Some(())
}

async fn hello_world() {
    let data = [0u8; 1];
    let mut reader = &data[..];

    let mut marker = [0u8; 1];
    read_exact(&mut reader, &mut marker).await.unwrap();
}

fn run_fut<T>(fut: impl Future<Output = T>) -> T {
    use std::sync::Arc;
    use std::task::{Context, Poll, Wake, Waker};

    struct MyWaker;
    impl Wake for MyWaker {
        fn wake(self: Arc<Self>) {
            unimplemented!()
        }
    }

    let waker = Waker::from(Arc::new(MyWaker));
    let mut context = Context::from_waker(&waker);

    let mut pinned = Box::pin(fut);
    loop {
        match pinned.as_mut().poll(&mut context) {
            Poll::Pending => continue,
            Poll::Ready(v) => return v,
        }
    }
}

fn main() {
    let x = 5;
    assert_eq!(run_fut(foo(&x, 7)), 31);
    assert_eq!(run_fut(build_aggregate(1, 2, 3, 4)), 10);
    assert_eq!(run_fut(includes_never(false, 4)), 16);
    assert_eq!(run_fut(partial_init(4)), 8);
    run_fut(hello_world());
}

Output:

error: internal compiler error: compiler/rustc_mir_transform/src/generator.rs:755:13: Broken MIR: generator contains type Never in MIR, but typeck only knows about {ResumeTy, bool, u32, impl Future<Output = [async output]>, (), impl Future<Output = [async output]>} and [bool, u32]
  --> tests/run-pass/async-fn.rs:31:53
   |
31 |   async fn includes_never(crash: bool, x: u32) -> u32 {
   |  _____________________________________________________^
32 | |     let mut result = async { x * x }.await;
33 | |     if !crash {
34 | |         return result;
...  |
40 | |     result
41 | | }
   | |_^

thread 'rustc' panicked at 'Box<dyn Any>', /rustc/84e918971d643c6a33067d5125214ab800ce5307/compiler/rustc_errors/src/lib.rs:1115:9
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

note: the compiler unexpectedly panicked. this is a bug.

note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md

note: rustc 1.60.0-nightly (84e918971 2022-01-21) running on x86_64-unknown-linux-gnu

query stack during panic:
#0 [optimized_mir] optimizing MIR for `includes_never::{closure#0}`
#1 [layout_of] computing layout of `[static generator@tests/run-pass/async-fn.rs:31:53: 41:2]`
end of query stack

This is a regression, the same code worked fine yesterday. Likely cause: #91032

[eholk]

@rustbot claim