Merge commit '0969bc6dde001e01e7e1f58c8ccd7750f8a49ae1' into sync_cg_clif-2021-03-29

2026-04-27 18:57:42 +03:00 · 2021-03-29 10:45:09 +02:00
parent cc4103089f 0969bc6dde
commit 5444b46234
49 changed files with 880 additions and 495 deletions
@@ -1,44 +0,0 @@
-name: Bootstrap rustc using cg_clif
-
-on:
-  - push
-
-jobs:
-  bootstrap_rustc:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Cache cargo installed crates
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v2
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Prepare dependencies
-      run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./prepare.sh
-
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        ./scripts/test_bootstrap.sh
@@ -7,11 +7,18 @@ on:
 jobs:
  build:
    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60

    strategy:
      fail-fast: false
      matrix:
-        os: [ubuntu-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu

    steps:
    - uses: actions/checkout@v2
@@ -36,6 +43,12 @@ jobs:
        path: target
        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}

+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+        rustup target add x86_64-pc-windows-gnu
+
    - name: Prepare dependencies
      run: |
        git config --global user.email "user@example.com"
@@ -43,6 +56,8 @@ jobs:
        ./prepare.sh

    - name: Test
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
      run: |
        # Enable backtraces for easier debugging
        export RUST_BACKTRACE=1
@@ -51,12 +66,16 @@ jobs:
        export COMPILE_RUNS=2
        export RUN_RUNS=2

+        # Enable extra checks
+        export CG_CLIF_ENABLE_VERIFIER=1
+
        ./test.sh

    - name: Package prebuilt cg_clif
      run: tar cvfJ cg_clif.tar.xz build

    - name: Upload prebuilt cg_clif
+      if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
      uses: actions/upload-artifact@v2
      with:
        name: cg_clif-${{ runner.os }}
@@ -0,0 +1,82 @@
+name: Various rustc tests
+
+on:
+  - push
+
+jobs:
+  bootstrap_rustc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_bootstrap.sh
+  rustc_test_suite:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_rustc_tests.sh
@@ -2,7 +2,7 @@
    // source for rustc_* is not included in the rust-src component; disable the errors about this
    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
    "rust-analyzer.assist.importMergeBehavior": "last",
-    "rust-analyzer.cargo.loadOutDirsFromCheck": true,
+    "rust-analyzer.cargo.runBuildScripts": true,
    "rust-analyzer.linkedProjects": [
        "./Cargo.toml",
        //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml",
@@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "cranelift-bforest"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "cranelift-entity",
 ]

 [[package]]
 name = "cranelift-codegen"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "byteorder",
 "cranelift-bforest",
@@ -65,8 +65,8 @@ dependencies = [

 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "cranelift-codegen-shared",
 "cranelift-entity",
@@ -74,18 +74,18 @@ dependencies = [

 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"

 [[package]]
 name = "cranelift-entity"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"

 [[package]]
 name = "cranelift-frontend"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "cranelift-codegen",
 "log",
@@ -95,8 +95,8 @@ dependencies = [

 [[package]]
 name = "cranelift-jit"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "anyhow",
 "cranelift-codegen",
@@ -113,8 +113,8 @@ dependencies = [

 [[package]]
 name = "cranelift-module"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "anyhow",
 "cranelift-codegen",
@@ -125,8 +125,8 @@ dependencies = [

 [[package]]
 name = "cranelift-native"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "cranelift-codegen",
 "target-lexicon",
@@ -134,8 +134,8 @@ dependencies = [

 [[package]]
 name = "cranelift-object"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
 "anyhow",
 "cranelift-codegen",
@@ -240,6 +240,15 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "memmap2"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.23.0"
@@ -310,6 +319,7 @@ dependencies = [
 "gimli",
 "indexmap",
 "libloading",
+ "memmap2",
 "object",
 "smallvec",
 "target-lexicon",
@@ -16,12 +16,13 @@ cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
 libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
+memmap2 = "0.2.1"

 # Uncomment to use local checkout of cranelift
 #[patch."https://github.com/bytecodealliance/wasmtime/"]
@@ -75,3 +76,6 @@ debug = false
 [profile.release.package.syn]
 opt-level = 0
 debug = false
+
+[package.metadata.rust-analyzer]
+rustc_private = true
@@ -34,70 +34,19 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo bui

 Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).

-### Cargo
-
 In the directory with your project (where you can do the usual `cargo build`), run:

 ```bash
-$ $cg_clif_dir/build/cargo.sh run
+$ $cg_clif_dir/build/cargo.sh build
 ```

-This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.

-### Rustc
-
-> You should prefer using the Cargo method.
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
-```
-
-### Jit mode
-
-In jit mode cg_clif will immediately execute your code without creating an executable file.
-
-> This requires all dependencies to be available as dynamic library.
-> The jit mode will probably need cargo integration to make this possible.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh jit
-```
-
-or
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
-```
-
-There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
-first called. It currently does not work with multi-threaded programs. When a not yet compiled
-function is called from another thread than the main thread, you will get an ICE.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh lazy-jit
-```
-
-### Shell
-
-These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
-
-```bash
-function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
-}
-
-function jit() {
-    jit_naked "fn main() { $@ }"
-}
-
-function jit_calc() {
-    jit 'println!("0x{:x}", ' $@ ');';
-}
-```
+For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).

 ## Env vars

-[see env_vars.md](docs/env_vars.md)
+See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift.

 ## Not yet supported

@@ -106,3 +55,20 @@ function jit_calc() {
      `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You
      have to specify specific registers instead.
 * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
+
+## License
+
+Licensed under either of
+
+  * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or
+    http://www.apache.org/licenses/LICENSE-2.0)
+  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+    http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you shall be dual licensed as above, without any
+additional terms or conditions.
@@ -55,6 +55,7 @@ ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
 ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"

 mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
+mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/"
 if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
    cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 fi
@@ -64,12 +65,18 @@ case "$build_sysroot" in
        ;;
    "llvm")
        cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/"
+        fi
        ;;
    "clif")
        echo "[BUILD] sysroot"
        dir=$(pwd)
        cd "$target_dir"
        time "$dir/build_sysroot/build_sysroot.sh"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh"
+        fi
        cp lib/rustlib/*/lib/libstd-* lib/
        ;;
    *)
@@ -16,9 +16,9 @@ dependencies = [

 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 dependencies = [
 "compiler_builtins",
 "rustc-std-workspace-core",
@@ -110,9 +110,9 @@ dependencies = [

 [[package]]
 name = "hashbrown"
-version = "0.9.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
 "compiler_builtins",
 "rustc-std-workspace-alloc",
@@ -132,18 +132,18 @@ dependencies = [

 [[package]]
 name = "libc"
-version = "0.2.86"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
+checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
 dependencies = [
 "rustc-std-workspace-core",
 ]

 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
 "adler",
 "autocfg",
@@ -28,7 +28,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
 if [[ "$1" != "--debug" ]]; then
    sysroot_channel='release'
    # FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release
+    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release
 else
    sysroot_channel='debug'
    cargo build --target "$TARGET_TRIPLE"
@@ -8,5 +8,8 @@
    to make it possible to use incremental mode for all analyses performed by rustc without caching
    object files when their content should have been changed by a change to cg_clif.</dd>
    <dt>CG_CLIF_DISPLAY_CG_TIME</dt>
-    <dd>If "1", display the time it took to perform codegen for a crate</dd>
+    <dd>If "1", display the time it took to perform codegen for a crate.</dd>
+    <dt>CG_CLIF_ENABLE_VERIFIER</dt>
+    <dd>Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once
+    before passing the clif ir to Cranelift for compilation.</dt>
 </dl>
@@ -0,0 +1,66 @@
+# Usage
+
+rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
+
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
+
+## Cargo
+
+In the directory with your project (where you can do the usual `cargo build`), run:
+
+```bash
+$ $cg_clif_dir/build/cargo.sh build
+```
+
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+
+## Rustc
+
+> You should prefer using the Cargo method.
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
+```
+
+## Jit mode
+
+In jit mode cg_clif will immediately execute your code without creating an executable file.
+
+> This requires all dependencies to be available as dynamic library.
+> The jit mode will probably need cargo integration to make this possible.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh jit
+```
+
+or
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
+```
+
+## Shell
+
+These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
+
+```bash
+function jit_naked() {
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
+}
+
+function jit() {
+    jit_naked "fn main() { $@ }"
+}
+
+function jit_calc() {
+    jit 'println!("0x{:x}", ' $@ ');';
+}
+```
@@ -621,6 +621,7 @@ struct PanicLocation {
 }

 #[no_mangle]
+#[cfg(not(windows))]
 pub fn get_tls() -> u8 {
    #[thread_local]
    static A: u8 = 42;
@@ -1,7 +1,4 @@
-#![feature(
-    no_core, start, lang_items, box_syntax, never_type, linkage,
-    extern_types, thread_local
-)]
+#![feature(no_core, lang_items, box_syntax, never_type, linkage, extern_types, thread_local)]
 #![no_core]
 #![allow(dead_code, non_camel_case_types)]

@@ -239,7 +236,7 @@ unsafe fn uninitialized<T>() -> T {

    assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);

-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
    {
        extern {
            #[linkage = "extern_weak"]
@@ -292,7 +289,7 @@ struct ExternTypeWrapper {

    from_decimal_string();

-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
    test_tls();

    #[cfg(all(not(jit), target_os = "linux"))]
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -e

-rustup component add rust-src rustc-dev llvm-tools-preview
 ./build_sysroot/prepare_sysroot_src.sh
 cargo install hyperfine || echo "Skipping hyperfine install"

@@ -1 +1,3 @@
-nightly-2021-03-05
+[toolchain]
+channel = "nightly-2021-03-29"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
@@ -4,7 +4,7 @@ dir=$(dirname "$0")
 source "$dir/config.sh"

 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat "$dir/rust-toolchain")
+TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")

 cmd=$1
 shift || true
@@ -2,15 +2,7 @@

 set -e

-unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
-   dylib_ext='so'
-elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
-else
-   echo "Unsupported os"
-   exit 1
-fi
+dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)

 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
@@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 export RUSTC=$dir"/bin/cg_clif"

 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
+'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir

 # FIXME fix `#[linkage = "extern_weak"]` without this
-if [[ "$unamestr" == 'Darwin' ]]; then
+if [[ "$(uname)" == 'Darwin' ]]; then
   export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi

@@ -8,7 +8,7 @@ case $1 in

        echo "=> Installing new nightly"
        rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
-        echo "nightly-${TOOLCHAIN}" > rust-toolchain
+        sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
        rustup component add rustfmt || true

        echo "=> Uninstalling all old nighlies"
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+
+./build.sh
+source build/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 5bd1147cad5..10d68a2ff14 100644
+--- a/Cargo.toml
+++ b/Cargo.toml
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+
+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
+
+ [patch."https://github.com/rust-lang/rust-clippy"]
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
+index 23e689fcae7..5f077b765b6 100644
+--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
+@@ -32,7 +32,6 @@ tempfile = "3.0.5"
+
+ [dependencies.parking_lot]
+ version = "0.11"
+-features = ["nightly"]
+
+ [target.'cfg(windows)'.dependencies]
+ winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
+++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+EOF
+
+cat > config.toml <<EOF
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+EOF
+popd
@@ -3,70 +3,10 @@ set -e

 cd "$(dirname "$0")/../"

-./build.sh
-source build/config.sh
+source ./scripts/setup_rust_fork.sh

 echo "[TEST] Bootstrap of rustc"
-git clone https://github.com/rust-lang/rust.git || true
 pushd rust
-git fetch
-git checkout -- .
-git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
-
-git apply - <<EOF
-diff --git a/Cargo.toml b/Cargo.toml
-index 5bd1147cad5..10d68a2ff14 100644
--- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- 
-+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
-+
- [patch."https://github.com/rust-lang/rust-clippy"]
- clippy_lints = { path = "src/tools/clippy/clippy_lints" }
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
--- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
-features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
-diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
-index d95b5b7f17f..00b6f0e3635 100644
--- a/library/alloc/Cargo.toml
-+++ b/library/alloc/Cargo.toml
-@@ -8,7 +8,7 @@ edition = "2018"
- 
- [dependencies]
- core = { path = "../core" }
-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
- 
- [dev-dependencies]
- rand = "0.7"
-EOF
-
-cat > config.toml <<EOF
-[llvm]
-ninja = false
-
-[build]
-rustc = "$(pwd)/../build/bin/cg_clif"
-cargo = "$(rustup which cargo)"
-full-bootstrap = true
-local-rebuild = true
-
-[rust]
-codegen-backends = ["cranelift"]
-EOF
-
 rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+cargo install ripgrep
+
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto" src/test/ui); do
+  rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+  rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
+# these all depend on unwinding support
+rm src/test/ui/backtrace.rs
+rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
+rm src/test/ui/array-slice-vec/slice-panic-*.rs
+rm src/test/ui/array-slice-vec/nested-vec-3.rs
+rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
+rm src/test/ui/issues/issue-26655.rs
+rm src/test/ui/issues/issue-29485.rs
+rm src/test/ui/issues/issue-30018-panic.rs
+rm src/test/ui/multi-panic.rs
+rm src/test/ui/sepcomp/sepcomp-unwind.rs
+rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
+rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/threads-sendsync/task-stderr.rs
+rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
+rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
+
+rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
+rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
+rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
+rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
+rm src/test/ui/generator/size-moved-locals.rs # same
+rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
+rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+
+rm src/test/incremental/hashes/inline_asm.rs # inline asm
+rm src/test/incremental/issue-72386.rs # same
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
+rm src/test/incremental/issue-49482.rs # same
+rm src/test/incremental/issue-54059.rs # same
+rm src/test/incremental/lto.rs # requires lto
+
+rm src/test/pretty/asm.rs # inline asm
+rm src/test/pretty/raw-str-nonexpr.rs # same
+
+rm -r src/test/run-pass-valgrind/unsized-locals
+
+rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
+rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
+rm src/test/ui/cfg/cfg-panic.rs
+rm src/test/ui/default-alloc-error-hook.rs
+rm -r src/test/ui/hygiene/
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
+rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
+rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+popd
@@ -71,14 +71,20 @@ function base_sysroot_tests() {
    echo "[AOT] mod_bench"
    $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
    $RUN_WRAPPER ./target/out/mod_bench
-
-    pushd rand
-    rm -r ./target || true
-    ../build/cargo.sh test --workspace
-    popd
 }

 function extended_sysroot_tests() {
+    pushd rand
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-random/rand"
+        ../build/cargo.sh test --workspace
+    else
+        echo "[AOT] rust-random/rand"
+        ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
+    fi
+    popd
+
    pushd simple-raytracer
    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
        echo "[BENCH COMPILE] ebobby/simple-raytracer"
@@ -92,27 +98,40 @@ function extended_sysroot_tests() {
    else
        echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
        echo "[COMPILE] ebobby/simple-raytracer"
-        ../cargo.sh build
+        ../build/cargo.sh build --target $TARGET_TRIPLE
        echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
    fi
    popd

    pushd build_sysroot/sysroot_src/library/core/tests
    echo "[TEST] libcore"
-    rm -r ./target || true
-    ../../../../../build/cargo.sh test
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        ../../../../../build/cargo.sh test
+    else
+        ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
+    fi
    popd

    pushd regex
    echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo.sh clean
+    cargo clean
    # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo.sh build --example shootout-regex-dna
-    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
-    diff -u res.txt examples/regexdna-output.txt
+    ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        cat examples/regexdna-input.txt \
+            | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | grep -v "Spawned thread" > res.txt
+        diff -u res.txt examples/regexdna-output.txt
+    fi

-    echo "[TEST] rust-lang/regex tests"
-    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-lang/regex tests"
+        ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    else
+        echo "[AOT] rust-lang/regex tests"
+        ../build/cargo.sh build --tests --target $TARGET_TRIPLE
+    fi
    popd
 }

@@ -11,9 +11,11 @@
 use crate::prelude::*;

 pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(
-        "kind  loc.idx   param    pass mode                            ty".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(
+            "kind  loc.idx   param    pass mode                            ty".to_string(),
+        );
+    }
 }

 pub(super) fn add_arg_comment<'tcx>(
@@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
    arg_abi_mode: PassMode,
    arg_layout: TyAndLayout<'tcx>,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
+
    let local = if let Some(local) = local {
        Cow::Owned(format!("{:?}", local))
    } else {
@@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>(
 }

 pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(String::new());
-    fx.add_global_comment(
-        "kind  local ty                              size align (abi,pref)".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(String::new());
+        fx.add_global_comment(
+            "kind  local ty                              size align (abi,pref)".to_string(),
+        );
+    }
 }

 pub(super) fn add_local_place_comments<'tcx>(
@@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>(
    place: CPlace<'tcx>,
    local: Local,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
    let TyAndLayout { ty, layout } = place.layout();
    let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
        layout;
@@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>(
            } else {
                Cow::Borrowed("")
            };
-            match ptr.base_and_offset() {
+            match ptr.debug_base_and_offset() {
                (crate::pointer::PointerBase::Addr(addr), offset) => {
                    ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
                }
@@ -1,6 +1,5 @@
 //! Handling of everything related to the calling convention. Also fills `fx.local_map`.

-#[cfg(debug_assertions)]
 mod comments;
 mod pass_mode;
 mod returning;
@@ -75,8 +74,9 @@ pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
        let func_id = import_function(self.tcx, self.cx.module, inst);
        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);

-        #[cfg(debug_assertions)]
-        self.add_comment(func_ref, format!("{:?}", inst));
+        if self.clif_comments.enabled() {
+            self.add_comment(func_ref, format!("{:?}", inst));
+        }

        func_ref
    }
@@ -92,8 +92,7 @@ pub(crate) fn lib_call(
        let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
        let call_inst = self.bcx.ins().call(func_ref, args);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
            self.add_comment(call_inst, format!("easy_call {}", name));
        }
        let results = self.bcx.inst_results(call_inst);
@@ -149,7 +148,6 @@ fn make_local_place<'tcx>(
        CPlace::new_stack_slot(fx, layout)
    };

-    #[cfg(debug_assertions)]
    self::comments::add_local_place_comments(fx, place, local);

    place
@@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_

    let ssa_analyzed = crate::analyze::analyze(fx);

-    #[cfg(debug_assertions)]
    self::comments::add_args_header_comment(fx);

    let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
@@ -228,7 +225,6 @@ enum ArgKind<'tcx> {
    fx.fn_abi = Some(fn_abi);
    assert!(block_params_iter.next().is_none(), "arg_value left behind");

-    #[cfg(debug_assertions)]
    self::comments::add_locals_header_comment(fx);

    for (local, arg_kind, ty) in func_params {
@@ -256,7 +252,6 @@ enum ArgKind<'tcx> {
                        CPlace::for_ptr(addr, val.layout())
                    };

-                    #[cfg(debug_assertions)]
                    self::comments::add_local_place_comments(fx, place, local);

                    assert_eq!(fx.local_map.push(place), local);
@@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
    let (func_ref, first_arg) = match instance {
        // Trait object call
        Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                let nop_inst = fx.bcx.ins().nop();
                fx.add_comment(
                    nop_inst,
@@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(

        // Indirect call
        None => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                let nop_inst = fx.bcx.ins().nop();
                fx.add_comment(nop_inst, "indirect call");
            }
@@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
    // FIXME find a cleaner way to support varargs
    if fn_sig.c_variadic {
        if !matches!(fn_sig.abi, Abi::C { .. }) {
-            fx.tcx.sess.span_fatal(
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
        }
        let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
        let abi_params = call_args
@@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
    });
    let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
    let mut offset = 0;
-    let mut block_params_iter = block_params.into_iter().copied();
+    let mut block_params_iter = block_params.iter().copied();
    for param in abi_params {
        let val = ptr.offset_i64(fx, offset).store(
            fx,
@@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
+    local: Option<mir::Local>,
+    local_field: Option<usize>,
    arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
    block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
@@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>(
        })
        .collect::<SmallVec<[_; 2]>>();

-    #[cfg(debug_assertions)]
    crate::abi::comments::add_arg_comment(
        fx,
        "arg",
@@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>(
        }
    };

-    #[cfg(not(debug_assertions))]
-    let _ = ret_param;
-
-    #[cfg(debug_assertions)]
    crate::abi::comments::add_arg_comment(
        fx,
        "ret",
@@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>(
                let results = fx
                    .bcx
                    .inst_results(call_inst)
-                    .into_iter()
+                    .iter()
                    .copied()
                    .collect::<SmallVec<[Value; 2]>>();
                let result =
@@ -3,6 +3,7 @@

 use crate::prelude::*;

+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_span::symbol::sym;

@@ -92,7 +93,7 @@ fn codegen_inner(
            bcx.finalize();
        }
        module
-            .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
            .unwrap();
        unwind_context.add_function(func_id, &ctx, module.isa());
    }
@@ -132,7 +133,7 @@ fn codegen_inner(
        bcx.finalize();
    }
    module
-        .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
        .unwrap();
    unwind_context.add_function(func_id, &ctx, module.isa());
 }
@@ -1,5 +1,6 @@
 //! Codegen of a single function

+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
@@ -7,11 +8,7 @@

 use crate::prelude::*;

-pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'_, 'tcx>,
-    instance: Instance<'tcx>,
-    linkage: Linkage,
-) {
+pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
    let tcx = cx.tcx;

    let _inst_guard =
@@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
    // Declare function
    let name = tcx.symbol_name(instance).name.to_string();
    let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();

    cx.cached_context.clear();

@@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
    let module = &mut cx.module;
    tcx.sess.time("define function", || {
        module
-            .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
            .unwrap()
    });

@@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
            codegen_stmt(fx, block, stmt);
        }

-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
            let mut terminator_head = "\n".to_string();
            bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
            let inst = fx.bcx.func.layout.last_inst(block).unwrap();
@@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>(

    fx.set_debug_loc(stmt.source_info);

-    #[cfg(false_debug_assertions)]
+    #[cfg(disabled)]
    match &stmt.kind {
        StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
        _ => {
-            let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
-            fx.add_comment(inst, format!("{:?}", stmt));
+            if fx.clif_comments.enabled() {
+                let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+                fx.add_comment(inst, format!("{:?}", stmt));
+            }
        }
    }

@@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>(
                    let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
                    lval.write_cvalue(fx, val);
                }
-                Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);

                    let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
                    lval.write_cvalue(fx, res);
                }
-                Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);

                    let res = if !fx.tcx.sess.overflow_checks() {
                        let val =
@@ -659,7 +657,9 @@ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
                        .val
                        .try_to_bits(fx.tcx.data_layout.pointer_size)
                        .unwrap();
-                    if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+                    if operand.layout().size.bytes() == 0 {
+                        // Do nothing for ZST's
+                    } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
                        let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
                        // FIXME use emit_small_memset where possible
                        let addr = lval.to_ptr().get_addr(fx);
@@ -832,25 +832,18 @@ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
            }
        }
        StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
-          src,
-          dst,
-          count,
-        }) => {
-            let dst = codegen_operand(fx, dst);
+        StatementKind::CopyNonOverlapping(inner) => {
+            let dst = codegen_operand(fx, &inner.dst);
            let pointee = dst
-              .layout()
-              .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
-              .expect("Expected pointer");
+                .layout()
+                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                .expect("Expected pointer");
            let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, src).load_scalar(fx);
-            let count = codegen_operand(fx, count).load_scalar(fx);
+            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
            let elem_size: u64 = pointee.size.bytes();
-            let bytes = if elem_size != 1 {
-               fx.bcx.ins().imul_imm(count, elem_size as i64)
-            } else {
-               count
-            };
+            let bytes =
+                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
            fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
        }
    }
@@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>(
        BinOp::Add | BinOp::Sub if !checked => None,
        BinOp::Mul if !checked => {
            let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            if fx.tcx.sess.target.is_like_windows {
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args =
+                    [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                fx.lib_call(
+                    "__multi3",
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    vec![],
+                    &args,
+                );
+                Some(ret_place.to_cvalue(fx))
+            } else {
+                Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            }
        }
        BinOp::Add | BinOp::Sub | BinOp::Mul => {
            assert!(checked);
            let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
            let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let param_types = vec![
-                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
-                AbiParam::new(types::I128),
-                AbiParam::new(types::I128),
-            ];
-            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
+            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
+                )
+            } else {
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(types::I128),
+                        AbiParam::new(types::I128),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
+                )
+            };
            let name = match (bin_op, is_signed) {
                (BinOp::Add, false) => "__rust_u128_addo",
                (BinOp::Add, true) => "__rust_i128_addo",
@@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
            Some(out_place.to_cvalue(fx))
        }
        BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Div => {
+        BinOp::Div | BinOp::Rem => {
            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
+            let name = match (bin_op, is_signed) {
+                (BinOp::Div, false) => "__udivti3",
+                (BinOp::Div, true) => "__divti3",
+                (BinOp::Rem, false) => "__umodti3",
+                (BinOp::Rem, true) => "__modti3",
+                _ => unreachable!(),
+            };
+            if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let ret = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
+                    vec![AbiParam::new(types::I64X2)],
+                    &args,
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
+                Some(ret_place.to_cvalue(fx))
            } else {
-                Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
-            }
-        }
-        BinOp::Rem => {
-            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
-            } else {
-                Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
+                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
            }
        }
        BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {
@@ -361,8 +361,7 @@ pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value {
        let _ = self.cx.module.define_data(msg_id, &data_ctx);

        let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
            self.add_comment(local_msg_id, msg);
        }
        self.bcx.ins().global_value(self.pointer_type, local_msg_id)
@@ -0,0 +1,41 @@
+macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
+    #[cfg(feature = "jit")]
+    #[allow(improper_ctypes)]
+    extern "C" {
+        $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+    }
+
+    #[cfg(feature = "jit")]
+    pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
+        for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
+            builder.symbol(name, val);
+        }
+    }
+}
+
+builtin_functions! {
+    register_functions_for_jit;
+
+    // integers
+    fn __multi3(a: i128, b: i128) -> i128;
+    fn __udivti3(n: u128, d: u128) -> u128;
+    fn __divti3(n: i128, d: i128) -> i128;
+    fn __umodti3(n: u128, d: u128) -> u128;
+    fn __modti3(n: i128, d: i128) -> i128;
+    fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+    // floats
+    fn __floattisf(i: i128) -> f32;
+    fn __floattidf(i: i128) -> f64;
+    fn __floatuntisf(i: u128) -> f32;
+    fn __floatuntidf(i: u128) -> f64;
+    fn __fixsfti(f: f32) -> i128;
+    fn __fixdfti(f: f64) -> i128;
+    fn __fixunssfti(f: f32) -> u128;
+    fn __fixunsdfti(f: f64) -> u128;
+}
@@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 ) -> CValue<'tcx> {
    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    }
    let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
    CValue::by_val(tls_ptr, layout)
 }
@@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>(
 ) -> CPlace<'tcx> {
    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", def_id));
+    }
    let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
    assert!(!layout.is_unsized(), "unsized statics aren't supported");
    assert!(
@@ -122,7 +124,9 @@ pub(crate) fn codegen_constant<'tcx>(
    };
    let const_val = match const_.val {
        ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => {
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
            assert!(substs.is_empty());
            assert!(promoted.is_none());

@@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                                data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
                            let local_data_id =
                                fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            }
                            fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                        }
                        Some(GlobalAlloc::Function(instance)) => {
@@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                            let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
                            let local_data_id =
                                fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            }
                            fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                        }
                        None => bug!("missing allocation {:?}", ptr.alloc_id),
@@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>(
    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);

    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    }
    let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
    crate::pointer::Pointer::new(global_ptr)
 }
@@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
    #[cfg(unix)]
    {
        use std::os::unix::ffi::OsStrExt;
-        return path.as_bytes();
+        path.as_bytes()
    }
    #[cfg(not(unix))]
    {
-        return path.to_str().unwrap().as_bytes();
+        path.to_str().unwrap().as_bytes()
    }
 }

@@ -119,11 +119,10 @@ fn module_codegen(
        tcx.sess.opts.debuginfo != DebugInfo::None,
    );
    super::predefine_mono_items(&mut cx, &mono_items);
-    for (mono_item, (linkage, visibility)) in mono_items {
-        let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+    for (mono_item, _) in mono_items {
        match mono_item {
            MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
            }
            MonoItem::Static(def_id) => {
                crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -163,6 +162,21 @@ pub(super) fn run_aot(
    metadata: EncodedMetadata,
    need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    use rustc_span::symbol::sym;
+
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != sym::windows && subsystem != sym::console {
+            tcx.sess.fatal(&format!(
+                "invalid windows subsystem `{}`, only \
+                                    `windows` and `console` are allowed",
+                subsystem
+            ));
+        }
+        subsystem.to_string()
+    });
+
    let mut work_products = FxHashMap::default();

    let cgus = if tcx.sess.opts.output_types.should_codegen() {
@@ -280,7 +294,7 @@ pub(super) fn run_aot(
            allocator_module,
            metadata_module,
            metadata,
-            windows_subsystem: None, // Windows is not yet supported
+            windows_subsystem,
            linker_info: LinkerInfo::new(tcx),
            crate_info: CrateInfo::new(tcx),
        },
@@ -5,8 +5,10 @@
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};

+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::config::EntryFnType;

 use cranelift_jit::{JITBuilder, JITModule};

@@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
    let mut jit_builder =
        JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
    jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
    jit_builder.symbols(imported_symbols);
    let mut jit_module = JITModule::new(jit_builder);
    assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());

-    let sig = Signature {
-        params: vec![
-            AbiParam::new(jit_module.target_config().pointer_type()),
-            AbiParam::new(jit_module.target_config().pointer_type()),
-        ],
-        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-    };
-    let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
-
    let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
    let mono_items = cgus
        .iter()
@@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {

    super::time(tcx, "codegen mono items", || {
        super::predefine_mono_items(&mut cx, &mono_items);
-        for (mono_item, (linkage, visibility)) in mono_items {
-            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+        for (mono_item, _) in mono_items {
            match mono_item {
                MonoItem::Fn(inst) => match backend_config.codegen_mode {
                    CodegenMode::Aot => unreachable!(),
                    CodegenMode::Jit => {
-                        cx.tcx
-                            .sess
-                            .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
                    }
                    CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                },
@@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
        tcx.sess.fatal("Inline asm is not supported in JIT mode");
    }

-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
    crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);

    tcx.sess.abort_if_errors();

    jit_module.finalize_definitions();
-
    let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };

-    let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
    println!(
        "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
    );

-    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-        unsafe { ::std::mem::transmute(finalized_main) };
-
    let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
    let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
        .chain(args.split(' '))
@@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
    BACKEND_CONFIG.with(|tls_backend_config| {
        assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
    });
-    CURRENT_MODULE
-        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));

-    let ret = f(args.len() as c_int, argv.as_ptr());
+    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);

-    std::process::exit(ret);
+    match entry_ty {
+        EntryFnType::Main => {
+            // FIXME set program arguments somehow
+
+            let main_sig = Signature {
+                params: vec![],
+                returns: vec![],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let main_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
+                .unwrap();
+            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
+            f();
+            std::process::exit(0);
+        }
+        EntryFnType::Start => {
+            let start_sig = Signature {
+                params: vec![
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                ],
+                returns: vec![AbiParam::new(
+                    jit_module.target_config().pointer_type(), /*isize*/
+                )],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let start_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
+                .unwrap();
+            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+                unsafe { ::std::mem::transmute(finalized_start) };
+            let ret = f(args.len() as c_int, argv.as_ptr());
+            std::process::exit(ret);
+        }
+    }
 }

 #[no_mangle]
@@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
            jit_module.prepare_for_function_redefine(func_id).unwrap();

            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
-            tcx.sess
-                .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));

            let (global_asm, _debug_context, unwind_context) = cx.finalize();
            assert!(global_asm.is_empty());
@@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
    imported_symbols
 }

-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
    let tcx = cx.tcx;

    let pointer_type = cx.module.target_config().pointer_type();
@@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t
        .define_function(
            func_id,
            &mut Context::for_function(trampoline),
-            &mut cranelift_codegen::binemit::NullTrapSink {},
+            &mut NullTrapSink {},
+            &mut NullStackMapSink {},
        )
        .unwrap();
 }
@@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>(
    mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
    cx.tcx.sess.time("predefine functions", || {
+        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
        for &(mono_item, (linkage, visibility)) in mono_items {
            match mono_item {
                MonoItem::Fn(instance) => {
                    let name = cx.tcx.symbol_name(instance).name.to_string();
                    let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
                    let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                    let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+                    let linkage = crate::linkage::get_clif_linkage(
+                        mono_item,
+                        linkage,
+                        visibility,
+                        is_compiler_builtins,
+                    );
                    cx.module.declare_function(&name, linkage, &sig).unwrap();
                }
                MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
@@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
    if template.is_empty() {
        // Black box
        return;
+    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        let true_ = fx.bcx.ins().iconst(types::I32, 1);
+        fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+        return;
    }

    let mut slot_size = Size::from_bytes(0);
@@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>(
        offset: None,
        size: u32::try_from(slot_size.bytes()).unwrap(),
    });
-    #[cfg(debug_assertions)]
-    fx.add_comment(stack_slot, "inline asm scratch slot");
+    if fx.clif_comments.enabled() {
+        fx.add_comment(stack_slot, "inline asm scratch slot");
+    }

    let inline_asm_func = fx
        .cx
@@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>(
        )
        .unwrap();
    let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(inline_asm_func, asm_name);
+    if fx.clif_comments.enabled() {
+        fx.add_comment(inline_asm_func, asm_name);
+    }

    for (_reg, offset, value) in inputs {
        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
@@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                let idx_bytes = match idx_const {
                    ConstValue::ByRef { alloc, offset } => {
                        let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
+                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                        alloc.get_bytes(fx, ptr, size).unwrap()
                    }
                    _ => unreachable!("{:?}", idx_const),
@@ -1,13 +1,4 @@
-#![feature(
-    rustc_private,
-    decl_macro,
-    type_alias_impl_trait,
-    associated_type_bounds,
-    never_type,
-    try_blocks,
-    box_patterns,
-    hash_drain_filter
-)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -57,6 +48,7 @@
 mod cast;
 mod codegen_i128;
 mod common;
+mod compiler_builtins;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend {

 impl CodegenBackend for CraneliftCodegenBackend {
    fn init(&self, sess: &Session) {
-        if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
-            sess.warn("LTO is not supported. You may get a linker error.");
+        use rustc_session::config::Lto;
+        match sess.lto() {
+            Lto::No | Lto::ThinLocal => {}
+            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
        }
    }

@@ -240,9 +234,9 @@ fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> {
        vec![]
    }

-    fn codegen_crate<'tcx>(
+    fn codegen_crate(
        &self,
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
        metadata: EncodedMetadata,
        need_metadata_module: bool,
    ) -> Box<dyn Any> {
@@ -252,9 +246,7 @@ fn codegen_crate<'tcx>(
            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                .unwrap_or_else(|err| tcx.sess.fatal(&err))
        };
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
-
-        res
+        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
    }

    fn join_codegen(
@@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
    let mut flags_builder = settings::builder();
    flags_builder.enable("is_pic").unwrap();
    flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    flags_builder
-        .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
-        .unwrap();
+    let enable_verifier =
+        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
+    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();

    let tls_model = match target_triple.binary_format {
        BinaryFormat::Elf => "elf_gd",
@@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {

    flags_builder.set("enable_simd", "true").unwrap();

+    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
    use rustc_session::config::OptLevel;
    match sess.opts.optimize {
        OptLevel::No => {
            flags_builder.set("opt_level", "none").unwrap();
        }
        OptLevel::Less | OptLevel::Default => {}
-        OptLevel::Aggressive => {
+        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
            flags_builder.set("opt_level", "speed_and_size").unwrap();
        }
-        OptLevel::Size | OptLevel::SizeMin => {
-            sess.warn("Optimizing for size is not supported. Just ignoring the request");
-        }
    }

    let flags = settings::Flags::new(flags_builder);
@@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
    mono_item: MonoItem<'_>,
    linkage: RLinkage,
    visibility: Visibility,
+    is_compiler_builtins: bool,
 ) -> Linkage {
    match (linkage, visibility) {
+        (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
        (RLinkage::External, Visibility::Default) => Linkage::Export,
        (RLinkage::Internal, Visibility::Default) => Linkage::Local,
        (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
@@ -1,3 +1,4 @@
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_session::config::EntryFnType;

@@ -100,12 +101,8 @@ fn create_entry_fn(
            bcx.seal_all_blocks();
            bcx.finalize();
        }
-        m.define_function(
-            cmain_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
+        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
+            .unwrap();
        unwind_context.add_function(cmain_func_id, &ctx, m.isa());
    }
 }
@@ -1,11 +1,11 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs

-use std::convert::TryFrom;
 use std::fs::File;
+use std::ops::Deref;
 use std::path::Path;

 use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@@ -17,38 +17,56 @@

 pub(crate) struct CraneliftMetadataLoader;

+struct StableMmap(memmap2::Mmap);
+
+impl Deref for StableMmap {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &*self.0
+    }
+}
+
+unsafe impl StableAddress for StableMmap {}
+
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
+        .map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
    fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
-        // Iterate over all entries in the archive:
-        while let Some(entry_result) = archive.next_entry() {
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
-            }
-        }
+        load_metadata_with(path, |data| {
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
+                .map_err(|e| format!("{:?}", e))?;

-        Err("couldn't find metadata entry".to_string())
+            for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
+            }
+
+            Err("couldn't find metadata entry".to_string())
+        })
    }

    fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
        use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
-        let buf = file
-            .section_by_name(".rustc")
-            .ok_or("no .rustc section")?
-            .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
-            .to_owned();
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
+
+        load_metadata_with(path, |data| {
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
+            file.section_by_name(".rustc")
+                .ok_or("no .rustc section")?
+                .data()
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
+        })
    }
 }

@@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
        BinOp::Shl => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
            fx.bcx.ins().ishl(lhs, actual_shift)
        }
        BinOp::Shr => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
            if signed {
                fx.bcx.ins().sshr(lhs, actual_shift)
            } else {
@@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                let lhs = in_lhs.load_scalar(fx);
                let rhs = in_rhs.load_scalar(fx);

-                return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
+                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
            }
            BinOp::Offset => {
                let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                let base_val = base.load_scalar(fx);
                let res = fx.bcx.ins().iadd(base_val, ptr_diff);
-                return CValue::by_val(res, base.layout());
+                CValue::by_val(res, base.layout())
            }
            _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
-        };
+        }
    } else {
        let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
        let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
@@ -181,7 +181,6 @@ fn for_context(ctx: &'a mut Context) -> Self {

 pub(super) fn optimize_function(
    ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))]
    clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
    combine_stack_addr_with_load_store(&mut ctx.func);
@@ -192,8 +191,7 @@ pub(super) fn optimize_function(

    remove_unused_stack_addr_and_stack_load(&mut opt_ctx);

-    #[cfg(debug_assertions)]
-    {
+    if clif_comments.enabled() {
        for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
            clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
        }
@@ -209,25 +207,27 @@ pub(super) fn optimize_function(
        for load in users.stack_load.clone().into_iter() {
            let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);

-            #[cfg(debug_assertions)]
-            for &store in &potential_stores {
-                clif_comments.add_comment(
-                    load,
-                    format!(
-                        "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &store in &potential_stores {
+                    clif_comments.add_comment(
+                        load,
+                        format!(
+                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
            }

            match *potential_stores {
                [] => {
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    if clif_comments.enabled() {
+                        clif_comments
+                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    }
                }
                [store]
                    if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@@ -237,9 +237,12 @@ pub(super) fn optimize_function(
                    // Only one store could have been the origin of the value.
                    let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];

-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, format!("Store to load forward {} -> {}", store, load));
+                    if clif_comments.enabled() {
+                        clif_comments.add_comment(
+                            load,
+                            format!("Store to load forward {} -> {}", store, load),
+                        );
+                    }

                    users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
                }
@@ -250,33 +253,35 @@ pub(super) fn optimize_function(
        for store in users.stack_store.clone().into_iter() {
            let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);

-            #[cfg(debug_assertions)]
-            for &load in &potential_loads {
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Potential load from store {} <- {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &load in &potential_loads {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Potential load from store {} <- {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
            }

            if potential_loads.is_empty() {
                // Never loaded; can safely remove all stores and the stack slot.
                // FIXME also remove stores when there is always a next store before a load.

-                #[cfg(debug_assertions)]
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Remove dead stack store {} of {}",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        stack_slot.0
-                    ),
-                );
+                if clif_comments.enabled() {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Remove dead stack store {} of {}",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            stack_slot.0
+                        ),
+                    );
+                }

                users.remove_dead_store(&mut opt_ctx.ctx.func, store);
            }
@@ -39,8 +39,7 @@ pub(crate) fn dangling(align: Align) -> Self {
        Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
    }

-    #[cfg(debug_assertions)]
-    pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
+    pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
        (self.base, self.offset)
    }

@@ -69,13 +69,15 @@

 #[derive(Debug)]
 pub(crate) struct CommentWriter {
+    enabled: bool,
    global_comments: Vec<String>,
    entity_comments: FxHashMap<AnyEntity, String>,
 }

 impl CommentWriter {
    pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
-        let global_comments = if cfg!(debug_assertions) {
+        let enabled = should_write_ir(tcx);
+        let global_comments = if enabled {
            vec![
                format!("symbol {}", tcx.symbol_name(instance).name),
                format!("instance {:?}", instance),
@@ -86,13 +88,17 @@ pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
            vec![]
        };

-        CommentWriter { global_comments, entity_comments: FxHashMap::default() }
+        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
    }
 }

-#[cfg(debug_assertions)]
 impl CommentWriter {
+    pub(crate) fn enabled(&self) -> bool {
+        self.enabled
+    }
+
    pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
+        debug_assert!(self.enabled);
        self.global_comments.push(comment.into());
    }

@@ -101,6 +107,8 @@ pub(crate) fn add_comment<S: Into<String> + AsRef<str>, E: Into<AnyEntity>>(
        entity: E,
        comment: S,
    ) {
+        debug_assert!(self.enabled);
+
        use std::collections::hash_map::Entry;
        match self.entity_comments.entry(entity.into()) {
            Entry::Occupied(mut occ) => {
@@ -179,7 +187,6 @@ fn write_instruction(
    }
 }

-#[cfg(debug_assertions)]
 impl FunctionCx<'_, '_, '_> {
    pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
        self.clif_comments.add_global_comment(comment);
@@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
    tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
 }

-pub(crate) fn write_ir_file<'tcx>(
-    tcx: TyCtxt<'tcx>,
+pub(crate) fn write_ir_file(
+    tcx: TyCtxt<'_>,
    name: &str,
    write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
@@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(

    let clif_file_name = clif_output_dir.join(name);

-    let res: std::io::Result<()> = try {
-        let mut file = std::fs::File::create(clif_file_name)?;
-        write(&mut file)?;
-    };
+    let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
    if let Err(err) = res {
        tcx.sess.warn(&format!("error writing ir file: {}", err));
    }
@@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
        )
        .unwrap();
    let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    {
+    if fx.clif_comments.enabled() {
        fx.add_comment(puts, "puts");
    }

@@ -2,7 +2,6 @@

 use crate::prelude::*;

-use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;

 fn codegen_field<'tcx>(
@@ -414,7 +413,7 @@ fn write_cvalue_maybe_transmute(
        self,
        fx: &mut FunctionCx<'_, '_, 'tcx>,
        from: CValue<'tcx>,
-        #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
+        method: &'static str,
    ) {
        fn transmute_value<'tcx>(
            fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -462,8 +461,7 @@ fn transmute_value<'tcx>(

        assert_eq!(self.layout().size, from.layout().size);

-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
            use cranelift_codegen::cursor::{Cursor, CursorPosition};
            let cur_block = match fx.bcx.cursor().position() {
                CursorPosition::After(block) => block,
@@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
            }
            // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
        }
+        (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+            if adt_def_a.did == adt_def_b.did =>
+        {
+            let mut types_a = substs_a.types();
+            let mut types_b = substs_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
        _ => {
            assert_eq!(
                from_ty, to_ty,