Stop using xargo for building the sysroot

It is currently broken. (see japaric/xargo#227)

This makes it easier to for example patch whole crates away.

.gitignore

@@ -4,3 +4,6 @@ target
 *.o
 perf.data
 perf.data.old
+/build_sysroot/sysroot
+/build_sysroot/sysroot_src
+/build_sysroot/Cargo.lock

.travis.yml

@@ -10,11 +10,7 @@ rust:
   - nightly
 
 script:
-  - rustup component add rust-src
-  - cargo install xargo || echo "Skipping xargo install"
-  - cargo install hyperfine || echo "Skipping hyperfine install"
-  - ./prepare_libcore.sh
-  - ./test.sh
+  - ./prepare.sh && ./test.sh
 
 env:
   - RUST_BACKTRACE=1

build_sysroot/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+authors = ["bjorn3 <[email protected]>"]
+name = "sysroot"
+version = "0.0.0"
+
+[dependencies]
+core = { path = "./sysroot_src/src/libcore" }
+compiler_builtins = "0.1"
+alloc = { path = "./sysroot_src/src/liballoc" }
+
+alloc_system = { path = "./alloc_system" }
+
+[patch.crates-io]
+rustc-std-workspace-core = { path = "./sysroot_src/src/tools/rustc-std-workspace-core" }
+compiler_builtins = { path = "./compiler_builtins" }

build_sysroot/alloc_system/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
+name = "alloc_system"
+version = "0.0.0"
+[lib]
+name = "alloc_system"
+path = "lib.rs"
+test = false
+doc = false
+[dependencies]
+core = { path = "../sysroot_src/src/libcore" }
+libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false }
+compiler_builtins = "0.1"

build_sysroot/build_sysroot.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+cd $(dirname "$0")
+
+# Cleanup for previous run
+cargo clean
+rm Cargo.lock 2>/dev/null || true
+rm -r sysroot 2>/dev/null || true
+
+# FIXME find a better way to get the target triple
+unamestr=`uname`
+if [[ "$unamestr" == 'Linux' ]]; then
+   TARGET_TRIPLE='x86_64-unknown-linux-gnu'
+elif [[ "$unamestr" == 'Darwin' ]]; then
+   TARGET_TRIPLE='x86_64-apple-darwin'
+else
+   echo "Unsupported os"
+   exit 1
+fi
+
+# Build libs
+export RUSTFLAGS="$RUSTFLAGS -Z force-unstable-if-unmarked --sysroot ../"
+if [[ "$1" == "--release" ]]; then
+    channel='release'
+    RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target $TARGET_TRIPLE --release
+else
+    channel='debug'
+    cargo build --target $TARGET_TRIPLE
+fi
+
+# Copy files to sysroot
+mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+cp target/$TARGET_TRIPLE/$channel/deps/*.rlib sysroot/lib/rustlib/$TARGET_TRIPLE/lib/

build_sysroot/compiler_builtins/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "compiler_builtins"
+# Make sure the `compiler_builtins` from crates.io doesn't take precedence over this
+# replacement by specifying a higher version than the one on crates.io.
+version = "0.1.100"
+authors = ["bjorn3 <[email protected]>"]
+edition = "2018"
+
+[lib]
+name = "compiler_builtins"
+path = "lib.rs"
+test = false
+doc = false
+
+[dependencies]
+core = { path = "../sysroot_src/src/libcore" }
+
+[features]
+rustc-dep-of-std = []
+c = []

build_sysroot/compiler_builtins/lib.rs

@@ -0,0 +1,3 @@
+#![feature(compiler_builtins)]
+#![compiler_builtins]
+#![no_std]

prepare_libcore.sh → build_sysroot/prepare_sysroot_src.sh

@@ -1,8 +1,9 @@
-#!/bin/bash --verbose
+#!/bin/bash
 set -e
+cd $(dirname "$0")
 
 SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/"
-DST_DIR="target/libcore"
+DST_DIR="sysroot_src"
 
 if [ ! -e $SRC_DIR ]; then
     echo "Please install rust-src component"
@@ -14,14 +15,17 @@ mkdir -p $DST_DIR/src
 cp -r $SRC_DIR/src $DST_DIR/
 
 pushd $DST_DIR
+echo "[GIT] init"
 git init
+echo "[GIT] add"
 git add .
+echo "[GIT] commit"
 git commit -m "Initial commit" -q
-git apply ../../patches/*.patch
+for file in $(ls ../../patches/ | grep -v patcha); do
+echo "[GIT] apply" $file
+git apply ../../patches/$file
+git commit -am "Patch $file"
+done
 popd
 
-# `alloc_system` has been merged with libstd, which doesn't build yet.
-# This copies the original source to the sysroot source dir to simplify building it
-cp -r alloc_system $DST_DIR/src/liballoc_system
-
 echo "Successfully prepared libcore for building"

build_sysroot/src/lib.rs

@@ -0,0 +1 @@
+#![no_std]

clean_all.sh

@@ -0,0 +1,4 @@
+#!/bin/bash --verbose
+set -e
+
+rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/,Cargo.lock} perf.data{,.old}

config.sh

@@ -19,5 +19,4 @@ else
 fi
 
 export RUSTFLAGS='-Zalways-encode-mir -Cpanic=abort -Zcodegen-backend='$(pwd)'/target/'$channel'/librustc_codegen_cranelift.'$dylib_ext
-export XARGO_RUST_SRC=$(pwd)'/target/libcore/src'
 RUSTC="rustc $RUSTFLAGS -L crate=target/out --out-dir target/out"

flamegraph.sh

@@ -5,7 +5,8 @@ source config.sh
 rm perf.data* || true
 
 # Profile compiling libcore
-perf record -F 9000 --call-graph dwarf -- $RUSTC --crate-type lib target/libcore/src/libcore/lib.rs --crate-name core
+perf record -F 9000 --call-graph dwarf \
+    -- $RUSTC --crate-type lib build_sysroot/sysroot_src/src/libcore/lib.rs --crate-name core
 
 # Generate the flamegraph
 perf script | ../FlameGraph/stackcollapse-perf.pl | grep cranelift | ../FlameGraph/flamegraph.pl > abc.svg

prepare.sh

@@ -0,0 +1,6 @@
+#!/bin/bash --verbose
+set -e
+
+rustup component add rust-src
+./build_sysroot/prepare_sysroot_src.sh
+cargo install hyperfine || echo "Skipping hyperfine install"

test.sh

@@ -21,26 +21,21 @@ SHOULD_RUN=1 $RUSTC --crate-type bin example/mini_core_hello_world.rs --cfg jit
 echo "[AOT] mini_core_hello_world"
 build_example_bin mini_core_hello_world example/mini_core_hello_world.rs
 
-pushd xargo
-rm -r ~/.xargo/HOST || true
-rm -r target || true
-time xargo build
-popd
+echo "[BUILD] sysroot"
+time ./build_sysroot/build_sysroot.sh
 
 # TODO linux linker doesn't accept duplicate definitions
-#$RUSTC --sysroot ~/.xargo/HOST example/alloc_example.rs --crate-type bin
+# echo "[BUILD+RUN] alloc_example"
+#$RUSTC --sysroot ./build_sysroot/sysroot example/alloc_example.rs --crate-type bin
 #./target/out/alloc_example
 
-$RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin
+echo "[BUILD] mod_bench"
+$RUSTC --sysroot ./build_sysroot/sysroot example/mod_bench.rs --crate-type bin
 
-echo "[BUILD] RUSTFLAGS=-Zmir-opt-level=3"
-pushd xargo
-rm -r ~/.xargo/HOST || true
-rm -r target || true
-time RUSTFLAGS="-Zmir-opt-level=3 $RUSTFLAGS" xargo build
-popd
+echo "[BUILD] sysroot in release mode"
+./build_sysroot/build_sysroot.sh --release
 
-COMPILE_MOD_BENCH_INLINE="$RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -Og --crate-name mod_bench_inline"
+COMPILE_MOD_BENCH_INLINE="$RUSTC --sysroot ./build_sysroot/sysroot example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -Og --crate-name mod_bench_inline"
 COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o target/out/mod_bench_llvm_0 -Cpanic=abort"
 COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o target/out/mod_bench_llvm_1 -Cpanic=abort"
 COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o target/out/mod_bench_llvm_2 -Cpanic=abort"