Auto merge of #34743 - badboy:llvm-upgrade, r=eddyb

LLVM upgrade

As discussed in https://internals.rust-lang.org/t/need-help-with-emscripten-port/3154/46 I'm trying to update the used LLVM checkout in Rust.

I basically took @shepmaster's code and applied it on top (though I did the commits manually, the [original commits have better descriptions](master...avr-rust:avr-support).

With these changes I was able to build rustc. `make check` throws one last error on `run-pass/issue-28950.rs`. Output: https://gist.github.com/badboy/bcdd3bbde260860b6159aa49070a9052

I took the metadata changes as is and they seem to work, though it now uses the module in another step. I'm not sure if this is the best and correct way.

Things to do:

* [x] ~~Make `run-pass/issue-28950.rs` pass~~ unrelated
* [x] Find out how the `PositionIndependentExecutable` setting is now used
* [x] Is the `llvm::legacy` still the right way to do these things?

cc @brson @alexcrichton

configure

@@ -1020,6 +1020,12 @@ then
             err "bad LLVM version: $LLVM_VERSION, need >=3.7"
             ;;
     esac
+
+    if "$CFG_LLVM_ROOT/bin/llvm-mc" -help | grep -- "-relocation-model"; then
+        msg "found older llvm-mc"
+        CFG_LLVM_MC_HAS_RELOCATION_MODEL=1
+        putvar CFG_LLVM_MC_HAS_RELOCATION_MODEL
+    fi
 fi
 
 # Even when the user overrides the choice of CC, still try to detect

mk/platform.mk

@@ -221,12 +221,19 @@ define CFG_MAKE_TOOLCHAIN
     LLVM_MC_RELOCATION_MODEL="default"
   endif
 
+  # LLVM changed this flag in 3.9
+  ifdef CFG_LLVM_MC_HAS_RELOCATION_MODEL
+    LLVM_MC_RELOC_FLAG := -relocation-model=$$(LLVM_MC_RELOCATION_MODEL)
+  else
+    LLVM_MC_RELOC_FLAG := -position-independent
+  endif
+
   # We're using llvm-mc as our assembler because it supports
   # .cfi pseudo-ops on mac
   CFG_ASSEMBLE_$(1)=$$(CPP_$(1)) -E $$(2) | \
                     $$(LLVM_MC_$$(CFG_BUILD)) \
                     -assemble \
-                    -relocation-model=$$(LLVM_MC_RELOCATION_MODEL) \
+                    $$(LLVM_MC_RELOC_FLAG) \
                     -filetype=obj \
                     -triple=$(1) \
                     -o=$$(1)

src/etc/mklldeps.py

@@ -77,6 +77,13 @@ def runErr(args):
         lib = lib.strip()[2:]
     elif lib[0] == '-':
         lib = lib.strip()[1:]
+    # If this actually points at a literal file then we're on MSVC which now
+    # prints full paths, so get just the name of the library and strip off the
+    # trailing ".lib"
+    elif os.path.exists(lib):
+        lib = os.path.basename(lib)[:-4]
+    elif lib[-4:] == '.lib':
+        lib = lib[:-4]
     f.write("#[link(name = \"" + lib + "\"")
     if not llvm_shared and 'LLVM' in lib:
         f.write(", kind = \"static\"")

src/librustc_back/target/aarch64_linux_android.rs

@@ -20,7 +20,7 @@ pub fn target() -> TargetResult {
         llvm_target: "aarch64-linux-android".to_string(),
         target_endian: "little".to_string(),
         target_pointer_width: "64".to_string(),
-        data_layout: "e-m:e-i64:64-i128:128-n32:64-S128".to_string(),
+        data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".to_string(),
         arch: "aarch64".to_string(),
         target_os: "android".to_string(),
         target_env: "".to_string(),

src/librustc_back/target/aarch64_unknown_linux_gnu.rs

@@ -18,7 +18,7 @@ pub fn target() -> TargetResult {
         target_endian: "little".to_string(),
         target_pointer_width: "64".to_string(),
         target_env: "gnu".to_string(),
-        data_layout: "e-m:e-i64:64-i128:128-n32:64-S128".to_string(),
+        data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".to_string(),
         arch: "aarch64".to_string(),
         target_os: "linux".to_string(),
         target_vendor: "unknown".to_string(),

src/librustc_llvm/build.rs

@@ -13,7 +13,7 @@ extern crate build_helper;
 
 use std::process::Command;
 use std::env;
-use std::path::PathBuf;
+use std::path::{PathBuf, Path};
 
 use build_helper::output;
 
@@ -135,8 +135,17 @@ fn main() {
             &lib[2..]
         } else if lib.starts_with("-") {
             &lib[1..]
+        } else if Path::new(lib).exists() {
+            // On MSVC llvm-config will print the full name to libraries, but
+            // we're only interested in the name part
+            let name = Path::new(lib).file_name().unwrap().to_str().unwrap();
+            name.trim_right_matches(".lib")
+        } else if lib.ends_with(".lib") {
+            // Some MSVC libraries just come up with `.lib` tacked on, so chop
+            // that off
+            lib.trim_right_matches(".lib")
         } else {
-            continue;
+            continue
         };
 
         // Don't need or want this library, but LLVM's CMake build system
@@ -145,7 +154,7 @@ fn main() {
         // library and it otherwise may just pull in extra dependencies on
         // libedit which we don't want
         if name == "LLVMLineEditor" {
-            continue;
+            continue
         }
 
         let kind = if name.starts_with("LLVM") {
@@ -165,7 +174,9 @@ fn main() {
     let mut cmd = Command::new(&llvm_config);
     cmd.arg("--ldflags");
     for lib in output(&mut cmd).split_whitespace() {
-        if is_crossed {
+        if lib.starts_with("-LIBPATH:") {
+                println!("cargo:rustc-link-search=native={}", &lib[9..]);
+        } else if is_crossed {
             if lib.starts_with("-L") {
                 println!("cargo:rustc-link-search=native={}",
                          lib[2..].replace(&host, &target));

src/librustc_llvm/lib.rs

@@ -226,7 +226,7 @@ impl Attributes {
 
     pub fn apply_callsite(&self, idx: usize, callsite: ValueRef) {
         unsafe {
-            LLVMAddCallSiteAttribute(callsite, idx as c_uint, self.regular.bits());
+            LLVMRustAddCallSiteAttribute(callsite, idx as c_uint, self.regular.bits());
             if self.dereferenceable_bytes != 0 {
                 LLVMAddDereferenceableCallSiteAttr(callsite, idx as c_uint,
                                                    self.dereferenceable_bytes);
@@ -1056,7 +1056,7 @@ extern {
     pub fn LLVMSetInstrParamAlignment(Instr: ValueRef,
                                       index: c_uint,
                                       align: c_uint);
-    pub fn LLVMAddCallSiteAttribute(Instr: ValueRef,
+    pub fn LLVMRustAddCallSiteAttribute(Instr: ValueRef,
                                     index: c_uint,
                                     Val: uint64_t);
     pub fn LLVMAddDereferenceableCallSiteAttr(Instr: ValueRef,
@@ -1561,7 +1561,7 @@ extern {
                                 Alignment: c_uint)
                                 -> ValueRef;
 
-    pub fn LLVMBuildAtomicCmpXchg(B: BuilderRef,
+    pub fn LLVMRustBuildAtomicCmpXchg(B: BuilderRef,
                                   LHS: ValueRef,
                                   CMP: ValueRef,
                                   RHS: ValueRef,
@@ -1591,9 +1591,6 @@ extern {
 
     /// Creates target data from a target layout string.
     pub fn LLVMCreateTargetData(StringRep: *const c_char) -> TargetDataRef;
-    /// Adds the target data to the given pass manager. The pass manager
-    /// references the target data only weakly.
-    pub fn LLVMAddTargetData(TD: TargetDataRef, PM: PassManagerRef);
     /// Number of bytes clobbered when doing a Store to *T.
     pub fn LLVMStoreSizeOfType(TD: TargetDataRef, Ty: TypeRef)
                                -> c_ulonglong;
@@ -2155,6 +2152,7 @@ extern {
 
     pub fn LLVMRustSetComdat(M: ModuleRef, V: ValueRef, Name: *const c_char);
     pub fn LLVMRustUnsetComdat(V: ValueRef);
+    pub fn LLVMRustSetModulePIELevel(M: ModuleRef);
 }
 
 // LLVM requires symbols from this library, but apparently they're not printed

src/librustc_trans/back/write.rs

@@ -24,6 +24,7 @@ use util::fs::link_or_copy;
 use errors::{self, Handler, Level, DiagnosticBuilder};
 use errors::emitter::Emitter;
 use syntax_pos::MultiSpan;
+use context::{is_pie_binary, get_reloc_model};
 
 use std::collections::HashMap;
 use std::ffi::{CStr, CString};
@@ -154,32 +155,11 @@ fn get_llvm_opt_size(optimize: config::OptLevel) -> llvm::CodeGenOptSize {
 }
 
 pub fn create_target_machine(sess: &Session) -> TargetMachineRef {
-    let reloc_model_arg = match sess.opts.cg.relocation_model {
-        Some(ref s) => &s[..],
-        None => &sess.target.target.options.relocation_model[..],
-    };
-    let reloc_model = match reloc_model_arg {
-        "pic" => llvm::RelocPIC,
-        "static" => llvm::RelocStatic,
-        "default" => llvm::RelocDefault,
-        "dynamic-no-pic" => llvm::RelocDynamicNoPic,
-        _ => {
-            sess.err(&format!("{:?} is not a valid relocation mode",
-                             sess.opts
-                                 .cg
-                                 .relocation_model));
-            sess.abort_if_errors();
-            bug!();
-        }
-    };
+    let reloc_model = get_reloc_model(sess);
 
     let opt_level = get_llvm_opt_level(sess.opts.optimize);
     let use_softfp = sess.opts.cg.soft_float;
 
-    let any_library = sess.crate_types.borrow().iter().any(|ty| {
-        *ty != config::CrateTypeExecutable
-    });
-
     let ffunction_sections = sess.target.target.options.function_sections;
     let fdata_sections = ffunction_sections;
 
@@ -220,7 +200,7 @@ pub fn create_target_machine(sess: &Session) -> TargetMachineRef {
             reloc_model,
             opt_level,
             use_softfp,
-            !any_library && reloc_model == llvm::RelocPIC,
+            is_pie_binary(sess),
             ffunction_sections,
             fdata_sections,
         )

src/librustc_trans/builder.rs

@@ -1083,7 +1083,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                          failure_order: AtomicOrdering,
                          weak: llvm::Bool) -> ValueRef {
         unsafe {
-            llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src,
+            llvm::LLVMRustBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src,
                                          order, failure_order, weak)
         }
     }

src/librustc_trans/context.rs

@@ -34,6 +34,7 @@ use rustc::ty::subst::{Substs, VecPerParamSpace};
 use rustc::ty::{self, Ty, TyCtxt};
 use session::config::NoDebugInfo;
 use session::Session;
+use session::config;
 use symbol_map::SymbolMap;
 use util::sha2::Sha256;
 use util::nodemap::{NodeMap, NodeSet, DefIdMap, FnvHashMap, FnvHashSet};
@@ -322,6 +323,38 @@ impl<'a, 'tcx> Iterator for CrateContextMaybeIterator<'a, 'tcx> {
     }
 }
 
+pub fn get_reloc_model(sess: &Session) -> llvm::RelocMode {
+    let reloc_model_arg = match sess.opts.cg.relocation_model {
+        Some(ref s) => &s[..],
+        None => &sess.target.target.options.relocation_model[..],
+    };
+
+    match reloc_model_arg {
+        "pic" => llvm::RelocPIC,
+        "static" => llvm::RelocStatic,
+        "default" => llvm::RelocDefault,
+        "dynamic-no-pic" => llvm::RelocDynamicNoPic,
+        _ => {
+            sess.err(&format!("{:?} is not a valid relocation mode",
+                             sess.opts
+                                 .cg
+                                 .relocation_model));
+            sess.abort_if_errors();
+            bug!();
+        }
+    }
+}
+
+fn is_any_library(sess: &Session) -> bool {
+    sess.crate_types.borrow().iter().any(|ty| {
+        *ty != config::CrateTypeExecutable
+    })
+}
+
+pub fn is_pie_binary(sess: &Session) -> bool {
+    !is_any_library(sess) && get_reloc_model(sess) == llvm::RelocPIC
+}
+
 unsafe fn create_context_and_module(sess: &Session, mod_name: &str) -> (ContextRef, ModuleRef) {
     let llcx = llvm::LLVMContextCreate();
     let mod_name = CString::new(mod_name).unwrap();
@@ -337,7 +370,25 @@ unsafe fn create_context_and_module(sess: &Session, mod_name: &str) -> (ContextR
         let data_layout = str::from_utf8(CStr::from_ptr(data_layout).to_bytes())
             .ok().expect("got a non-UTF8 data-layout from LLVM");
 
-        if sess.target.target.data_layout != data_layout {
+        // Unfortunately LLVM target specs change over time, and right now we
+        // don't have proper support to work with any more than one
+        // `data_layout` than the one that is in the rust-lang/rust repo. If
+        // this compiler is configured against a custom LLVM, we may have a
+        // differing data layout, even though we should update our own to use
+        // that one.
+        //
+        // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we
+        // disable this check entirely as we may be configured with something
+        // that has a different target layout.
+        //
+        // Unsure if this will actually cause breakage when rustc is configured
+        // as such.
+        //
+        // FIXME(#34960)
+        let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or("");
+        let custom_llvm_used = cfg_llvm_root.trim() != "";
+
+        if !custom_llvm_used && sess.target.target.data_layout != data_layout {
             bug!("data-layout for builtin `{}` target, `{}`, \
                   differs from LLVM default, `{}`",
                  sess.target.target.llvm_target,
@@ -352,6 +403,11 @@ unsafe fn create_context_and_module(sess: &Session, mod_name: &str) -> (ContextR
     let llvm_target = sess.target.target.llvm_target.as_bytes();
     let llvm_target = CString::new(llvm_target).unwrap();
     llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr());
+
+    if is_pie_binary(sess) {
+        llvm::LLVMRustSetModulePIELevel(llmod);
+    }
+
     (llcx, llmod)
 }
 
@@ -558,7 +614,9 @@ impl<'tcx> LocalCrateContext<'tcx> {
                                                           &llmod_id[..]);
 
             let dbg_cx = if shared.tcx.sess.opts.debuginfo != NoDebugInfo {
-                Some(debuginfo::CrateDebugContext::new(llmod))
+                let dctx = debuginfo::CrateDebugContext::new(llmod);
+                debuginfo::metadata::compile_unit_metadata(shared, &dctx, shared.tcx.sess);
+                Some(dctx)
             } else {
                 None
             };