Skip to content

Commit

Permalink
Use Mmap to open the rmeta file.
Browse files Browse the repository at this point in the history
Because those files are quite large, contribute significantly to peak
memory usage, but only a small fraction of the data is ever read.
  • Loading branch information
nnethercote committed Oct 31, 2018
1 parent de9666f commit 39c5a8a
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 5 deletions.
2 changes: 2 additions & 0 deletions src/Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2280,13 +2280,15 @@ version = "0.0.0"
dependencies = [
"flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"proc_macro 0.0.0",
"rustc 0.0.0",
"rustc_data_structures 0.0.0",
"rustc_errors 0.0.0",
"rustc_metadata_utils 0.0.0",
"rustc_target 0.0.0",
"serialize 0.0.0",
"stable_deref_trait 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"syntax 0.0.0",
"syntax_ext 0.0.0",
"syntax_pos 0.0.0",
Expand Down
6 changes: 4 additions & 2 deletions src/librustc_metadata/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ crate-type = ["dylib"]
[dependencies]
flate2 = "1.0"
log = "0.4"
memmap = "0.6"
proc_macro = { path = "../libproc_macro" }
rustc = { path = "../librustc" }
rustc_target = { path = "../librustc_target" }
rustc_data_structures = { path = "../librustc_data_structures" }
rustc_errors = { path = "../librustc_errors" }
rustc_target = { path = "../librustc_target" }
rustc_metadata_utils = { path = "../librustc_metadata_utils" }
serialize = { path = "../libserialize" }
stable_deref_trait = "1.0.0"
syntax = { path = "../libsyntax" }
syntax_ext = { path = "../libsyntax_ext" }
syntax_pos = { path = "../libsyntax_pos" }
rustc_metadata_utils = { path = "../librustc_metadata_utils" }
2 changes: 2 additions & 0 deletions src/librustc_metadata/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
extern crate libc;
#[macro_use]
extern crate log;
extern crate memmap;
extern crate stable_deref_trait;
#[macro_use]
extern crate syntax;
extern crate syntax_pos;
Expand Down
26 changes: 23 additions & 3 deletions src/librustc_metadata/locator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,12 +243,14 @@ use std::cmp;
use std::fmt;
use std::fs;
use std::io::{self, Read};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::time::Instant;

use flate2::read::DeflateDecoder;

use rustc_data_structures::owning_ref::OwningRef;

pub struct CrateMismatch {
path: PathBuf,
got: String,
Expand Down Expand Up @@ -856,6 +858,19 @@ fn get_metadata_section(target: &Target,
return ret;
}

/// A trivial wrapper for `Mmap` that implements `StableDeref`.
struct StableDerefMmap(memmap::Mmap);

impl Deref for StableDerefMmap {
type Target = [u8];

fn deref(&self) -> &[u8] {
self.0.deref()
}
}

unsafe impl stable_deref_trait::StableDeref for StableDerefMmap {}

fn get_metadata_section_imp(target: &Target,
flavor: CrateFlavor,
filename: &Path,
Expand Down Expand Up @@ -892,9 +907,14 @@ fn get_metadata_section_imp(target: &Target,
}
}
CrateFlavor::Rmeta => {
let buf = fs::read(filename).map_err(|_|
format!("failed to read rmeta metadata: '{}'", filename.display()))?;
rustc_erase_owner!(OwningRef::new(buf).map_owner_box())
// mmap the file, because only a small fraction of it is read.
let file = std::fs::File::open(filename).map_err(|_|
format!("failed to open rmeta metadata: '{}'", filename.display()))?;
let mmap = unsafe { memmap::Mmap::map(&file) };
let mmap = mmap.map_err(|_|
format!("failed to mmap rmeta metadata: '{}'", filename.display()))?;

rustc_erase_owner!(OwningRef::new(StableDerefMmap(mmap)).map_owner_box())
}
};
let blob = MetadataBlob(raw_bytes);
Expand Down

0 comments on commit 39c5a8a

Please sign in to comment.