Skip to content

Commit

Permalink
Include unpublishable path dependencies in package
Browse files Browse the repository at this point in the history 
Rebase of: rust-lang#4735

RFC: rust-lang/rfcs#2224

Before this patch, all path deps must be published before the root crate
is published to the registry.

This patch allow `cargo package` to include files of path dependencies
in the generated package if the dependencies have the manifest key
"package.publish" set to false, which means the crate cannot be
registered as a sharable crate by itself at a registry, but still
allowed to be published along with the root crate.

Co-authored-by: J.W <[email protected]>
  • Loading branch information
@haraldh
haraldh and J.W committed Aug 21, 2020
1 parent 868a1cf commit 7e4ee1b
Show file tree
Hide file tree
Showing 6 changed files with 380 additions and 46 deletions.
62 changes: 47 additions & 15 deletions src/cargo/ops/cargo_package.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,15 @@ use log::debug;
use tar::{Archive, Builder, EntryType, Header};

use crate::core::compiler::{BuildConfig, CompileMode, DefaultExecutor, Executor};
use crate::core::{Feature, Shell, Verbosity, Workspace};
use crate::core::{EitherManifest, Feature, Shell, Verbosity, Workspace};
use crate::core::{Package, PackageId, PackageSet, Resolve, Source, SourceId};
use crate::sources::PathSource;
use crate::util::errors::{CargoResult, CargoResultExt};
use crate::util::paths;
use crate::util::toml::TomlManifest;
use crate::util::toml::{read_manifest, TomlManifest};
use crate::util::{self, restricted_names, Config, FileLock};
use crate::{drop_println, ops};
use same_file::is_same_file;

pub struct PackageOpts<'cfg> {
pub config: &'cfg Config,
Expand Down Expand Up @@ -56,7 +57,7 @@ enum FileContents {

enum GeneratedFile {
/// Generates `Cargo.toml` by rewriting the original.
Manifest,
Manifest(Package),
/// Generates `Cargo.lock` in some cases (like if there is a binary).
Lockfile,
/// Adds a `.cargo-vcs_info.json` file if in a (clean) git repo.
Expand Down Expand Up @@ -149,6 +150,7 @@ fn build_ar_list(
) -> CargoResult<Vec<ArchiveFile>> {
let mut result = Vec::new();
let root = pkg.root();
let manifest_path = pkg.manifest_path();
for src_file in src_files {
let rel_path = src_file.strip_prefix(&root)?.to_path_buf();
check_filename(&rel_path, &mut ws.config().shell())?;
Expand All @@ -158,18 +160,48 @@ fn build_ar_list(
anyhow::format_err!("non-utf8 path in source directory: {}", rel_path.display())
})?
.to_string();
match rel_str.as_ref() {

let rel_filename = rel_path
.file_name()
.unwrap()
.to_str()
.ok_or_else(|| {
anyhow::format_err!("non-utf8 path in source directory: {}", rel_path.display())
})?
.to_string();

match rel_filename.as_ref() {
"Cargo.toml" => {
result.push(ArchiveFile {
rel_path: PathBuf::from("Cargo.toml.orig"),
rel_str: "Cargo.toml.orig".to_string(),
contents: FileContents::OnDisk(src_file),
});
result.push(ArchiveFile {
rel_path,
rel_str,
contents: FileContents::Generated(GeneratedFile::Manifest),
});
if is_same_file(&src_file, manifest_path)? {
result.push(ArchiveFile {
rel_path: PathBuf::from("Cargo.toml.orig"),
rel_str: "Cargo.toml.orig".to_string(),
contents: FileContents::OnDisk(src_file),
});
result.push(ArchiveFile {
rel_path,
rel_str,
contents: FileContents::Generated(GeneratedFile::Manifest(pkg.clone())),
});
} else {
let (manifest, _) =
read_manifest(&src_file, pkg.package_id().source_id(), ws.config())?;
if let EitherManifest::Real(manifest) = manifest {
let new_pkg = Package::new(manifest, &rel_path);

let orig_path_str = rel_str.clone() + ".orig";
result.push(ArchiveFile {
rel_path: PathBuf::from(&orig_path_str),
rel_str: orig_path_str,
contents: FileContents::OnDisk(src_file),
});
result.push(ArchiveFile {
rel_path,
rel_str,
contents: FileContents::Generated(GeneratedFile::Manifest(new_pkg)),
});
}
}
}
"Cargo.lock" => continue,
VCS_INFO_FILE => anyhow::bail!(
Expand Down Expand Up @@ -519,7 +551,7 @@ fn tar(
}
FileContents::Generated(generated_kind) => {
let contents = match generated_kind {
GeneratedFile::Manifest => pkg.to_registry_toml(ws)?,
GeneratedFile::Manifest(ref pkg) => pkg.to_registry_toml(ws)?,
GeneratedFile::Lockfile => build_lock(ws)?,
GeneratedFile::VcsInfo(s) => s,
};
Expand Down
50 changes: 38 additions & 12 deletions src/cargo/sources/path.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::fmt::{self, Debug, Formatter};
use std::fs;
use std::path::{Path, PathBuf};
Expand All @@ -6,10 +7,12 @@ use filetime::FileTime;
use ignore::gitignore::GitignoreBuilder;
use ignore::Match;
use log::{trace, warn};
use same_file::is_same_file;

use crate::core::source::MaybePackage;
use crate::core::{Dependency, Package, PackageId, Source, SourceId, Summary};
use crate::ops;
use crate::util::toml::is_remote_source;
use crate::util::{internal, paths, CargoResult, CargoResultExt, Config};

pub struct PathSource<'cfg> {
Expand Down Expand Up @@ -145,6 +148,8 @@ impl<'cfg> PathSource<'cfg> {
}
};

let mut remote_prefixes = HashMap::<PathBuf, bool>::new();

let mut filter = |path: &Path, is_dir: bool| -> CargoResult<bool> {
let relative_path = path.strip_prefix(root)?;

Expand All @@ -155,7 +160,22 @@ impl<'cfg> PathSource<'cfg> {
return Ok(true);
}

ignore_should_package(relative_path, is_dir)
let res = ignore_should_package(relative_path, is_dir);
if let Ok(true) = res {
for (ref prefix, is_remote) in remote_prefixes.iter() {
if path.starts_with(prefix) {
return Ok(!is_remote);
}
}
if let Some(dirpath) = path.parent() {
if !is_same_file(dirpath, root).unwrap_or(false) {
let is_remote = is_remote_source(&dirpath.join("Cargo.toml"));
remote_prefixes.insert(dirpath.to_path_buf(), is_remote);
return Ok(!is_remote);
}
}
}
res
};

// Attempt Git-prepopulate only if no `include` (see rust-lang/cargo#4135).
Expand Down Expand Up @@ -267,7 +287,7 @@ impl<'cfg> PathSource<'cfg> {
_ => None,
});

let mut subpackages_found = Vec::new();
let mut ignored_subpackages_found = Vec::new();

for (file_path, is_dir) in index_files.chain(untracked) {
let file_path = file_path?;
Expand All @@ -283,16 +303,20 @@ impl<'cfg> PathSource<'cfg> {
// The `target` directory is never included.
Some("target") => continue,

// Keep track of all sub-packages found and also strip out all
// matches we've found so far. Note, though, that if we find
// our own `Cargo.toml`, we keep going.
// Keep track of all published sub-packages found and also
// strip out all matches we've found so far. Note, though,
// that if we find our own `Cargo.toml` we keep going.
Some("Cargo.toml") => {
let path = file_path.parent().unwrap();
if path != pkg_path {
if !is_same_file(&path, pkg_path).unwrap_or(false) {
warn!("subpackage found: {}", path.display());
ret.retain(|p| !p.starts_with(path));
subpackages_found.push(path.to_path_buf());
continue;
if is_remote_source(&file_path) {
// if package is allowed to be published,
// then path dependency will become crates.io deps
ret.retain(|p| !p.starts_with(&path));
ignored_subpackages_found.push(path.to_path_buf());
continue;
}
}
}

Expand All @@ -301,7 +325,10 @@ impl<'cfg> PathSource<'cfg> {

// If this file is part of any other sub-package we've found so far,
// skip it.
if subpackages_found.iter().any(|p| file_path.starts_with(p)) {
if ignored_subpackages_found
.iter()
.any(|p| file_path.starts_with(p))
{
continue;
}

Expand Down Expand Up @@ -400,8 +427,7 @@ impl<'cfg> PathSource<'cfg> {
ret.push(path.to_path_buf());
return Ok(());
}
// Don't recurse into any sub-packages that we have.
if !is_root && path.join("Cargo.toml").exists() {
if !is_root && is_remote_source(&path.join("Cargo.toml")) {
return Ok(());
}

Expand Down
90 changes: 74 additions & 16 deletions src/cargo/util/toml/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::str;
Expand Down Expand Up @@ -167,6 +168,41 @@ and this will become a hard error in the future.",
Err(first_error.context("could not parse input as TOML"))
}

#[derive(Deserialize)]
struct SourceManifest {
package: Option<SourcePackage>,
project: Option<SourcePackage>,
}

#[derive(Deserialize)]
struct SourcePackage {
publish: Option<VecStringOrBool>,
}

/// Check if the manifest file (if exists) has package.publish=true
///
/// We do not validate the manifest here to avoid dealing with recursion.
pub fn is_remote_source(manifest_path: &Path) -> bool {
use ::std::io::Read;

if let Ok(mut file) = fs::File::open(manifest_path) {
let mut content = String::new();
if file.read_to_string(&mut content).is_ok() {
if let Ok(config) = toml::from_str::<SourceManifest>(&content) {
let pkg = config.package.or(config.project);
if let Some(pkg) = pkg {
return match pkg.publish {
Some(VecStringOrBool::VecString(ref registries)) => !registries.is_empty(),
Some(VecStringOrBool::Bool(false)) => false,
Some(VecStringOrBool::Bool(true)) | None => true,
};
}
}
}
}
false
}

type TomlLibTarget = TomlTarget;
type TomlBinTarget = TomlTarget;
type TomlExampleTarget = TomlTarget;
Expand Down Expand Up @@ -820,7 +856,7 @@ pub struct TomlProject {
resolver: Option<String>,
}

#[derive(Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TomlWorkspace {
members: Option<Vec<String>>,
#[serde(rename = "default-members")]
Expand Down Expand Up @@ -901,13 +937,14 @@ impl TomlManifest {
example: self.example.clone(),
test: self.test.clone(),
bench: self.bench.clone(),
dependencies: map_deps(config, self.dependencies.as_ref(), all)?,
dependencies: map_deps(config, self.dependencies.as_ref(), all, package_root)?,
dev_dependencies: map_deps(
config,
self.dev_dependencies
.as_ref()
.or_else(|| self.dev_dependencies2.as_ref()),
TomlDependency::is_version_specified,
package_root,
)?,
dev_dependencies2: None,
build_dependencies: map_deps(
Expand All @@ -916,6 +953,7 @@ impl TomlManifest {
.as_ref()
.or_else(|| self.build_dependencies2.as_ref()),
all,
package_root,
)?,
build_dependencies2: None,
features: self.features.clone(),
Expand All @@ -926,13 +964,19 @@ impl TomlManifest {
Ok((
k.clone(),
TomlPlatform {
dependencies: map_deps(config, v.dependencies.as_ref(), all)?,
dependencies: map_deps(
config,
v.dependencies.as_ref(),
all,
package_root,
)?,
dev_dependencies: map_deps(
config,
v.dev_dependencies
.as_ref()
.or_else(|| v.dev_dependencies2.as_ref()),
TomlDependency::is_version_specified,
package_root,
)?,
dev_dependencies2: None,
build_dependencies: map_deps(
Expand All @@ -941,6 +985,7 @@ impl TomlManifest {
.as_ref()
.or_else(|| v.build_dependencies2.as_ref()),
all,
package_root,
)?,
build_dependencies2: None,
},
Expand All @@ -954,7 +999,7 @@ impl TomlManifest {
},
replace: None,
patch: None,
workspace: None,
workspace: self.workspace.clone(),
badges: self.badges.clone(),
cargo_features,
});
Expand All @@ -963,6 +1008,7 @@ impl TomlManifest {
config: &Config,
deps: Option<&BTreeMap<String, TomlDependency>>,
filter: impl Fn(&TomlDependency) -> bool,
root: &Path,
) -> CargoResult<Option<BTreeMap<String, TomlDependency>>> {
let deps = match deps {
Some(deps) => deps,
Expand All @@ -971,28 +1017,40 @@ impl TomlManifest {
let deps = deps
.iter()
.filter(|(_k, v)| filter(v))
.map(|(k, v)| Ok((k.clone(), map_dependency(config, v)?)))
.map(|(k, v)| Ok((k.clone(), map_dependency(config, v, root)?)))
.collect::<CargoResult<BTreeMap<_, _>>>()?;
Ok(Some(deps))
}

fn map_dependency(config: &Config, dep: &TomlDependency) -> CargoResult<TomlDependency> {
fn map_dependency(
config: &Config,
dep: &TomlDependency,
root: &Path,
) -> CargoResult<TomlDependency> {
match dep {
TomlDependency::Detailed(d) => {
let mut d = d.clone();
// Path dependencies become crates.io deps.
d.path.take();
let mut nd = d.clone();
if let Some(ref path) = d.path {
let path = root.join(PathBuf::from(path));
if is_remote_source(&path.join("Cargo.toml")) {
// if package is allowed to be published,
// then path dependency will become crates.io deps
// Path dependencies become crates.io deps.
nd.path.take();
}
}

// Same with git dependencies.
d.git.take();
d.branch.take();
d.tag.take();
d.rev.take();
nd.git.take();
nd.branch.take();
nd.tag.take();
nd.rev.take();
// registry specifications are elaborated to the index URL
if let Some(registry) = d.registry.take() {
if let Some(registry) = nd.registry.take() {
let src = SourceId::alt_registry(config, &registry)?;
d.registry_index = Some(src.url().to_string());
nd.registry_index = Some(src.url().to_string());
}
Ok(TomlDependency::Detailed(d))
Ok(TomlDependency::Detailed(nd))
}
TomlDependency::Simple(s) => Ok(TomlDependency::Detailed(DetailedTomlDependency {
version: Some(s.clone()),
Expand Down
Loading

0 comments on commit 7e4ee1b

Please sign in to comment.