Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add pixi build deps #1195

Open
wants to merge 4 commits into
base: feat/pixi-build
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
323 changes: 217 additions & 106 deletions Cargo.lock

Large diffs are not rendered by default.

38 changes: 26 additions & 12 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -130,18 +130,32 @@ regex = "1.11.1"
async-recursion = "1.1.1"

# Rattler crates
rattler = { version = "0.28.0", default-features = false, features = ["cli-tools", "indicatif"] }
rattler_cache = { version = "0.2.8", default-features = false }
rattler_conda_types = { version = "0.29.0", default-features = false }
rattler_digest = { version = "1.0.3", default-features = false, features = ["serde"] }
rattler_index = { version = "0.19.34", default-features = false }
rattler_networking = { version = "0.21.5", default-features = false }
rattler_redaction = { version = "0.1.3" }
rattler_repodata_gateway = { version = "0.21.20", default-features = false, features = ["gateway"] }
rattler_shell = { version = "0.22.5", default-features = false, features = ["sysinfo"] }
rattler_solve = { version = "1.2.1", default-features = false, features = ["resolvo", "serde"] }
rattler_virtual_packages = { version = "1.1.8", default-features = false }
rattler_package_streaming = { version = "0.22.11", default-features = false }
# rattler = { version = "0.28.0", default-features = false, features = ["cli-tools", "indicatif"] }
# rattler_cache = { version = "0.2.8", default-features = false }
# rattler_conda_types = { version = "0.29.0", default-features = false }
# rattler_digest = { version = "1.0.3", default-features = false, features = ["serde"] }
# rattler_index = { version = "0.19.34", default-features = false }
# rattler_networking = { version = "0.21.5", default-features = false }
# rattler_redaction = { version = "0.1.3" }
# rattler_repodata_gateway = { version = "0.21.20", default-features = false, features = ["gateway"] }
# rattler_shell = { version = "0.22.5", default-features = false, features = ["sysinfo"] }
# rattler_solve = { version = "1.2.1", default-features = false, features = ["resolvo", "serde"] }
# rattler_virtual_packages = { version = "1.1.8", default-features = false }
# rattler_package_streaming = { version = "0.22.11", default-features = false }
rattler = { git = "https://github.com/conda/rattler", branch = "main", default-features = false, features = ["cli-tools", "indicatif"]}
rattler_conda_types = { git = "https://github.com/conda/rattler", branch = "main", default-features = false }
rattler_cache = { git = "https://github.com/conda/rattler", branch = "main", default-features = false}
rattler_digest = { git = "https://github.com/conda/rattler", branch = "main" , default-features = false, features = ["serde"]}
rattler_networking = { git = "https://github.com/conda/rattler", branch = "main", default-features = false}
rattler_package_streaming = { git = "https://github.com/conda/rattler", branch = "main", default-features = false}
rattler_repodata_gateway = { git = "https://github.com/conda/rattler", branch = "main", default-features = false, features = ["gateway"] }
rattler_redaction = { git = "https://github.com/conda/rattler", branch = "main" }
rattler_index = { git = "https://github.com/conda/rattler", branch = "main", default-features = false}
rattler_shell = { git = "https://github.com/conda/rattler", branch = "main", default-features = false, features = ["sysinfo"]}
rattler_solve = { git = "https://github.com/conda/rattler", branch = "main" , default-features = false, features = ["resolvo", "serde"]}
rattler_virtual_packages = { git = "https://github.com/conda/rattler", branch = "main", default-features = false}


lazy_static = "1.5.0"

[dev-dependencies]
Expand Down
23 changes: 22 additions & 1 deletion src/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ pub async fn skip_existing(
let channels = if only_local {
vec![
Channel::from_directory(&first_output.build_configuration.directories.output_dir)
.base_url,
.base_url
.url()
.as_ref()
.clone(),
]
} else {
all_channels
Expand Down Expand Up @@ -103,12 +106,15 @@ pub async fn run_build(
output: Output,
tool_configuration: &tool_configuration::Configuration,
) -> miette::Result<(Output, PathBuf)> {
eprintln!("before create build dir");
output
.build_configuration
.directories
.create_build_dir()
.into_diagnostic()?;

eprintln!("after create build dir");

let span = tracing::info_span!("Running build for", recipe = output.identifier());
let _enter = span.enter();
output.record_build_start();
Expand All @@ -120,28 +126,43 @@ pub async fn run_build(
.await
.into_diagnostic()?;

eprintln!("before build or fetch cache");

let output = output.build_or_fetch_cache(tool_configuration).await?;

eprintln!("after build or fetch cache");

let output = output
.resolve_dependencies(tool_configuration)
.await
.into_diagnostic()?;

eprintln!("before install env");

output
.install_environments(tool_configuration)
.await
.into_diagnostic()?;

eprintln!("after install env");

eprintln!("before run build script");
output.run_build_script().await.into_diagnostic()?;

eprintln!("after run build script");

// Package all the new files
let (result, paths_json) = output
.create_package(tool_configuration)
.await
.into_diagnostic()?;

eprintln!("before record artifact ");

output.record_artifact(&result, &paths_json);

eprintln!("after record artifact ");

let span = tracing::info_span!("Running package tests");
let enter = span.enter();

Expand Down
10 changes: 8 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,10 @@ pub async fn get_build_output(
.channel
.clone()
.into_iter()
.map(|c| Channel::from_str(c, &tool_config.channel_config).map(|c| c.base_url))
.map(|c| {
Channel::from_str(c, &tool_config.channel_config)
.map(|c| c.base_url.url().clone().as_ref().clone())
})
.collect::<Result<Vec<_>, _>>()
.into_diagnostic()?;

Expand Down Expand Up @@ -451,7 +454,10 @@ pub async fn run_test_from_args(
.channel
.unwrap_or_else(|| vec!["conda-forge".to_string()])
.into_iter()
.map(|name| Channel::from_str(name, &tool_config.channel_config).map(|c| c.base_url))
.map(|name| {
Channel::from_str(name, &tool_config.channel_config)
.map(|c| c.base_url.url().as_ref().clone())
})
.collect::<Result<Vec<_>, _>>()
.into_diagnostic()?;

Expand Down
4 changes: 2 additions & 2 deletions src/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ pub fn build_reindexed_channels(
// Reindex the output channel from the files on disk
index(output_dir, Some(&build_configuration.target_platform))?;

Ok(iter::once(output_channel.base_url)
Ok(iter::once(output_channel.base_url.url().as_ref().clone())
.chain(build_configuration.channels.iter().cloned())
.collect())
}
Expand Down Expand Up @@ -828,7 +828,7 @@ mod test {
file_name: "test-1.2.3-h123.tar.bz2".into(),
url: Url::from_str("https://test.com/test/linux-64/test-1.2.3-h123.tar.bz2")
.unwrap(),
channel: "test".into(),
channel: Some("test".to_string()),
}],
};

Expand Down
10 changes: 9 additions & 1 deletion src/package_test/run_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,15 @@ pub async fn run_test(
tracing::info!("Creating test environment in '{}'", prefix.display());

let mut channels = config.channels.clone();
channels.insert(0, Channel::from_directory(tmp_repo.path()).base_url);
channels.insert(
0,
Channel::from_directory(tmp_repo.path())
.base_url
.url()
.as_ref()
.clone()
.clone(),
);

let host_platform = config.host_platform.clone().unwrap_or_else(|| {
if target_platform == Platform::NoArch {
Expand Down
4 changes: 2 additions & 2 deletions src/render/resolved_dependencies.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ impl ResolvedDependencies {
.render(long),
record.package_record.version.to_string(),
record.package_record.build.to_string(),
short_channel(&record.channel),
short_channel(&record.channel.clone().unwrap_or_default()),
record
.package_record
.size
Expand All @@ -315,7 +315,7 @@ impl ResolvedDependencies {
"".to_string(),
record.package_record.version.to_string(),
record.package_record.build.to_string(),
short_channel(&record.channel),
short_channel(&record.channel.clone().unwrap_or_default()),
record
.package_record
.size
Expand Down
6 changes: 4 additions & 2 deletions src/render/solver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,17 @@ fn print_as_table(packages: &[RepoDataRecord]) {
.iter()
.sorted_by_key(|p| p.package_record.name.as_normalized())
{
let channel_short = if package.channel.contains('/') {
let channel_short = if package.channel.clone().unwrap_or_default().contains('/') {
package
.channel
.clone()
.unwrap_or_default()
.rsplit('/')
.find(|s| !s.is_empty())
.expect("yep will crash if ")
.to_string()
} else {
package.channel.to_string()
package.channel.clone().unwrap_or_default()
};

table.add_row(vec![
Expand Down
6 changes: 3 additions & 3 deletions src/script.rs
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,9 @@ impl Interpreter for BashInterpreter {

if !output.status.success() {
let status_code = output.status.code().unwrap_or(1);
tracing::error!("Script failed with status {}", status_code);
tracing::error!("Work directory: '{}'", args.work_dir.display());
tracing::error!("{}", DEBUG_HELP);
eprintln!("Script failed with status {}", status_code);
eprintln!("Work directory: '{}'", args.work_dir.display());
eprintln!("{}", DEBUG_HELP);
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Script failed".to_string(),
Expand Down
6 changes: 6 additions & 0 deletions src/source/copy_dir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ impl<'a> CopyDir<'a> {

pub fn run(self) -> Result<CopyDirResult, SourceError> {
// Create the to path because we're going to copy the contents only
eprintln!("Creating directory: {:?}", self.to_path);
create_dir_all(self.to_path)?;

let mut result = CopyDirResult {
Expand All @@ -183,6 +184,11 @@ impl<'a> CopyDir<'a> {
exclude_globs: make_glob_match_map(self.globvec.exclude_globs())?,
};

eprintln!(
"Copying files from {:?} to {:?}",
self.from_path, self.to_path
);

let copied_paths = WalkBuilder::new(self.from_path)
// disregard global gitignore
.git_global(self.use_git_global)
Expand Down
9 changes: 9 additions & 0 deletions src/source/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,14 @@ pub async fn fetch_sources(
work_dir.to_path_buf()
};

eprintln!("src target directory is {:?}", src.target_directory());

eprintln!("work dir is {:?}", work_dir);

eprintln!("dest dir is {:?}", dest_dir);

eprintln!("src path is {:?}", src_path);

// Create folder if it doesn't exist
if !dest_dir.exists() {
fs::create_dir_all(&dest_dir)?;
Expand All @@ -231,6 +239,7 @@ pub async fn fetch_sources(
.run()
},
)?;
eprintln!("copied paths are {:?}", copy_result.copied_paths());
tracing::info!(
"Copied {} files into isolated environment",
copy_result.copied_paths().len()
Expand Down
1 change: 1 addition & 0 deletions src/tool_configuration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ impl ConfigurationBuilder {
jlap_enabled: true,
zstd_enabled: self.use_zstd,
bz2_enabled: self.use_bz2,
sharded_enabled: false,
cache_action: Default::default(),
},
per_channel: Default::default(),
Expand Down