Add toolchain management

README.md

@@ -192,6 +192,20 @@ This is done to create a unified experience of Python installations and to avoid
 incompatibilities created by different Python distributions.  Most importantly this also
 means you never need to compile a Python any more, it just downloads prepared binaries.
 
+## Managing Python Toolchains
+
+You can register custom Python toolchains with `rye toolchain register`:
+
+```
+$ rye toolchain register ~/Downloads/pypy3.9-v7.3.11-macos_arm64/bin/python
+Registered /Users/mitsuhiko/Downloads/pypy3.9-v7.3.11-macos_arm64/bin/python as [email protected]
+```
+
+Afterwards you can pin it, in this case with `rye pin [email protected]`.  The auto detection of
+the name might not be great, in which case you can provide an explicit name with `--name`.
+To remove downloaded or linked toolchains, you can use the `rye toolchain remove` command.
+To list what's available, use `rye toolchain list`.
+
 ## Global Tools
 
 If you want tools to be installed into isolated virtualenvs (like pipsi and pipx), you

rye/find-downloads.py

@@ -152,10 +152,11 @@ def _sort_key(info):
 
 
 print("// generated code, do not edit")
+print("use std::borrow::Cow;")
 print("pub const CPYTHON_VERSIONS: &[(PythonVersion, &str, &str, &str)] = &[")
 for py_ver, choices in sorted(
     final_results.items(), key=lambda x: x[0], reverse=True
 ):
     for (arch, platform), url in sorted(choices.items()):
-        print('    (PythonVersion { kind: "cpython", major: %d, minor: %d, patch: %d }, "%s", "%s", "%s"),' % (py_ver + (arch, platform, url)))
+        print('    (PythonVersion { kind: Cow::Borrowed("cpython"), major: %d, minor: %d, patch: %d, suffix: None }, "%s", "%s", "%s"),' % (py_ver + (arch, platform, url)))
 print("];")

rye/src/bootstrap.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::env::consts::{ARCH, OS};
 use std::io::Write;
 use std::os::unix::fs::symlink;
@@ -9,11 +10,17 @@ use anyhow::{bail, Error};
 use console::style;
 use indicatif::{ProgressBar, ProgressStyle};
 
-use crate::config::{get_app_dir, get_py_bin, get_py_dir};
-use crate::sources::{get_download_url, PythonVersion};
+use crate::config::{get_app_dir, get_canonical_py_path, get_py_bin};
+use crate::sources::{get_download_url, PythonVersion, PythonVersionRequest};
 use crate::utils::{unpack_tarball, CommandOutput};
 
-const SELF_PYTHON_VERSION: &str = "3.10.9";
+const SELF_PYTHON_VERSION: PythonVersionRequest = PythonVersionRequest {
+    kind: Some(Cow::Borrowed("cpython")),
+    major: 3,
+    minor: Some(10),
+    patch: None,
+    suffix: None,
+};
 const SELF_SITE_PACKAGES: &str = "python3.10/site-packages";
 
 /// Bootstraps the venv for rye itself
@@ -28,7 +35,7 @@ pub fn ensure_self_venv(output: CommandOutput) -> Result<PathBuf, Error> {
         eprintln!("Bootstrapping rye internals");
     }
 
-    let version = fetch(SELF_PYTHON_VERSION, output)?;
+    let version = fetch(&SELF_PYTHON_VERSION, output)?;
     let py_bin = get_py_bin(&version)?;
 
     // initialize the virtualenv
@@ -107,13 +114,26 @@ pub fn get_pip_module(venv: &Path) -> PathBuf {
 }
 
 /// Fetches a version if missing.
-pub fn fetch(version: &str, output: CommandOutput) -> Result<PythonVersion, Error> {
+pub fn fetch(
+    version: &PythonVersionRequest,
+    output: CommandOutput,
+) -> Result<PythonVersion, Error> {
+    if let Ok(version) = PythonVersion::try_from(version.clone()) {
+        let py_path = get_canonical_py_path(&version)?;
+        if py_path.is_dir() | py_path.is_file() {
+            if output == CommandOutput::Verbose {
+                eprintln!("Python version already downloaded. Skipping.");
+            }
+            return Ok(version);
+        }
+    }
+
     let (version, url) = match get_download_url(version, OS, ARCH) {
         Some(result) => result,
         None => bail!("unknown version {}", version),
     };
 
-    let target_dir = get_py_dir(&version)?;
+    let target_dir = get_canonical_py_path(&version)?;
     if output == CommandOutput::Verbose {
         eprintln!("target dir: {}", target_dir.display());
     }

rye/src/cli/fetch.rs

@@ -19,6 +19,6 @@ pub struct Args {
 
 pub fn execute(cmd: Args) -> Result<(), Error> {
     let output = CommandOutput::from_quiet_and_verbose(cmd.quiet, cmd.verbose);
-    fetch(&cmd.version, output)?;
+    fetch(&cmd.version.parse()?, output)?;
     Ok(())
 }

rye/src/cli/mod.rs

@@ -12,6 +12,7 @@ mod run;
 mod shim;
 mod show;
 mod sync;
+mod toolchain;
 mod uninstall;
 
 #[derive(Parser, Debug)]
@@ -32,6 +33,7 @@ enum Command {
     Run(run::Args),
     Show(show::Args),
     Sync(sync::Args),
+    Toolchain(toolchain::Args),
     Uninstall(uninstall::Args),
 }
 
@@ -52,6 +54,7 @@ pub fn execute() -> Result<(), Error> {
         Command::Run(cmd) => run::execute(cmd),
         Command::Show(cmd) => show::execute(cmd),
         Command::Sync(cmd) => sync::execute(cmd),
+        Command::Toolchain(cmd) => toolchain::execute(cmd),
         Command::Uninstall(cmd) => uninstall::execute(cmd),
     }
 }

rye/src/cli/pin.rs

@@ -1,40 +1,32 @@
-use std::env::{
-    self,
-    consts::{ARCH, OS},
-};
+use std::env;
 use std::fs;
 
 use anyhow::{anyhow, Error};
 use clap::Parser;
 
-use crate::{pyproject::PyProject, sources::get_download_url};
+use crate::config::get_pinnable_version;
+use crate::pyproject::PyProject;
+use crate::sources::PythonVersionRequest;
 
 /// Pins a Python version to this project.
 #[derive(Parser, Debug)]
 pub struct Args {
-    /// The version of Python to fetch.
+    /// The version of Python to pin.
     version: String,
 }
 
 pub fn execute(cmd: Args) -> Result<(), Error> {
-    let (version, _) = get_download_url(&cmd.version, OS, ARCH)
-        .ok_or_else(|| anyhow!("unsupported version for this platform"))?;
-
-    // pin in a format known to other toolchains for as long as we're under cpython
-    let serialized_version = version.to_string();
-    let to_write = if let Some(rest) = serialized_version.strip_prefix("cpython@") {
-        rest
-    } else {
-        &serialized_version
-    };
+    let req: PythonVersionRequest = cmd.version.parse()?;
+    let to_write = get_pinnable_version(&req)
+        .ok_or_else(|| anyhow!("unsupported/unknown version for this platform"))?;
 
     let version_file = match PyProject::discover() {
         Ok(proj) => proj.root_path().join(".python-version"),
         Err(_) => env::current_dir()?.join(".python-version"),
     };
     fs::write(&version_file, format!("{}\n", to_write))?;
 
-    eprintln!("pinned {} in {}", version, version_file.display());
+    eprintln!("pinned {} in {}", to_write, version_file.display());
 
     Ok(())
 }

rye/src/cli/toolchain.rs

@@ -0,0 +1,157 @@
+use std::cmp::Reverse;
+use std::collections::HashMap;
+use std::env::consts::{ARCH, OS};
+use std::fs;
+use std::os::unix::fs::symlink;
+use std::path::PathBuf;
+use std::process::Command;
+
+use anyhow::{bail, Error};
+use clap::Parser;
+use console::style;
+use serde::Deserialize;
+
+use crate::config::{get_canonical_py_path, list_known_toolchains};
+use crate::sources::{iter_downloadable, PythonVersion};
+
+const INSPECT_SCRIPT: &str = r#"
+import json
+import platform
+print(json.dumps({
+    "python_implementation": platform.python_implementation(),
+    "python_version": platform.python_version(),
+}))
+"#;
+
+#[derive(Debug, Deserialize)]
+struct InspectInfo {
+    python_implementation: String,
+    python_version: String,
+}
+
+/// Helper utility to manage Python toolchains.
+#[derive(Parser, Debug)]
+pub struct Args {
+    #[command(subcommand)]
+    command: SubCommand,
+}
+
+/// Register a Python binary.
+///
+/// Rye by default will automatically download Python releases from the internet.
+/// Howver it's also possible to register already available local Python
+/// installations.  This allows you to use rye with self compiled Pythons.
+#[derive(Parser, Debug)]
+pub struct RegisterCommand {
+    /// Path to the Python binary.
+    path: PathBuf,
+    /// Name of the toolchain.  If not provided a name is auto detected.
+    #[arg(short, long)]
+    name: Option<String>,
+}
+
+/// Removes a toolchain.
+#[derive(Parser, Debug)]
+pub struct RemoveCommand {
+    /// Name and version of the toolchain.
+    version: String,
+}
+
+/// List all registered toolchains
+#[derive(Parser, Debug)]
+pub struct ListCommand {
+    /// Also include non installed, but downloadable toolchains
+    #[arg(long)]
+    include_downloadable: bool,
+}
+
+#[derive(Parser, Debug)]
+enum SubCommand {
+    Fetch(crate::cli::fetch::Args),
+    List(ListCommand),
+    Register(RegisterCommand),
+    Remove(RemoveCommand),
+}
+
+pub fn execute(cmd: Args) -> Result<(), Error> {
+    match cmd.command {
+        SubCommand::Register(args) => register(args),
+        SubCommand::Fetch(args) => crate::cli::fetch::execute(args),
+        SubCommand::List(args) => list(args),
+        SubCommand::Remove(args) => remove(args),
+    }
+}
+
+fn register(cmd: RegisterCommand) -> Result<(), Error> {
+    let output = Command::new(&cmd.path)
+        .arg("-c")
+        .arg(INSPECT_SCRIPT)
+        .output()?;
+    if !output.status.success() {
+        bail!("passed path does not appear to be a valid Python installation");
+    }
+
+    let info: InspectInfo = serde_json::from_slice(&output.stdout)?;
+    let target_version = match cmd.name {
+        Some(ref name) => format!("{}@{}", name, info.python_version),
+        None => {
+            let name = if info.python_implementation.eq_ignore_ascii_case("cpython") {
+                "custom-cpython"
+            } else {
+                &info.python_implementation
+            };
+            format!("{}@{}", name.to_ascii_lowercase(), info.python_version)
+        }
+    };
+    let target_version: PythonVersion = target_version.parse()?;
+    let target = get_canonical_py_path(&target_version)?;
+
+    if target.is_file() || target.is_dir() {
+        bail!("target Python path {} is already in use", target.display());
+    }
+
+    symlink(&cmd.path, target)?;
+    println!("Registered {} as {}", cmd.path.display(), target_version);
+
+    Ok(())
+}
+
+pub fn remove(cmd: RemoveCommand) -> Result<(), Error> {
+    let ver: PythonVersion = cmd.version.parse()?;
+    let path = get_canonical_py_path(&ver)?;
+    if path.is_file() {
+        fs::remove_file(&path)?;
+        eprintln!("Removed toolchain link {}", &ver);
+    } else if path.is_dir() {
+        fs::remove_dir_all(&path)?;
+        eprintln!("Removed installed toolchain {}", &ver);
+    } else {
+        eprintln!("Toolchain is not installed");
+    }
+    Ok(())
+}
+
+fn list(cmd: ListCommand) -> Result<(), Error> {
+    let mut toolchains = list_known_toolchains()?
+        .into_iter()
+        .map(|version| (version, true))
+        .collect::<HashMap<_, _>>();
+
+    if cmd.include_downloadable {
+        for version in iter_downloadable(OS, ARCH) {
+            toolchains.entry(version).or_insert(false);
+        }
+    }
+
+    let mut versions = toolchains.into_iter().collect::<Vec<_>>();
+    versions.sort_by_cached_key(|a| (!a.1, a.0.kind.to_string(), Reverse(a.clone())));
+
+    for (version, installed) in versions {
+        if installed {
+            println!("{}", style(&version).green());
+        } else {
+            println!("{} (downloadable)", style(version).dim());
+        }
+    }
+    Ok(())
+}