diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d40389b20..113b8a2f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -396,6 +396,11 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }}/crates/uv-trampoline + - name: "Test committed binaries" + working-directory: ${{ github.workspace }} + run: | + rustup target add ${{ matrix.target-arch }}-pc-windows-msvc + cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc # Build and copy the new binaries - name: "Build" working-directory: ${{ github.workspace }}/crates/uv-trampoline @@ -403,9 +408,11 @@ jobs: cargo build --target ${{ matrix.target-arch }}-pc-windows-msvc cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-console.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-console.exe cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-gui.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-gui.exe - - name: "Test" - working-directory: ${{ github.workspace }}/crates/uv-trampoline - run: cargo test --target ${{ matrix.target-arch }}-pc-windows-msvc --test * + - name: "Test new binaries" + working-directory: ${{ github.workspace }} + run: | + # We turn off the default "production" test feature since these are debug binaries + cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc --no-default-features typos: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 28a28b420..c3e38ba11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4770,6 +4770,7 @@ dependencies = [ "uv-pep440", "uv-platform-tags", "uv-pypi-types", + "uv-trampoline-builder", "uv-warnings", "walkdir", "zip", @@ -5268,6 +5269,20 @@ dependencies = [ "uv-virtualenv", ] +[[package]] +name = "uv-trampoline-builder" +version = "0.0.1" +dependencies = [ + "anyhow", + "assert_cmd", + "assert_fs", + "fs-err", + "thiserror", + "uv-fs", + "which", + "zip", +] + [[package]] name = "uv-types" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index a2358c19f..3f4a71dca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ uv-settings = { path = "crates/uv-settings" } uv-shell = { path = "crates/uv-shell" } uv-state = { path = "crates/uv-state" } uv-static = { path = "crates/uv-static" } +uv-trampoline-builder = { path = "crates/uv-trampoline-builder" } uv-tool = { path = "crates/uv-tool" } uv-types = { path = "crates/uv-types" } uv-version = { path = "crates/uv-version" } diff --git a/crates/uv-install-wheel/Cargo.toml b/crates/uv-install-wheel/Cargo.toml index 97f5cf57a..d7dcd8fd1 100644 --- a/crates/uv-install-wheel/Cargo.toml +++ b/crates/uv-install-wheel/Cargo.toml @@ -28,6 +28,7 @@ uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-platform-tags = { workspace = true } uv-pypi-types = { workspace = true } +uv-trampoline-builder = { workspace = true } uv-warnings = { workspace = true } clap = { workspace = true, optional = true, features = ["derive"] } diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index 83300d42d..69249229a 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -97,4 +97,6 @@ pub enum Error { MismatchedVersion(Version, Version), #[error("Invalid egg-link")] InvalidEggLink(PathBuf), + #[error(transparent)] + LauncherError(#[from] uv_trampoline_builder::Error), } diff --git a/crates/uv-install-wheel/src/wheel.rs b/crates/uv-install-wheel/src/wheel.rs index 559237e16..cf640348e 100644 --- a/crates/uv-install-wheel/src/wheel.rs +++ b/crates/uv-install-wheel/src/wheel.rs @@ -1,11 +1,8 @@ use std::collections::HashMap; -use std::io::{BufReader, Cursor, Read, Seek, Write}; +use std::io; +use std::io::{BufReader, Read, Seek, Write}; use std::path::{Path, PathBuf}; -use std::{env, io}; -use crate::record::RecordEntry; -use crate::script::Script; -use crate::{Error, Layout}; use data_encoding::BASE64URL_NOPAD; use fs_err as fs; use fs_err::{DirEntry, File}; @@ -13,39 +10,17 @@ use mailparse::parse_headers; use rustc_hash::FxHashMap; use sha2::{Digest, Sha256}; use tracing::{instrument, warn}; +use walkdir::WalkDir; + use uv_cache_info::CacheInfo; use uv_fs::{relative_to, Simplified}; use uv_normalize::PackageName; use uv_pypi_types::DirectUrl; -use walkdir::WalkDir; -use zip::write::FileOptions; -use zip::ZipWriter; +use uv_trampoline_builder::windows_script_launcher; -const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-gui.exe"); - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-console.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe"); +use crate::record::RecordEntry; +use crate::script::Script; +use crate::{Error, Layout}; /// Wrapper script template function /// @@ -158,87 +133,6 @@ fn format_shebang(executable: impl AsRef, os_name: &str, relocatable: bool format!("#!{executable}") } -/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as -/// stored zip file. -/// -/// -#[allow(unused_variables)] -pub(crate) fn windows_script_launcher( - launcher_python_script: &str, - is_gui: bool, - python_executable: impl AsRef, -) -> Result, Error> { - // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain - // compilation on all platforms. - if cfg!(not(windows)) { - return Err(Error::NotWindows); - } - - let launcher_bin: &[u8] = match env::consts::ARCH { - #[cfg(all(windows, target_arch = "x86"))] - "x86" => { - if is_gui { - LAUNCHER_I686_GUI - } else { - LAUNCHER_I686_CONSOLE - } - } - #[cfg(all(windows, target_arch = "x86_64"))] - "x86_64" => { - if is_gui { - LAUNCHER_X86_64_GUI - } else { - LAUNCHER_X86_64_CONSOLE - } - } - #[cfg(all(windows, target_arch = "aarch64"))] - "aarch64" => { - if is_gui { - LAUNCHER_AARCH64_GUI - } else { - LAUNCHER_AARCH64_CONSOLE - } - } - #[cfg(windows)] - arch => { - return Err(Error::UnsupportedWindowsArch(arch)); - } - #[cfg(not(windows))] - arch => &[], - }; - - let mut payload: Vec = Vec::new(); - { - // We're using the zip writer, but with stored compression - // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 - // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 - let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); - let mut archive = ZipWriter::new(Cursor::new(&mut payload)); - let error_msg = "Writing to Vec should never fail"; - archive.start_file("__main__.py", stored).expect(error_msg); - archive - .write_all(launcher_python_script.as_bytes()) - .expect(error_msg); - archive.finish().expect(error_msg); - } - - let python = python_executable.as_ref(); - let python_path = python.simplified_display().to_string(); - - let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); - launcher.extend_from_slice(launcher_bin); - launcher.extend_from_slice(&payload); - launcher.extend_from_slice(python_path.as_bytes()); - launcher.extend_from_slice( - &u32::try_from(python_path.as_bytes().len()) - .expect("File Path to be smaller than 4GB") - .to_le_bytes(), - ); - launcher.extend_from_slice(&LAUNCHER_MAGIC_NUMBER); - - Ok(launcher) -} - /// Returns a [`PathBuf`] to `python[w].exe` for script execution. /// /// @@ -1075,54 +969,6 @@ mod test { Ok(()) } - #[test] - #[cfg(all(windows, target_arch = "x86"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_I686_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_I686_GUI.len() - ); - assert!( - super::LAUNCHER_I686_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_I686_CONSOLE.len() - ); - } - - #[test] - #[cfg(all(windows, target_arch = "x86_64"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_X86_64_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_X86_64_GUI.len() - ); - assert!( - super::LAUNCHER_X86_64_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_X86_64_CONSOLE.len() - ); - } - - #[test] - #[cfg(all(windows, target_arch = "aarch64"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_AARCH64_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_AARCH64_GUI.len() - ); - assert!( - super::LAUNCHER_AARCH64_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_AARCH64_CONSOLE.len() - ); - } - #[test] fn test_script_executable() -> Result<()> { // Test with adjacent pythonw.exe diff --git a/crates/uv-trampoline-builder/Cargo.toml b/crates/uv-trampoline-builder/Cargo.toml new file mode 100644 index 000000000..f4c54e15a --- /dev/null +++ b/crates/uv-trampoline-builder/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "uv-trampoline-builder" +version = "0.0.1" +publish = false +description = "Builds launchers for `uv-trampoline`" + +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[features] +default = ["production"] + +# Expect tests to run against production builds of `uv-trampoline` binaries, rather than debug builds +production = [] + +[lints] +workspace = true + +[dependencies] +uv-fs = { workspace = true } +thiserror = { workspace = true } +zip = { workspace = true } + +[dev-dependencies] +assert_cmd = { version = "2.0.16" } +assert_fs = { version = "1.1.2" } +anyhow = { version = "1.0.89" } +fs-err = { workspace = true } +which = { workspace = true } diff --git a/crates/uv-trampoline-builder/src/lib.rs b/crates/uv-trampoline-builder/src/lib.rs new file mode 100644 index 000000000..3b1503f7c --- /dev/null +++ b/crates/uv-trampoline-builder/src/lib.rs @@ -0,0 +1,406 @@ +use std::io::{Cursor, Write}; +use std::path::Path; + +use thiserror::Error; +use uv_fs::Simplified; +use zip::write::FileOptions; +use zip::ZipWriter; + +#[cfg(all(windows, target_arch = "x86"))] +const LAUNCHER_I686_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-gui.exe"); + +#[cfg(all(windows, target_arch = "x86"))] +const LAUNCHER_I686_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-console.exe"); + +#[cfg(all(windows, target_arch = "x86_64"))] +const LAUNCHER_X86_64_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe"); + +#[cfg(all(windows, target_arch = "x86_64"))] +const LAUNCHER_X86_64_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe"); + +#[cfg(all(windows, target_arch = "aarch64"))] +const LAUNCHER_AARCH64_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe"); + +#[cfg(all(windows, target_arch = "aarch64"))] +const LAUNCHER_AARCH64_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe"); + +/// The kind of trampoline launcher to create. +/// +/// See [`uv-trampoline::bounce::TrampolineKind`]. +enum LauncherKind { + /// The trampoline should execute itself, it's a zipped Python script. + Script, + /// The trampoline should just execute Python, it's a proxy Python executable. + Python, +} + +impl LauncherKind { + const fn magic_number(&self) -> &'static [u8; 4] { + match self { + Self::Script => b"UVSC", + Self::Python => b"UVPY", + } + } +} + +/// Note: The caller is responsible for adding the path of the wheel we're installing. +#[derive(Error, Debug)] +pub enum Error { + #[error( + "Unable to create Windows launcher for: {0} (only x86_64, x86, and arm64 are supported)" + )] + UnsupportedWindowsArch(&'static str), + #[error("Unable to create Windows launcher on non-Windows platform")] + NotWindows, +} + +#[allow(clippy::unnecessary_wraps, unused_variables)] +fn get_launcher_bin(gui: bool) -> Result<&'static [u8], Error> { + Ok(match std::env::consts::ARCH { + #[cfg(all(windows, target_arch = "x86"))] + "x86" => { + if gui { + LAUNCHER_I686_GUI + } else { + LAUNCHER_I686_CONSOLE + } + } + #[cfg(all(windows, target_arch = "x86_64"))] + "x86_64" => { + if gui { + LAUNCHER_X86_64_GUI + } else { + LAUNCHER_X86_64_CONSOLE + } + } + #[cfg(all(windows, target_arch = "aarch64"))] + "aarch64" => { + if gui { + LAUNCHER_AARCH64_GUI + } else { + LAUNCHER_AARCH64_CONSOLE + } + } + #[cfg(windows)] + arch => { + return Err(Error::UnsupportedWindowsArch(arch)); + } + #[cfg(not(windows))] + _ => &[], + }) +} + +/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as +/// stored zip file. +/// +/// +#[allow(unused_variables)] +pub fn windows_script_launcher( + launcher_python_script: &str, + is_gui: bool, + python_executable: impl AsRef, +) -> Result, Error> { + // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain + // compilation on all platforms. + if cfg!(not(windows)) { + return Err(Error::NotWindows); + } + + let launcher_bin: &[u8] = get_launcher_bin(is_gui)?; + + let mut payload: Vec = Vec::new(); + { + // We're using the zip writer, but with stored compression + // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 + // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 + let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut archive = ZipWriter::new(Cursor::new(&mut payload)); + let error_msg = "Writing to Vec should never fail"; + archive.start_file("__main__.py", stored).expect(error_msg); + archive + .write_all(launcher_python_script.as_bytes()) + .expect(error_msg); + archive.finish().expect(error_msg); + } + + let python = python_executable.as_ref(); + let python_path = python.simplified_display().to_string(); + + let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); + launcher.extend_from_slice(launcher_bin); + launcher.extend_from_slice(&payload); + launcher.extend_from_slice(python_path.as_bytes()); + launcher.extend_from_slice( + &u32::try_from(python_path.as_bytes().len()) + .expect("file path should be smaller than 4GB") + .to_le_bytes(), + ); + launcher.extend_from_slice(LauncherKind::Script.magic_number()); + + Ok(launcher) +} + +/// A minimal .exe launcher binary for Python. +/// +/// Sort of equivalent to a `python` symlink on Unix. +#[allow(unused_variables)] +pub fn windows_python_launcher( + python_executable: impl AsRef, + is_gui: bool, +) -> Result, Error> { + // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain + // compilation on all platforms. + if cfg!(not(windows)) { + return Err(Error::NotWindows); + } + + let launcher_bin: &[u8] = get_launcher_bin(is_gui)?; + + let python = python_executable.as_ref(); + let python_path = python.simplified_display().to_string(); + + let mut launcher: Vec = Vec::with_capacity(launcher_bin.len()); + launcher.extend_from_slice(launcher_bin); + launcher.extend_from_slice(python_path.as_bytes()); + launcher.extend_from_slice( + &u32::try_from(python_path.as_bytes().len()) + .expect("file path should be smaller than 4GB") + .to_le_bytes(), + ); + launcher.extend_from_slice(LauncherKind::Python.magic_number()); + + Ok(launcher) +} + +#[cfg(all(test, windows))] +#[allow(clippy::print_stdout)] +mod test { + use std::io::Write; + use std::path::Path; + use std::process::Command; + + use anyhow::Result; + use assert_cmd::prelude::OutputAssertExt; + use assert_fs::prelude::PathChild; + use fs_err::File; + + use which::which; + + use super::{windows_python_launcher, windows_script_launcher}; + + #[test] + #[cfg(all(windows, target_arch = "x86", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_I686_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_I686_GUI.len() + ); + assert!( + super::LAUNCHER_I686_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_I686_CONSOLE.len() + ); + } + + #[test] + #[cfg(all(windows, target_arch = "x86_64", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_X86_64_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_X86_64_GUI.len() + ); + assert!( + super::LAUNCHER_X86_64_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_X86_64_CONSOLE.len() + ); + } + + #[test] + #[cfg(all(windows, target_arch = "aarch64", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_AARCH64_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_AARCH64_GUI.len() + ); + assert!( + super::LAUNCHER_AARCH64_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_AARCH64_CONSOLE.len() + ); + } + + /// Utility script for the test. + fn get_script_launcher(shebang: &str, is_gui: bool) -> String { + if is_gui { + format!( + r##"{shebang} +# -*- coding: utf-8 -*- +import re +import sys + +def make_gui() -> None: + from tkinter import Tk, ttk + root = Tk() + root.title("uv Test App") + frm = ttk.Frame(root, padding=10) + frm.grid() + ttk.Label(frm, text="Hello from uv-trampoline-gui.exe").grid(column=0, row=0) + root.mainloop() + +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) + sys.exit(make_gui()) +"## + ) + } else { + format!( + r##"{shebang} +# -*- coding: utf-8 -*- +import re +import sys + +def main_console() -> None: + print("Hello from uv-trampoline-console.exe", file=sys.stdout) + print("Hello from uv-trampoline-console.exe", file=sys.stderr) + for arg in sys.argv[1:]: + print(arg, file=sys.stderr) + +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) + sys.exit(main_console()) +"## + ) + } + } + + /// See [`uv-install-wheel::wheel::format_shebang`]. + fn format_shebang(executable: impl AsRef) -> String { + // Convert the executable to a simplified path. + let executable = executable.as_ref().display().to_string(); + format!("#!{executable}") + } + + #[test] + fn console_script_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let console_bin_path = temp_dir.child("launcher.console.exe"); + + // Locate an arbitrary python installation from PATH + let python_executable_path = which("python")?; + + // Generate Launcher Script + let launcher_console_script = + get_script_launcher(&format_shebang(&python_executable_path), false); + + // Generate Launcher Payload + let console_launcher = + windows_script_launcher(&launcher_console_script, false, &python_executable_path)?; + + // Create Launcher + File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; + + println!( + "Wrote Console Launcher in {}", + console_bin_path.path().display() + ); + + let stdout_predicate = "Hello from uv-trampoline-console.exe\r\n"; + let stderr_predicate = "Hello from uv-trampoline-console.exe\r\n"; + + // Test Console Launcher + #[cfg(windows)] + Command::new(console_bin_path.path()) + .assert() + .success() + .stdout(stdout_predicate) + .stderr(stderr_predicate); + + let args_to_test = vec!["foo", "bar", "foo bar", "foo \"bar\"", "foo 'bar'"]; + let stderr_predicate = format!("{}{}\r\n", stderr_predicate, args_to_test.join("\r\n")); + + // Test Console Launcher (with args) + Command::new(console_bin_path.path()) + .args(args_to_test) + .assert() + .success() + .stdout(stdout_predicate) + .stderr(stderr_predicate); + + Ok(()) + } + + #[test] + fn console_python_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let console_bin_path = temp_dir.child("launcher.console.exe"); + + // Locate an arbitrary python installation from PATH + let python_executable_path = which("python")?; + + // Generate Launcher Payload + let console_launcher = windows_python_launcher(&python_executable_path, false)?; + + // Create Launcher + File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; + + println!( + "Wrote Python Launcher in {}", + console_bin_path.path().display() + ); + + // Test Console Launcher + Command::new(console_bin_path.path()) + .arg("-c") + .arg("print('Hello from Python Launcher')") + .assert() + .success() + .stdout("Hello from Python Launcher\r\n"); + + Ok(()) + } + + #[test] + #[ignore] + fn gui_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let gui_bin_path = temp_dir.child("launcher.gui.exe"); + + // Locate an arbitrary pythonw installation from PATH + let pythonw_executable_path = which("pythonw")?; + + // Generate Launcher Script + let launcher_gui_script = + get_script_launcher(&format_shebang(&pythonw_executable_path), true); + + // Generate Launcher Payload + let gui_launcher = + windows_script_launcher(&launcher_gui_script, true, &pythonw_executable_path)?; + + // Create Launcher + File::create(gui_bin_path.path())?.write_all(gui_launcher.as_ref())?; + + println!("Wrote GUI Launcher in {}", gui_bin_path.path().display()); + + // Test GUI Launcher + // NOTICE: This will spawn a GUI and will wait until you close the window. + Command::new(gui_bin_path.path()).assert().success(); + + Ok(()) + } +} diff --git a/crates/uv-trampoline/Cargo.lock b/crates/uv-trampoline/Cargo.lock index 0611026d7..5ae4861c1 100644 --- a/crates/uv-trampoline/Cargo.lock +++ b/crates/uv-trampoline/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler" @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "linux-raw-sys" @@ -346,9 +346,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags", "errno", diff --git a/crates/uv-trampoline/README.md b/crates/uv-trampoline/README.md index 8ff40aa82..3a4de1933 100644 --- a/crates/uv-trampoline/README.md +++ b/crates/uv-trampoline/README.md @@ -12,17 +12,20 @@ LLD and add the `rustup` targets: ```shell sudo apt install llvm clang lld -rustup target add i686-pc-windows-msvc -rustup target add x86_64-pc-windows-msvc -rustup target add aarch64-pc-windows-msvc +cargo install cargo-xwin +rustup toolchain install nightly-2024-10-27 +rustup component add rust-src --toolchain nightly-2024-10-27-x86_64-unknown-linux-gnu +rustup target add --toolchain nightly-2024-10-27 i686-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 x86_64-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 aarch64-pc-windows-msvc ``` -Then, build the trampolines for both supported architectures: +Then, build the trampolines for all supported architectures: ```shell -cargo +nightly-2024-06-08 xwin build --xwin-arch x86 --release --target i686-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target x86_64-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target aarch64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --xwin-arch x86 --release --target i686-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target x86_64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target aarch64-pc-windows-msvc ``` ### Cross-compiling from macOS @@ -32,22 +35,25 @@ LLVM and add the `rustup` targets: ```shell brew install llvm -rustup target add i686-pc-windows-msvc -rustup target add x86_64-pc-windows-msvc -rustup target add aarch64-pc-windows-msvc +cargo install cargo-xwin +rustup toolchain install nightly-2024-10-27 +rustup component add rust-src --toolchain nightly-2024-10-27-aarch64-apple-darwin +rustup target add --toolchain nightly-2024-10-27 i686-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 x86_64-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 aarch64-pc-windows-msvc ``` -Then, build the trampolines for both supported architectures: +Then, build the trampolines for all supported architectures: ```shell -cargo +nightly-2024-06-08 xwin build --release --target i686-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target x86_64-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target aarch64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target i686-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target x86_64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target aarch64-pc-windows-msvc ``` ### Updating the prebuilt executables -After building the trampolines for both supported architectures: +After building the trampolines for all supported architectures: ```shell cp target/aarch64-pc-windows-msvc/release/uv-trampoline-console.exe trampolines/uv-trampoline-aarch64-console.exe diff --git a/crates/uv-trampoline/rust-toolchain.toml b/crates/uv-trampoline/rust-toolchain.toml index a0c842080..fce0096ac 100644 --- a/crates/uv-trampoline/rust-toolchain.toml +++ b/crates/uv-trampoline/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-06-08" +channel = "nightly-2024-10-27" diff --git a/crates/uv-trampoline/src/bounce.rs b/crates/uv-trampoline/src/bounce.rs index 22c59340a..a1cdd176d 100644 --- a/crates/uv-trampoline/src/bounce.rs +++ b/crates/uv-trampoline/src/bounce.rs @@ -33,27 +33,60 @@ use windows::Win32::{ use crate::{eprintln, format}; -const MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; const PATH_LEN_SIZE: usize = size_of::(); const MAX_PATH_LEN: u32 = 32 * 1024; -/// Transform ` ` to `python `. +/// The kind of trampoline. +enum TrampolineKind { + /// The trampoline should execute itself, it's a zipped Python script. + Script, + /// The trampoline should just execute Python, it's a proxy Python executable. + Python, +} + +impl TrampolineKind { + const fn magic_number(&self) -> &'static [u8; 4] { + match self { + Self::Script => b"UVSC", + Self::Python => b"UVPY", + } + } + + fn from_buffer(buffer: &[u8]) -> Option { + if buffer.ends_with(Self::Script.magic_number()) { + Some(Self::Script) + } else if buffer.ends_with(Self::Python.magic_number()) { + Some(Self::Python) + } else { + None + } + } +} + +/// Transform ` ` to `python ` or `python ` +/// depending on the [`TrampolineKind`]. fn make_child_cmdline() -> CString { let executable_name = std::env::current_exe().unwrap_or_else(|_| { eprintln!("Failed to get executable name"); exit_with_status(1); }); - let python_exe = find_python_exe(executable_name.as_ref()); + let (kind, python_exe) = read_trampoline_metadata(executable_name.as_ref()); let mut child_cmdline = Vec::::new(); push_quoted_path(python_exe.as_ref(), &mut child_cmdline); child_cmdline.push(b' '); - // Use the full executable name because CMD only passes the name of the executable (but not the path) - // when e.g. invoking `black` instead of `/Scripts/black` and Python then fails - // to find the file. Unfortunately, this complicates things because we now need to split the executable - // from the arguments string... - push_quoted_path(executable_name.as_ref(), &mut child_cmdline); + // Only execute the trampoline again if it's a script, otherwise, just invoke Python. + match kind { + TrampolineKind::Python => {} + TrampolineKind::Script => { + // Use the full executable name because CMD only passes the name of the executable (but not the path) + // when e.g. invoking `black` instead of `/Scripts/black` and Python then fails + // to find the file. Unfortunately, this complicates things because we now need to split the executable + // from the arguments string... + push_quoted_path(executable_name.as_ref(), &mut child_cmdline); + } + } push_arguments(&mut child_cmdline); @@ -86,17 +119,21 @@ fn push_quoted_path(path: &Path, command: &mut Vec) { command.extend(br#"""#); } -/// Reads the executable binary from the back to find the path to the Python executable that is written -/// after the ZIP file content. +/// Reads the executable binary from the back to find: +/// +/// * The path to the Python executable +/// * The kind of trampoline we are executing /// /// The executable is expected to have the following format: -/// * The file must end with the magic number 'UVUV'. +/// +/// * The file must end with the magic number 'UVPY' or 'UVSC' (identifying the trampoline kind) /// * The last 4 bytes (little endian) are the length of the path to the Python executable. /// * The path encoded as UTF-8 comes right before the length /// /// # Panics +/// /// If there's any IO error, or the file does not conform to the specified format. -fn find_python_exe(executable_name: &Path) -> PathBuf { +fn read_trampoline_metadata(executable_name: &Path) -> (TrampolineKind, PathBuf) { let mut file_handle = File::open(executable_name).unwrap_or_else(|_| { print_last_error_and_exit(&format!( "Failed to open executable '{}'", @@ -117,6 +154,7 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { let mut buffer: Vec = Vec::new(); let mut bytes_to_read = 1024.min(u32::try_from(file_size).unwrap_or(u32::MAX)); + let mut kind; let path: String = loop { // SAFETY: Casting to usize is safe because we only support 64bit systems where usize is guaranteed to be larger than u32. buffer.resize(bytes_to_read as usize, 0); @@ -135,13 +173,14 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { // Truncate the buffer to the actual number of bytes read. buffer.truncate(read_bytes); - if !buffer.ends_with(&MAGIC_NUMBER) { - eprintln!("Magic number 'UVUV' not found at the end of the file. Did you append the magic number, the length and the path to the python executable at the end of the file?"); + let Some(inner_kind) = TrampolineKind::from_buffer(&buffer) else { + eprintln!("Magic number 'UVSC' or 'UVPY' not found at the end of the file. Did you append the magic number, the length and the path to the python executable at the end of the file?"); exit_with_status(1); - } + }; + kind = inner_kind; // Remove the magic number - buffer.truncate(buffer.len() - MAGIC_NUMBER.len()); + buffer.truncate(buffer.len() - kind.magic_number().len()); let path_len = match buffer.get(buffer.len() - PATH_LEN_SIZE..) { Some(path_len) => { @@ -177,7 +216,7 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { } else { // SAFETY: Casting to u32 is safe because `path_len` is guaranteed to be less than 32KBs, // MAGIC_NUMBER is 4 bytes and PATH_LEN_SIZE is 4 bytes. - bytes_to_read = (path_len + MAGIC_NUMBER.len() + PATH_LEN_SIZE) as u32; + bytes_to_read = (path_len + kind.magic_number().len() + PATH_LEN_SIZE) as u32; if u64::from(bytes_to_read) > file_size { eprintln!("The length of the python executable path exceeds the file size. Verify that the path length is appended to the end of the launcher script as a u32 in little endian"); @@ -201,10 +240,12 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { }; // NOTICE: dunce adds 5kb~ - dunce::canonicalize(path.as_path()).unwrap_or_else(|_| { + let path = dunce::canonicalize(path.as_path()).unwrap_or_else(|_| { eprintln!("Failed to canonicalize script path"); exit_with_status(1); - }) + }); + + (kind, path) } fn push_arguments(output: &mut Vec) { diff --git a/crates/uv-trampoline/tests/harness.rs b/crates/uv-trampoline/tests/harness.rs deleted file mode 100644 index 0af6bdc04..000000000 --- a/crates/uv-trampoline/tests/harness.rs +++ /dev/null @@ -1,269 +0,0 @@ -use std::io::{Cursor, Write}; -use std::path::Path; -use std::process::Command; -use std::{env, io}; - -use anyhow::Result; -use assert_cmd::prelude::OutputAssertExt; -use assert_fs::prelude::PathChild; -use fs_err::File; -use thiserror::Error; -use which::which; -use zip::write::FileOptions; -use zip::ZipWriter; - -const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-i686-gui.exe"); - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-i686-console.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-x86_64-gui.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-x86_64-console.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-aarch64-gui.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-aarch64-console.exe"); - -/// Note: The caller is responsible for adding the path of the wheel we're installing. -#[derive(Error, Debug)] -pub enum Error { - #[error(transparent)] - Io(#[from] io::Error), - #[error( - "Unable to create Windows launcher for: {0} (only x86_64, x86, and arm64 are supported)" - )] - UnsupportedWindowsArch(&'static str), - #[error("Unable to create Windows launcher on non-Windows platform")] - NotWindows, -} - -/// Wrapper script template function -/// -/// -fn get_script_launcher(shebang: &str, is_gui: bool) -> String { - if is_gui { - format!( - r##"{shebang} -# -*- coding: utf-8 -*- -import re -import sys - -def make_gui() -> None: - from tkinter import Tk, ttk - root = Tk() - root.title("uv Test App") - frm = ttk.Frame(root, padding=10) - frm.grid() - ttk.Label(frm, text="Hello from uv-trampoline-gui.exe").grid(column=0, row=0) - root.mainloop() - -if __name__ == "__main__": - sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) - sys.exit(make_gui()) -"## - ) - } else { - format!( - r##"{shebang} -# -*- coding: utf-8 -*- -import re -import sys - -def main_console() -> None: - print("Hello from uv-trampoline-console.exe", file=sys.stdout) - print("Hello from uv-trampoline-console.exe", file=sys.stderr) - for arg in sys.argv[1:]: - print(arg, file=sys.stderr) - -if __name__ == "__main__": - sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) - sys.exit(main_console()) -"## - ) - } -} - -/// Format the shebang for a given Python executable. -/// -/// Like pip, if a shebang is non-simple (too long or contains spaces), we use `/bin/sh` as the -/// executable. -/// -/// See: -fn format_shebang(executable: impl AsRef) -> String { - // Convert the executable to a simplified path. - let executable = executable.as_ref().display().to_string(); - format!("#!{executable}") -} - -/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as -/// stored zip file. -/// -/// -#[allow(unused_variables)] -fn windows_script_launcher( - launcher_python_script: &str, - is_gui: bool, - python_executable: impl AsRef, -) -> Result, Error> { - // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain - // compilation on all platforms. - if cfg!(not(windows)) { - return Err(Error::NotWindows); - } - - let launcher_bin: &[u8] = match env::consts::ARCH { - #[cfg(all(windows, target_arch = "x86"))] - "x86" => { - if is_gui { - LAUNCHER_I686_GUI - } else { - LAUNCHER_I686_CONSOLE - } - } - #[cfg(all(windows, target_arch = "x86_64"))] - "x86_64" => { - if is_gui { - LAUNCHER_X86_64_GUI - } else { - LAUNCHER_X86_64_CONSOLE - } - } - #[cfg(all(windows, target_arch = "aarch64"))] - "aarch64" => { - if is_gui { - LAUNCHER_AARCH64_GUI - } else { - LAUNCHER_AARCH64_CONSOLE - } - } - #[cfg(windows)] - arch => { - return Err(Error::UnsupportedWindowsArch(arch)); - } - #[cfg(not(windows))] - arch => &[], - }; - - let mut payload: Vec = Vec::new(); - { - // We're using the zip writer, but with stored compression - // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 - // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 - let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); - let mut archive = ZipWriter::new(Cursor::new(&mut payload)); - let error_msg = "Writing to Vec should never fail"; - archive.start_file("__main__.py", stored).expect(error_msg); - archive - .write_all(launcher_python_script.as_bytes()) - .expect(error_msg); - archive.finish().expect(error_msg); - } - - let python = python_executable.as_ref(); - let python_path = python.display().to_string(); - - let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); - launcher.extend_from_slice(launcher_bin); - launcher.extend_from_slice(&payload); - launcher.extend_from_slice(python_path.as_bytes()); - launcher.extend_from_slice( - &u32::try_from(python_path.as_bytes().len()) - .expect("File Path to be smaller than 4GB") - .to_le_bytes(), - ); - launcher.extend_from_slice(&LAUNCHER_MAGIC_NUMBER); - - Ok(launcher) -} - -#[test] -fn generate_console_launcher() -> Result<()> { - // Create Temp Dirs - let temp_dir = assert_fs::TempDir::new()?; - let console_bin_path = temp_dir.child("launcher.console.exe"); - - // Locate an arbitrary python installation from PATH - let python_executable_path = which("python")?; - - // Generate Launcher Script - let launcher_console_script = - get_script_launcher(&format_shebang(&python_executable_path), false); - - // Generate Launcher Payload - let console_launcher = - windows_script_launcher(&launcher_console_script, false, &python_executable_path)?; - - // Create Launcher - File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; - - println!( - "Wrote Console Launcher in {}", - console_bin_path.path().display() - ); - - let stdout_predicate = "Hello from uv-trampoline-console.exe\r\n"; - let stderr_predicate = "Hello from uv-trampoline-console.exe\r\n"; - - // Test Console Launcher - #[cfg(windows)] - Command::new(console_bin_path.path()) - .assert() - .success() - .stdout(stdout_predicate) - .stderr(stderr_predicate); - - let args_to_test = vec!["foo", "bar", "foo bar", "foo \"bar\"", "foo 'bar'"]; - let stderr_predicate = format!("{}{}\r\n", stderr_predicate, args_to_test.join("\r\n")); - - // Test Console Launcher (with args) - #[cfg(windows)] - Command::new(console_bin_path.path()) - .args(args_to_test) - .assert() - .success() - .stdout(stdout_predicate) - .stderr(stderr_predicate); - - Ok(()) -} - -#[test] -#[ignore] -fn generate_gui_launcher() -> Result<()> { - // Create Temp Dirs - let temp_dir = assert_fs::TempDir::new()?; - let gui_bin_path = temp_dir.child("launcher.gui.exe"); - - // Locate an arbitrary pythonw installation from PATH - let pythonw_executable_path = which("pythonw")?; - - // Generate Launcher Script - let launcher_gui_script = get_script_launcher(&format_shebang(&pythonw_executable_path), true); - - // Generate Launcher Payload - let gui_launcher = - windows_script_launcher(&launcher_gui_script, true, &pythonw_executable_path)?; - - // Create Launcher - File::create(gui_bin_path.path())?.write_all(gui_launcher.as_ref())?; - - println!("Wrote GUI Launcher in {}", gui_bin_path.path().display()); - - // Test GUI Launcher - // NOTICE: This will spawn a GUI and will wait until you close the window. - #[cfg(windows)] - Command::new(gui_bin_path.path()).assert().success(); - - Ok(()) -} diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe index 5b6ea9cab..773c2f9ba 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe index 1f6524111..f0fe2a3bb 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe index 3f0bed6a1..ee1e70e71 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe index 936aedb5e..6edd2f511 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe index 8ba2f2b5a..f437f03f0 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe index 9f503fe51..39069c743 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe differ