From 42973cd9cb098e59f75e287d72a425af8a614e91 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:46:58 +0000 Subject: [PATCH] feat: add linehaul info to uv-client (#2493) ## Summary Closes #1958 This adds linehaul metadata to uv's user-agent when pep 508 markers are provided to the RegistryClientBuilder. Thanks to #2381, we were able to leverage most information from markers and avoid inconsistency. Linehaul is meant to be accompanying metadata pip sends in it's user agent when talking to registries. You can see this output by running something like `python -c 'from pip._internal.network.session import user_agent; print(user_agent())'`. In PyPI, this metadata processed by the [linehaul-cloud-function](https://github.com/pypi/linehaul-cloud-function). More info about linehaul can be found in #1958. Below are some examples from pip: * Linux GHA: `pip/24.0 {"ci":true,"cpu":"x86_64","distro":{"id":"jammy","libc":{"lib":"glibc","version":"2.35"},"name":"Ubuntu","version":"22.04"},"implementation":{"name":"CPython","version":"3.12.2"},"installer":{"name":"pip","version":"24.0"},"openssl_version":"OpenSSL 3.0.2 15 Mar 2022","python":"3.12.2","rustc_version":"1.76.0","system":{"name":"Linux","release":"6.5.0-1016-azure"}}` * Windows GHA: `pip/24.0 {"ci":true,"cpu":"AMD64","implementation":{"name":"CPython","version":"3.12.2"},"installer":{"name":"pip","version":"24.0"},"openssl_version":"OpenSSL 3.0.13 30 Jan 2024","python":"3.12.2","rustc_version":"1.76.0","system":{"name":"Windows","release":"2022Server"}}` * OSX GHA: `pip/24.0 {"ci":true,"cpu":"arm64","distro":{"name":"macOS","version":"14.2.1"},"implementation":{"name":"CPython","version":"3.12.2"},"installer":{"name":"pip","version":"24.0"},"openssl_version":"OpenSSL 3.0.13 30 Jan 2024","python":"3.12.2","rustc_version":"1.76.0","system":{"name":"Darwin","release":"23.2.0"}}` Here's how uv results look like (sorry for the keys not having the same order): * Linux GHA: `uv/0.1.21 {"installer":{"name":"uv","version":"0.1.21"},"python":"3.12.2","implementation":{"name":"CPython","version":"3.12.2"},"distro":{"name":"Ubuntu","version":"22.04","id":"jammy","libc":null},"system":{"name":"Linux","release":"6.5.0-1016-azure"},"cpu":"x86_64","openssl_version":null,"setuptools_version":null,"rustc_version":null,"ci":true}` * Windows GHA: `uv/0.1.21 {"installer":{"name":"uv","version":"0.1.21"},"python":"3.12.2","implementation":{"name":"CPython","version":"3.12.2"},"distro":null,"system":{"name":"Windows","release":"2022Server"},"cpu":"AMD64","openssl_version":null,"setuptools_version":null,"rustc_version":null,"ci":true}` * OSX GHA: `uv/0.1.21 {"installer":{"name":"uv","version":"0.1.21"},"python":"3.12.2","implementation":{"name":"CPython","version":"3.12.2"},"distro":{"name":"macOS","version":"14.2.1","id":null,"libc":null},"system":{"name":"Darwin","release":"23.2.0"},"cpu":"arm64","openssl_version":null,"setuptools_version":null,"rustc_version":null,"ci":true}` Distro information (such as the one pip uses `from pip._vendor import distro` to retrieve instead of `platform` module) was not retrieved from markers. Instead, the linux release codename/name/version uses `sys-info` crate, adding about 50us of extra overhead on linux. The distro osx version re-used the [mac_os version implementation](https://github.com/astral-sh/uv/blob/99c992e38b220fbcda09b0b43602b3db2321480b/crates/platform-host/src/mac_os.rs) from #2381 which adds about 20us of overhead on osx. I tried to use other crates to avoid re-introducing `mac_os.rs` but most of them didn't yield satisfactory performance (40ms-60ms~) or had the wrong values needed (e.g. darwin version vs osx version). I also didn't add libc retrieval or rustc retrieval as those seem to add substantial overhead due to querying `ldd` or `rustc`. PyPy version detection was also not added to avoid adding extra overhead to [support PyPy for linehaul](https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L123). All other behavior was kept 1-1 to match what pip's linehaul implementation does (as of 24.0). This also aligns with what was discussed in #1958. ## Test Plan Added new integration test to uv-client. --------- Co-authored-by: konstin --- Cargo.lock | 114 ++++++++++++++ Cargo.toml | 2 + crates/uv-client/Cargo.toml | 4 + crates/uv-client/src/base_client.rs | 35 ++++- crates/uv-client/src/lib.rs | 3 + crates/uv-client/src/linehaul.rs | 134 ++++++++++++++++ crates/uv-client/src/mac_version.rs | 36 +++++ crates/uv-client/src/registry_client.rs | 32 +++- crates/uv-client/tests/user_agent_version.rs | 155 ++++++++++++++++++- crates/uv/src/commands/pip_compile.rs | 2 + crates/uv/src/commands/pip_install.rs | 1 + crates/uv/src/commands/pip_sync.rs | 1 + crates/uv/src/commands/venv.rs | 1 + 13 files changed, 512 insertions(+), 8 deletions(-) create mode 100644 crates/uv-client/src/linehaul.rs create mode 100644 crates/uv-client/src/mac_version.rs diff --git a/Cargo.lock b/Cargo.lock index 064164b79..1afb21f54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,6 +885,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1956,6 +1965,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2168,6 +2186,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.18" @@ -2256,6 +2280,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_info" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" +dependencies = [ + "log", + "winapi", +] + [[package]] name = "overload" version = "0.1.1" @@ -2459,6 +2493,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "plist" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" +dependencies = [ + "base64 0.21.7", + "indexmap 2.2.5", + "line-wrap", + "quick-xml", + "serde", + "time", +] + [[package]] name = "png" version = "0.17.13" @@ -2487,6 +2535,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2684,6 +2738,15 @@ dependencies = [ "toml", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.35" @@ -3210,6 +3273,12 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + [[package]] name = "same-file" version = "1.0.6" @@ -3547,6 +3616,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3726,6 +3805,37 @@ dependencies = [ "tikv-jemalloc-sys", ] +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.8.4" @@ -4361,9 +4471,11 @@ dependencies = [ "hyper 0.14.28", "insta", "install-wheel-rs", + "os_info", "pep440_rs", "pep508_rs", "platform-tags", + "plist", "pypi-types", "reqwest", "reqwest-middleware", @@ -4376,6 +4488,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sys-info", "task-local-extensions", "tempfile", "thiserror", @@ -4388,6 +4501,7 @@ dependencies = [ "uv-auth", "uv-cache", "uv-fs", + "uv-interpreter", "uv-normalize", "uv-version", "uv-warnings", diff --git a/Cargo.toml b/Cargo.toml index e9f06c4d0..e2d920629 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ owo-colors = { version = "4.0.0" } pathdiff = { version = "0.2.1" } petgraph = { version = "0.6.4" } platform-info = { version = "2.0.2" } +plist = { version = "1.6.0" } pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "addbaf184891d66a2dfd93d241a66d13bfe5de86" } pyo3 = { version = "0.20.3" } pyo3-log = { version = "0.9.0" } @@ -89,6 +90,7 @@ serde = { version = "1.0.197" } serde_json = { version = "1.0.114" } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } +sys-info = { version = "0.9.1" } target-lexicon = { version = "0.12.14" } task-local-extensions = { version = "0.1.4" } tempfile = { version = "3.9.0" } diff --git a/crates/uv-client/Cargo.toml b/crates/uv-client/Cargo.toml index c4ed0ca07..2481c0d6f 100644 --- a/crates/uv-client/Cargo.toml +++ b/crates/uv-client/Cargo.toml @@ -14,6 +14,7 @@ platform-tags = { path = "../platform-tags" } uv-auth = { path = "../uv-auth" } uv-cache = { path = "../uv-cache" } uv-fs = { path = "../uv-fs", features = ["tokio"] } +uv-interpreter = { path = "../uv-interpreter" } uv-normalize = { path = "../uv-normalize" } uv-version = { path = "../uv-version" } uv-warnings = { path = "../uv-warnings" } @@ -28,6 +29,7 @@ fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } html-escape = { workspace = true } http = { workspace = true } +plist = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } @@ -37,6 +39,7 @@ rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } +sys-info = { workspace = true } task-local-extensions = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } @@ -56,4 +59,5 @@ webpki-roots = { version = "0.25.4" } anyhow = { workspace = true } hyper = { version = "0.14.28", features = ["server", "http1"] } insta = { version = "1.36.1" } +os_info = { version = "3.7.0", default-features = false } tokio = { workspace = true, features = ["fs", "macros"] } diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index b0fab037f..1420626de 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -1,3 +1,5 @@ +use pep508_rs::MarkerEnvironment; +use platform_tags::Platform; use reqwest::{Client, ClientBuilder}; use reqwest_middleware::ClientWithMiddleware; use reqwest_retry::policies::ExponentialBackoff; @@ -12,21 +14,24 @@ use uv_fs::Simplified; use uv_version::version; use uv_warnings::warn_user_once; +use crate::linehaul::LineHaul; use crate::middleware::OfflineMiddleware; use crate::tls::Roots; use crate::{tls, Connectivity}; /// A builder for an [`RegistryClient`]. #[derive(Debug, Clone)] -pub struct BaseClientBuilder { +pub struct BaseClientBuilder<'a> { keyring_provider: KeyringProvider, native_tls: bool, retries: u32, connectivity: Connectivity, client: Option, + markers: Option<&'a MarkerEnvironment>, + platform: Option<&'a Platform>, } -impl BaseClientBuilder { +impl BaseClientBuilder<'_> { pub fn new() -> Self { Self { keyring_provider: KeyringProvider::default(), @@ -34,11 +39,13 @@ impl BaseClientBuilder { connectivity: Connectivity::Online, retries: 3, client: None, + markers: None, + platform: None, } } } -impl BaseClientBuilder { +impl<'a> BaseClientBuilder<'a> { #[must_use] pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self { self.keyring_provider = keyring_provider; @@ -69,9 +76,29 @@ impl BaseClientBuilder { self } + #[must_use] + pub fn markers(mut self, markers: &'a MarkerEnvironment) -> Self { + self.markers = Some(markers); + self + } + + #[must_use] + pub fn platform(mut self, platform: &'a Platform) -> Self { + self.platform = Some(platform); + self + } + pub fn build(self) -> BaseClient { // Create user agent. - let user_agent_string = format!("uv/{}", version()); + let mut user_agent_string = format!("uv/{}", version()); + + // Add linehaul metadata. + if let Some(markers) = self.markers { + let linehaul = LineHaul::new(markers, self.platform); + if let Ok(output) = serde_json::to_string(&linehaul) { + user_agent_string += &format!(" {}", output); + } + } // Timeout options, matching https://doc.rust-lang.org/nightly/cargo/reference/config.html#httptimeout // `UV_REQUEST_TIMEOUT` is provided for backwards compatibility with v0.1.6 diff --git a/crates/uv-client/src/lib.rs b/crates/uv-client/src/lib.rs index c74b92ac5..41c2bc92e 100644 --- a/crates/uv-client/src/lib.rs +++ b/crates/uv-client/src/lib.rs @@ -2,6 +2,7 @@ pub use base_client::BaseClient; pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy}; pub use error::{BetterReqwestError, Error, ErrorKind}; pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError}; +pub use linehaul::LineHaul; pub use registry_client::{ Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum, VersionFiles, @@ -14,6 +15,8 @@ mod error; mod flat_index; mod html; mod httpcache; +mod linehaul; +mod mac_version; mod middleware; mod registry_client; mod remote_metadata; diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs new file mode 100644 index 000000000..415333ffd --- /dev/null +++ b/crates/uv-client/src/linehaul.rs @@ -0,0 +1,134 @@ +use std::env; + +use serde::{Deserialize, Serialize}; +use tracing::instrument; + +use pep508_rs::MarkerEnvironment; +use platform_tags::{Os, Platform}; +use uv_version::version; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Installer { + pub name: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Implementation { + pub name: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Libc { + pub lib: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Distro { + pub name: Option, + pub version: Option, + pub id: Option, + pub libc: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct System { + pub name: Option, + pub release: Option, +} + +/// Linehaul structs were derived from +/// . +/// For the sake of parity, the nullability of all the values was kept intact. +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct LineHaul { + pub installer: Option, + pub python: Option, + pub implementation: Option, + pub distro: Option, + pub system: Option, + pub cpu: Option, + pub openssl_version: Option, + pub setuptools_version: Option, + pub rustc_version: Option, + pub ci: Option, +} + +/// Implements Linehaul information format as defined by +/// . +/// This metadata is added to the user agent to enrich PyPI statistics. +impl LineHaul { + /// Initializes Linehaul information based on PEP 508 markers. + #[instrument(name = "linehaul", skip_all)] + pub fn new(markers: &MarkerEnvironment, platform: Option<&Platform>) -> Self { + // https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L87 + let looks_like_ci = ["BUILD_BUILDID", "BUILD_ID", "CI", "PIP_IS_CI"] + .iter() + .find_map(|&var_name| env::var(var_name).ok().map(|_| true)); + + let libc = match platform.map(|platform| platform.os()) { + Some(Os::Manylinux { major, minor }) => Some(Libc { + lib: Some("glibc".to_string()), + version: Some(format!("{major}.{minor}")), + }), + Some(Os::Musllinux { major, minor }) => Some(Libc { + lib: Some("musl".to_string()), + version: Some(format!("{major}.{minor}")), + }), + _ => None, + }; + + // Build Distro as Linehaul expects. + let distro: Option = if cfg!(target_os = "linux") { + // Gather distribution info from /etc/os-release. + sys_info::linux_os_release().ok().map(|info| Distro { + // e.g., Jammy, Focal, etc. + id: info.version_codename, + // e.g., Ubuntu, Fedora, etc. + name: info.name, + // e.g., 22.04, etc. + version: info.version_id, + // e.g., glibc 2.38, musl 1.2 + libc, + }) + } else if cfg!(target_os = "macos") { + Some(Distro { + // N/A + id: None, + // pip hardcodes distro name to macOS. + name: Some("macOS".to_string()), + // Same as python's platform.mac_ver[0]. + version: crate::mac_version::get_mac_os_version().ok(), + // N/A + libc: None, + }) + } else { + // Always empty on Windows. + None + }; + + Self { + installer: Option::from(Installer { + name: Some("uv".to_string()), + version: Some(version().to_string()), + }), + python: Some(markers.python_full_version.version.to_string()), + implementation: Option::from(Implementation { + name: Some(markers.platform_python_implementation.to_string()), + version: Some(markers.python_full_version.version.to_string()), + }), + distro, + system: Option::from(System { + name: Some(markers.platform_system.to_string()), + release: Some(markers.platform_release.to_string()), + }), + cpu: Some(markers.platform_machine.to_string()), + openssl_version: None, // Should probably always be None in uv. + setuptools_version: None, // Should probably always be None in uv. + rustc_version: None, // Calling rustc --version is likely too slow. + ci: looks_like_ci, + } + } +} diff --git a/crates/uv-client/src/mac_version.rs b/crates/uv-client/src/mac_version.rs new file mode 100644 index 000000000..33558b730 --- /dev/null +++ b/crates/uv-client/src/mac_version.rs @@ -0,0 +1,36 @@ +use platform_tags::PlatformError; +use serde::Deserialize; + +/// Get the macOS version from the SystemVersion.plist file. +pub(crate) fn get_mac_os_version() -> Result { + // This is actually what python does + // https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428 + #[derive(Deserialize)] + #[serde(rename_all = "PascalCase")] + struct SystemVersion { + product_version: String, + } + let system_version: SystemVersion = + plist::from_file("/System/Library/CoreServices/SystemVersion.plist") + .map_err(|err| PlatformError::OsVersionDetectionError(err.to_string()))?; + + let invalid_mac_os_version = || { + PlatformError::OsVersionDetectionError(format!( + "Invalid macOS version {}", + system_version.product_version + )) + }; + match system_version + .product_version + .split('.') + .collect::>() + .as_slice() + { + [major, minor] | [major, minor, _] => { + let _major = major.parse::().map_err(|_| invalid_mac_os_version())?; + let _minor = minor.parse::().map_err(|_| invalid_mac_os_version())?; + Ok(system_version.product_version) + } + _ => Err(invalid_mac_os_version()), + } +} diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 23e500bb3..e448f4031 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -17,6 +17,8 @@ use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use distribution_types::{BuiltDist, File, FileLocation, IndexUrl, IndexUrls, Name}; use install_wheel_rs::metadata::{find_archive_dist_info, is_metadata_entry}; use pep440_rs::Version; +use pep508_rs::MarkerEnvironment; +use platform_tags::Platform; use pypi_types::{Metadata23, SimpleJson}; use uv_auth::KeyringProvider; use uv_cache::{Cache, CacheBucket, WheelCache}; @@ -31,7 +33,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind}; /// A builder for an [`RegistryClient`]. #[derive(Debug, Clone)] -pub struct RegistryClientBuilder { +pub struct RegistryClientBuilder<'a> { index_urls: IndexUrls, keyring_provider: KeyringProvider, native_tls: bool, @@ -39,9 +41,11 @@ pub struct RegistryClientBuilder { connectivity: Connectivity, cache: Cache, client: Option, + markers: Option<&'a MarkerEnvironment>, + platform: Option<&'a Platform>, } -impl RegistryClientBuilder { +impl RegistryClientBuilder<'_> { pub fn new(cache: Cache) -> Self { Self { index_urls: IndexUrls::default(), @@ -51,11 +55,13 @@ impl RegistryClientBuilder { connectivity: Connectivity::Online, retries: 3, client: None, + markers: None, + platform: None, } } } -impl RegistryClientBuilder { +impl<'a> RegistryClientBuilder<'a> { #[must_use] pub fn index_urls(mut self, index_urls: IndexUrls) -> Self { self.index_urls = index_urls; @@ -98,6 +104,18 @@ impl RegistryClientBuilder { self } + #[must_use] + pub fn markers(mut self, markers: &'a MarkerEnvironment) -> Self { + self.markers = Some(markers); + self + } + + #[must_use] + pub fn platform(mut self, platform: &'a Platform) -> Self { + self.platform = Some(platform); + self + } + pub fn build(self) -> RegistryClient { // Build a base client let mut builder = BaseClientBuilder::new(); @@ -106,6 +124,14 @@ impl RegistryClientBuilder { builder = builder.client(client) } + if let Some(markers) = self.markers { + builder = builder.markers(markers) + } + + if let Some(platform) = self.platform { + builder = builder.platform(platform) + } + let client = builder .retries(self.retries) .connectivity(self.connectivity) diff --git a/crates/uv-client/tests/user_agent_version.rs b/crates/uv-client/tests/user_agent_version.rs index 68d7eefe6..ea4fb7d95 100644 --- a/crates/uv-client/tests/user_agent_version.rs +++ b/crates/uv-client/tests/user_agent_version.rs @@ -4,9 +4,11 @@ use hyper::header::USER_AGENT; use hyper::server::conn::Http; use hyper::service::service_fn; use hyper::{Body, Request, Response}; +use pep508_rs::{MarkerEnvironment, StringVersion}; +use platform_tags::{Arch, Os, Platform}; use tokio::net::TcpListener; - use uv_cache::Cache; +use uv_client::LineHaul; use uv_client::RegistryClientBuilder; use uv_version::version; @@ -60,3 +62,154 @@ async fn test_user_agent_has_version() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_user_agent_has_linehaul() -> Result<()> { + // Set up the TCP listener on a random available port + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + + // Spawn the server loop in a background task + tokio::spawn(async move { + let svc = service_fn(move |req: Request| { + // Get User Agent Header and send it back in the response + let user_agent = req + .headers() + .get(USER_AGENT) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + .unwrap_or_default(); // Empty Default + future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent))) + }); + // Start Hyper Server + let (socket, _) = listener.accept().await.unwrap(); + Http::new() + .http1_keep_alive(false) + .serve_connection(socket, svc) + .with_upgrades() + .await + .expect("Server Started"); + }); + + // Add some representative markers for an Ubuntu CI runner + let markers = MarkerEnvironment { + implementation_name: "cpython".to_string(), + implementation_version: StringVersion { + string: "3.12.2".to_string(), + version: "3.12.2".parse()?, + }, + os_name: "posix".to_string(), + platform_machine: "x86_64".to_string(), + platform_python_implementation: "CPython".to_string(), + platform_release: "6.5.0-1016-azure".to_string(), + platform_system: "Linux".to_string(), + platform_version: "#16~22.04.1-Ubuntu SMP Fri Feb 16 15:42:02 UTC 2024".to_string(), + python_full_version: StringVersion { + string: "3.12.2".to_string(), + version: "3.12.2".parse()?, + }, + python_version: StringVersion { + string: "3.12".to_string(), + version: "3.12".parse()?, + }, + sys_platform: "linux".to_string(), + }; + // Linux only + let platform = Platform::new( + Os::Manylinux { + major: 2, + minor: 38, + }, + Arch::X86_64, + ); + + // Initialize uv-client + let cache = Cache::temp()?; + let mut builder = RegistryClientBuilder::new(cache).markers(&markers); + + if cfg!(target_os = "linux") { + builder = builder.platform(&platform); + } + let client = builder.build(); + + // Send request to our dummy server + let res = client + .uncached_client() + .get(format!("http://{addr}")) + .send() + .await?; + + // Check the HTTP status + assert!(res.status().is_success()); + + // Check User Agent + let body = res.text().await?; + + // Unpack User-Agent with linehaul + let (uv_version, uv_linehaul) = body + .split_once(' ') + .expect("Failed to split User-Agent header."); + + // Deserializing Linehaul + let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?; + + // Assert uv version + assert_eq!(uv_version, format!("uv/{}", version())); + + // Assert linehaul + let installer_info = linehaul.installer.unwrap(); + let system_info = linehaul.system.unwrap(); + let impl_info = linehaul.implementation.unwrap(); + + assert_eq!(installer_info.name.unwrap(), "uv".to_string()); + assert_eq!(installer_info.version.unwrap(), version()); + + assert_eq!(system_info.name.unwrap(), markers.platform_system); + assert_eq!(system_info.release.unwrap(), markers.platform_release); + + assert_eq!( + impl_info.name.unwrap(), + markers.platform_python_implementation + ); + assert_eq!( + impl_info.version.unwrap(), + markers.python_full_version.version.to_string() + ); + + assert_eq!( + linehaul.python.unwrap(), + markers.python_full_version.version.to_string() + ); + assert_eq!(linehaul.cpu.unwrap(), markers.platform_machine); + + assert_eq!(linehaul.openssl_version, None); + assert_eq!(linehaul.setuptools_version, None); + assert_eq!(linehaul.rustc_version, None); + + #[cfg(windows)] + assert_eq!(linehaul.distro, None); + + // Using os_info as to confirm our values are as expected in both Linux and OSX. + #[cfg(target_os = "linux")] + { + let info = os_info::get(); + let distro_info = linehaul.distro.unwrap(); + assert_eq!(distro_info.id.unwrap(), info.codename().unwrap()); + assert_eq!(distro_info.name.unwrap(), info.os_type().to_string()); + assert_eq!(distro_info.version.unwrap(), info.version().to_string()); + assert!(distro_info.libc.is_some()); + } + + // Using os_info as sys-info yields Darwin version, and not mac release version. + #[cfg(target_os = "macos")] + { + let info = os_info::get(); + let distro_info = linehaul.distro.unwrap(); + assert_eq!(distro_info.id, None); + assert_eq!(distro_info.name.unwrap(), "macOS"); + assert_eq!(distro_info.version.unwrap(), info.version().to_string()); + assert_eq!(distro_info.libc, None); + } + + Ok(()) +} diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index d30e2b2bc..8afd94c85 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -198,6 +198,8 @@ pub(crate) async fn pip_compile( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(&markers) + .platform(interpreter.platform()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 0e7ed8600..5effd07a0 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -192,6 +192,7 @@ pub(crate) async fn pip_install( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(markers) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index 6a4b5d2be..714ea20d4 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -126,6 +126,7 @@ pub(crate) async fn pip_sync( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(venv.interpreter().markers()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index caa3e63d8..c1045e50f 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -151,6 +151,7 @@ async fn venv_impl( .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) .connectivity(connectivity) + .markers(interpreter.markers()) .build(); // Resolve the flat indexes from `--find-links`.