From ffb45112abbee281d09a983f6817473be651ee85 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 23 Jan 2026 10:29:39 -0500 Subject: [PATCH] PEP 792: plumb statuses into internal representation (#17631) --- Cargo.lock | 26 +-- Cargo.toml | 2 +- crates/uv-cache/src/lib.rs | 4 +- crates/uv-client/src/registry_client.rs | 251 +++++++++++++++++++-- crates/uv-client/src/rkyvutil.rs | 4 +- crates/uv-pypi-types/src/project_status.rs | 8 +- crates/uv-pypi-types/src/simple_json.rs | 3 + crates/uv-resolver/src/version_map.rs | 8 +- crates/uv/tests/it/cache_clean.rs | 4 +- 9 files changed, 270 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0fd11ba98..edd46aa40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -920,7 +920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1499,7 +1499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2095,7 +2095,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.0", "system-configuration", "tokio", "tower-service", @@ -2346,7 +2346,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2416,7 +2416,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -3518,7 +3518,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3958,9 +3958,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2e88acca7157d83d789836a3987dafc12bc3d88a050e54b8fe9ea4aaa29d20" +checksum = "360b333c61ae24e5af3ae7c8660bd6b21ccd8200dbbc5d33c2454421e85b9c69" dependencies = [ "bytecheck", "bytes", @@ -3978,9 +3978,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6dffea3c91fa91a3c0fc8a061b0e27fef25c6304728038a6d6bcb1c58ba9bd" +checksum = "7c02f8cdd12b307ab69fe0acf4cd2249c7460eb89dce64a0febadf934ebb6a9e" dependencies = [ "proc-macro2", "quote", @@ -4089,7 +4089,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -4849,7 +4849,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -7490,7 +7490,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1f2308fce..39701fc85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ reqsign = { version = "0.18.1", features = ["aws", "google", "default-context"], reqwest = { version = "0.12.22", default-features = false, features = ["json", "gzip", "deflate", "zstd", "stream", "system-proxy", "rustls-tls", "rustls-tls-native-roots", "socks", "multipart", "http2", "blocking"] } reqwest-middleware = { version = "0.4.2", package = "astral-reqwest-middleware", features = ["multipart"] } reqwest-retry = { version = "0.8.0", package = "astral-reqwest-retry" } -rkyv = { version = "0.8.8", features = ["bytecheck"] } +rkyv = { version = "0.8.14", features = ["bytecheck"] } rmp-serde = { version = "1.3.0" } rust-netrc = { version = "0.1.2" } rustc-hash = { version = "2.0.0" } diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 252f42e12..78e7f3922 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -1145,7 +1145,7 @@ pub enum CacheBucket { /// * `simple-v0/pypi/.rkyv` /// * `simple-v0//.rkyv` /// - /// The response is parsed into `uv_client::SimpleMetadata` before storage. + /// The response is parsed into `uv_client::SimpleDetailMetadata` before storage. Simple, /// A cache of unzipped wheels, stored as directories. This is used internally within the cache. /// When other buckets need to store directories, they should persist them to @@ -1174,7 +1174,7 @@ impl CacheBucket { Self::Interpreter => "interpreter-v4", // Note that when bumping this, you'll also need to bump it // in `crates/uv/tests/it/cache_clean.rs`. - Self::Simple => "simple-v18", + Self::Simple => "simple-v19", // Note that when bumping this, you'll also need to bump it // in `crates/uv/tests/it/cache_prune.rs`. Self::Wheels => "wheels-v5", diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index a1378aa1c..cd42cdcef 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -29,6 +29,7 @@ use uv_normalize::PackageName; use uv_pep440::Version; use uv_pep508::MarkerEnvironment; use uv_platform_tags::Platform; +use uv_pypi_types::ProjectStatus; use uv_pypi_types::{ PypiSimpleDetail, PypiSimpleIndex, PyxSimpleDetail, PyxSimpleIndex, ResolutionMetadata, }; @@ -643,6 +644,7 @@ impl RegistryClient { data.files, data.core_metadata, package_name, + data.project_status, &url, ) } @@ -658,6 +660,7 @@ impl RegistryClient { data.files, data.core_metadata, package_name, + data.project_status, &url, ) } @@ -670,7 +673,12 @@ impl RegistryClient { let data: PypiSimpleDetail = serde_json::from_slice(bytes.as_ref()) .map_err(|err| Error::from_json_err(err, url.clone()))?; - SimpleDetailMetadata::from_pypi_files(data.files, package_name, &url) + SimpleDetailMetadata::from_pypi_files( + data.files, + package_name, + data.project_status, + &url, + ) } MediaType::PypiV1Html | MediaType::TextHtml => { let text = response @@ -1369,9 +1377,15 @@ impl SimpleIndexMetadata { } } +/// Detail response for a Python package from a Simple API index. +/// +/// Abstracts over both HTML and JSON index formats. #[derive(Default, Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] #[rkyv(derive(Debug))] -pub struct SimpleDetailMetadata(Vec); +pub struct SimpleDetailMetadata { + project_status: ProjectStatus, + versions: Vec, +} #[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] #[rkyv(derive(Debug))] @@ -1383,12 +1397,13 @@ pub struct SimpleDetailMetadatum { impl SimpleDetailMetadata { pub fn iter(&self) -> impl DoubleEndedIterator { - self.0.iter() + self.versions.iter() } fn from_pypi_files( files: Vec, package_name: &PackageName, + project_status: ProjectStatus, base: &Url, ) -> Self { let mut version_map: BTreeMap = BTreeMap::default(); @@ -1423,8 +1438,8 @@ impl SimpleDetailMetadata { } } - Self( - version_map + Self { + versions: version_map .into_iter() .map(|(version, files)| SimpleDetailMetadatum { version, @@ -1432,13 +1447,15 @@ impl SimpleDetailMetadata { metadata: None, }) .collect(), - ) + project_status, + } } fn from_pyx_files( files: Vec, mut core_metadata: FxHashMap, package_name: &PackageName, + project_status: ProjectStatus, base: &Url, ) -> Self { let mut version_map: BTreeMap = BTreeMap::default(); @@ -1473,8 +1490,8 @@ impl SimpleDetailMetadata { } } - Self( - version_map + Self { + versions: version_map .into_iter() .map(|(version, files)| { let metadata = @@ -1495,7 +1512,8 @@ impl SimpleDetailMetadata { } }) .collect(), - ) + project_status, + } } /// Read the [`SimpleDetailMetadata`] from an HTML index. @@ -1505,13 +1523,18 @@ impl SimpleDetailMetadata { url: &DisplaySafeUrl, ) -> Result { let SimpleDetailHTML { - project_status: _, + project_status, base, files, } = SimpleDetailHTML::parse(text, url) .map_err(|err| Error::from_html_err(err, url.clone()))?; - Ok(Self::from_pypi_files(files, package_name, base.as_url())) + Ok(Self::from_pypi_files( + files, + package_name, + project_status, + base.as_url(), + )) } } @@ -1520,17 +1543,17 @@ impl IntoIterator for SimpleDetailMetadata { type IntoIter = std::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + self.versions.into_iter() } } impl ArchivedSimpleDetailMetadata { pub fn iter(&self) -> impl DoubleEndedIterator> { - self.0.iter() + self.versions.iter() } pub fn datum(&self, i: usize) -> Option<&rkyv::Archived> { - self.0.get(i) + self.versions.get(i) } } @@ -1835,6 +1858,7 @@ mod tests { let simple_metadata = SimpleDetailMetadata::from_pypi_files( data.files, &PackageName::from_str("pyflyby").unwrap(), + data.project_status, &base, ); let versions: Vec = simple_metadata @@ -1844,6 +1868,205 @@ mod tests { assert_eq!(versions, ["1.7.8".to_string()]); } + /// Test for project statuses from PyPI's JSON detail response. + #[test] + fn project_status_pypi_json() { + // Minimized from https://pypi.org/simple/pepy/ + let json = r#" + { + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "pepy-2.1.1.tar.gz", + "hashes": { + "sha256": "cec463c444b71d1664229121897b22df753dc91fabb2113d1c89992638c90829" + }, + "provenance": null, + "requires-python": ">=3.7", + "size": 15399, + "upload-time": "2022-11-14T17:14:53.935145Z", + "url": "https://files.pythonhosted.org/packages/78/7e/123d89ce0e999e957e53f0b985f734565c93b9a698af53586fc2a1be0dbf/pepy-2.1.1.tar.gz", + "yanked": false + } + ], + "meta": { + "_last-serial": 15765070, + "api-version": "1.4" + }, + "name": "pepy", + "project-status": { + "status": "archived" + }, + "versions": [ + "2.1.1" + ] + } + "#; + + let data: PypiSimpleDetail = serde_json::from_str(json).unwrap(); + let base = DisplaySafeUrl::parse("https://pypi.org/simple/pepy/").unwrap(); + let simple_metadata = SimpleDetailMetadata::from_pypi_files( + data.files, + &PackageName::from_str("pepy").unwrap(), + data.project_status, + &base, + ); + + insta::assert_debug_snapshot!(simple_metadata, @r#" + SimpleDetailMetadata { + project_status: ProjectStatus { + status: Archived, + reason: None, + }, + versions: [ + SimpleDetailMetadatum { + version: "2.1.1", + files: VersionFiles { + wheels: [], + source_dists: [ + VersionSourceDist { + name: SourceDistFilename { + name: PackageName( + "pepy", + ), + version: "2.1.1", + extension: TarGz, + }, + file: File { + dist_info_metadata: false, + filename: "pepy-2.1.1.tar.gz", + hashes: HashDigests( + [ + HashDigest { + algorithm: Sha256, + digest: "cec463c444b71d1664229121897b22df753dc91fabb2113d1c89992638c90829", + }, + ], + ), + requires_python: Some( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: "3.7", + }, + ], + ), + ), + size: Some( + 15399, + ), + upload_time_utc_ms: Some( + 1668446093935, + ), + url: AbsoluteUrl( + UrlString( + "https://files.pythonhosted.org/packages/78/7e/123d89ce0e999e957e53f0b985f734565c93b9a698af53586fc2a1be0dbf/pepy-2.1.1.tar.gz", + ), + ), + yanked: Some( + Bool( + false, + ), + ), + zstd: None, + }, + }, + ], + }, + metadata: None, + }, + ], + } + "#); + } + + /// Test for project statuses from PyPI's HTML detail response. + #[test] + fn project_status_pypi_html() { + // Minimized from https://pypi.org/simple/pepy/ + let html = r#" + + + + + Links for pepy + + +

Links for pepy

+ pepy-2.1.1.tar.gz
+ + + + "#; + + let base = DisplaySafeUrl::parse("https://pypi.org/simple/pepy/").unwrap(); + let simple_metadata = + SimpleDetailMetadata::from_html(html, &PackageName::from_str("pepy").unwrap(), &base) + .unwrap(); + insta::assert_debug_snapshot!(simple_metadata, @r#" + SimpleDetailMetadata { + project_status: ProjectStatus { + status: Archived, + reason: None, + }, + versions: [ + SimpleDetailMetadatum { + version: "2.1.1", + files: VersionFiles { + wheels: [], + source_dists: [ + VersionSourceDist { + name: SourceDistFilename { + name: PackageName( + "pepy", + ), + version: "2.1.1", + extension: TarGz, + }, + file: File { + dist_info_metadata: false, + filename: "pepy-2.1.1.tar.gz", + hashes: HashDigests( + [ + HashDigest { + algorithm: Sha256, + digest: "cec463c444b71d1664229121897b22df753dc91fabb2113d1c89992638c90829", + }, + ], + ), + requires_python: Some( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: "3.7", + }, + ], + ), + ), + size: None, + upload_time_utc_ms: None, + url: AbsoluteUrl( + UrlString( + "https://files.pythonhosted.org/packages/78/7e/123d89ce0e999e957e53f0b985f734565c93b9a698af53586fc2a1be0dbf/pepy-2.1.1.tar.gz", + ), + ), + yanked: None, + zstd: None, + }, + }, + ], + }, + metadata: None, + }, + ], + } + "#); + } + /// Test for AWS Code Artifact registry /// /// See: diff --git a/crates/uv-client/src/rkyvutil.rs b/crates/uv-client/src/rkyvutil.rs index fad4f0039..2a2c682d6 100644 --- a/crates/uv-client/src/rkyvutil.rs +++ b/crates/uv-client/src/rkyvutil.rs @@ -144,7 +144,7 @@ where } /// Deserialize this owned archived value into the original - /// `SimpleMetadata`. + /// `SimpleDetailMetadata`. /// /// Note that because this type has a `Deref` impl, this method requires /// fully-qualified syntax. So, if `o` is an `OwnedValue`, then use @@ -163,7 +163,7 @@ where fn deref(&self) -> &A::Archived { // SAFETY: We've validated that our underlying buffer is a valid - // archive for SimpleMetadata in the constructor, so we can skip + // archive for SimpleDetailMetadata in the constructor, so we can skip // validation here. Since we don't mutate the buffer, this conversion // is guaranteed to be correct. #[allow(unsafe_code)] diff --git a/crates/uv-pypi-types/src/project_status.rs b/crates/uv-pypi-types/src/project_status.rs index c1dc41920..6ff879753 100644 --- a/crates/uv-pypi-types/src/project_status.rs +++ b/crates/uv-pypi-types/src/project_status.rs @@ -10,7 +10,10 @@ use tracing::info; use uv_small_str::SmallString; /// The status marker for a project. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[derive( + Clone, Copy, Debug, Default, Eq, PartialEq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, +)] +#[rkyv(derive(Debug))] pub enum Status { #[default] Active, @@ -48,7 +51,8 @@ impl<'de> Deserialize<'de> for Status { /// The project status information. /// /// This includes a status marker and an optional reason for the status. -#[derive(Clone, Debug, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +#[rkyv(derive(Debug))] pub struct ProjectStatus { pub status: Status, pub reason: Option, diff --git a/crates/uv-pypi-types/src/simple_json.rs b/crates/uv-pypi-types/src/simple_json.rs index b1443fb35..e6841fdbf 100644 --- a/crates/uv-pypi-types/src/simple_json.rs +++ b/crates/uv-pypi-types/src/simple_json.rs @@ -135,6 +135,9 @@ impl<'de> Deserialize<'de> for PypiFile { #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct PyxSimpleDetail { + /// PEP 792 project status information. + #[serde(default)] + pub project_status: ProjectStatus, /// The list of [`PyxFile`]s available for download sorted by filename. pub files: Vec, /// The core metadata for the project, keyed by version. diff --git a/crates/uv-resolver/src/version_map.rs b/crates/uv-resolver/src/version_map.rs index b0f334336..95fbd9cec 100644 --- a/crates/uv-resolver/src/version_map.rs +++ b/crates/uv-resolver/src/version_map.rs @@ -342,7 +342,7 @@ enum VersionMapInner { Eager(VersionMapEager), /// Some distributions might be fully materialized (i.e., by initializing /// a `VersionMap` with a `FlatDistributions`), but some distributions - /// might still be in their "raw" `SimpleMetadata` format. In this case, a + /// might still be in their "raw" `SimpleDetailMetadata` format. In this case, a /// `PrioritizedDist` isn't actually created in memory until the /// specific version has been requested. Lazy(VersionMapLazy), @@ -363,7 +363,7 @@ struct VersionMapEager { /// /// The idea here is that some packages have a lot of versions published, and /// needing to materialize a full `VersionMap` with all corresponding metadata -/// for every version in memory is expensive. Since a `SimpleMetadata` can be +/// for every version in memory is expensive. Since a `SimpleDetailMetadata` can be /// materialized with very little cost (via `rkyv` in the warm cached case), /// avoiding another conversion step into a fully filled out `VersionMap` can /// provide substantial savings in some cases. @@ -638,7 +638,7 @@ enum LazyPrioritizedDist { /// `FlatDistributions`. OnlyFlat(PrioritizedDist), /// Represents a lazily constructed distribution from an index into a - /// `VersionFiles` from `SimpleMetadata`. + /// `VersionFiles` from `SimpleDetailMetadata`. OnlySimple(SimplePrioritizedDist), /// Combines the above. This occurs when we have data from both a flat /// distribution and a simple distribution. @@ -651,7 +651,7 @@ enum LazyPrioritizedDist { /// Represents a lazily initialized `PrioritizedDist`. #[derive(Debug)] struct SimplePrioritizedDist { - /// An offset into `SimpleMetadata` corresponding to a `SimpleMetadatum`. + /// An offset into `SimpleDetailMetadata` corresponding to a `SimpleMetadatum`. /// This provides access to a `VersionFiles` that is used to construct a /// `PrioritizedDist`. datum_index: usize, diff --git a/crates/uv/tests/it/cache_clean.rs b/crates/uv/tests/it/cache_clean.rs index eecf5455d..2646fb765 100644 --- a/crates/uv/tests/it/cache_clean.rs +++ b/crates/uv/tests/it/cache_clean.rs @@ -111,7 +111,7 @@ fn clean_package_pypi() -> Result<()> { // Assert that the `.rkyv` file is created for `iniconfig`. let rkyv = context .cache_dir - .child("simple-v18") + .child("simple-v19") .child("pypi") .child("iniconfig.rkyv"); assert!( @@ -189,7 +189,7 @@ fn clean_package_index() -> Result<()> { // Assert that the `.rkyv` file is created for `iniconfig`. let rkyv = context .cache_dir - .child("simple-v18") + .child("simple-v19") .child("index") .child("e8208120cae3ba69") .child("iniconfig.rkyv");