From e2d7d2026bf0673d527e22eb82c10c4be562ca05 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 17 May 2024 17:32:42 -0400 Subject: [PATCH] Add direct URL conversion to lockfile (#3633) ## Summary Similar to #3630, but for direct URL distributions (for both wheels and source distributions). --- crates/distribution-types/src/lib.rs | 1 + crates/uv-resolver/src/lock.rs | 95 ++++++++++++++++++++++++---- crates/uv/tests/lock.rs | 32 ++++++++++ 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index 213ffba51..c50d9a56b 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -202,6 +202,7 @@ pub struct DirectUrlBuiltDist { pub filename: WheelFilename, /// The URL without the subdirectory fragment. pub location: Url, + /// The subdirectory fragment, if any. pub subdirectory: Option, /// The URL with the subdirectory fragment. pub url: VerbatimUrl, diff --git a/crates/uv-resolver/src/lock.rs b/crates/uv-resolver/src/lock.rs index 288c0d67c..38500d963 100644 --- a/crates/uv-resolver/src/lock.rs +++ b/crates/uv-resolver/src/lock.rs @@ -12,8 +12,8 @@ use url::Url; use distribution_filename::WheelFilename; use distribution_types::{ BuiltDist, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist, Dist, FileLocation, - GitSourceDist, IndexUrl, PathBuiltDist, PathSourceDist, RegistryBuiltDist, RegistryBuiltWheel, - RegistrySourceDist, Resolution, ResolvedDist, ToUrlError, + GitSourceDist, IndexUrl, ParsedArchiveUrl, PathBuiltDist, PathSourceDist, RegistryBuiltDist, + RegistryBuiltWheel, RegistrySourceDist, Resolution, ResolvedDist, ToUrlError, }; use pep440_rs::Version; use pep508_rs::{MarkerEnvironment, VerbatimUrl}; @@ -256,8 +256,27 @@ impl Distribution { let built_dist = BuiltDist::Path(path_dist); Dist::Built(built_dist) } - // TODO: Handle other kinds of sources. - _ => todo!(), + SourceKind::Direct(direct) => { + let filename: WheelFilename = self.wheels[best_wheel_index].filename.clone(); + let url = Url::from(ParsedArchiveUrl { + url: self.id.source.url.clone(), + subdirectory: direct.subdirectory.as_ref().map(PathBuf::from), + }); + let direct_dist = DirectUrlBuiltDist { + filename, + location: self.id.source.url.clone(), + subdirectory: direct.subdirectory.as_ref().map(PathBuf::from), + url: VerbatimUrl::from_url(url), + }; + let built_dist = BuiltDist::DirectUrl(direct_dist); + Dist::Built(built_dist) + } + SourceKind::Git(_) => { + unreachable!("Wheels cannot come from Git sources") + } + SourceKind::Directory => { + unreachable!("Wheels cannot come from directory sources") + } }; } @@ -304,9 +323,21 @@ impl Distribution { let source_dist = distribution_types::SourceDist::Git(git_dist); Dist::Source(source_dist) } - - // TODO: Handle other kinds of sources. - _ => todo!(), + SourceKind::Direct(direct) => { + let url = Url::from(ParsedArchiveUrl { + url: self.id.source.url.clone(), + subdirectory: direct.subdirectory.as_ref().map(PathBuf::from), + }); + let direct_dist = DirectUrlSourceDist { + name: self.id.name.clone(), + location: self.id.source.url.clone(), + subdirectory: direct.subdirectory.as_ref().map(PathBuf::from), + url: VerbatimUrl::from_url(url), + }; + let source_dist = distribution_types::SourceDist::DirectUrl(direct_dist); + Dist::Source(source_dist) + } + SourceKind::Registry => todo!(), }; } @@ -426,14 +457,26 @@ impl Source { fn from_direct_built_dist(direct_dist: &DirectUrlBuiltDist) -> Source { Source { - kind: SourceKind::Direct, + kind: SourceKind::Direct(DirectSource { + subdirectory: direct_dist + .subdirectory + .as_deref() + .and_then(Path::to_str) + .map(ToString::to_string), + }), url: direct_dist.url.to_url(), } } fn from_direct_source_dist(direct_dist: &DirectUrlSourceDist) -> Source { Source { - kind: SourceKind::Direct, + kind: SourceKind::Direct(DirectSource { + subdirectory: direct_dist + .subdirectory + .as_deref() + .and_then(Path::to_str) + .map(ToString::to_string), + }), url: direct_dist.url.to_url(), } } @@ -513,7 +556,7 @@ impl std::str::FromStr for Source { url, }), "direct" => Ok(Source { - kind: SourceKind::Direct, + kind: SourceKind::Direct(DirectSource::from_url(&mut url)), url, }), "path" => Ok(Source { @@ -562,7 +605,7 @@ impl<'de> serde::Deserialize<'de> for Source { pub(crate) enum SourceKind { Registry, Git(GitSource), - Direct, + Direct(DirectSource), Path, Directory, } @@ -572,7 +615,7 @@ impl SourceKind { match *self { SourceKind::Registry => "registry", SourceKind::Git(_) => "git", - SourceKind::Direct => "direct", + SourceKind::Direct(_) => "direct", SourceKind::Path => "path", SourceKind::Directory => "directory", } @@ -584,12 +627,38 @@ impl SourceKind { /// _not_ be present. fn requires_hash(&self) -> bool { match *self { - SourceKind::Registry | SourceKind::Direct | SourceKind::Path => true, + SourceKind::Registry | SourceKind::Direct(_) | SourceKind::Path => true, SourceKind::Git(_) | SourceKind::Directory => false, } } } +#[derive( + Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, serde::Deserialize, serde::Serialize, +)] +pub(crate) struct DirectSource { + subdirectory: Option, +} + +impl DirectSource { + /// Extracts a direct source reference from the query pairs in the given URL. + /// + /// This also removes the query pairs and hash fragment from the given + /// URL in place. + fn from_url(url: &mut Url) -> DirectSource { + let subdirectory = url.query_pairs().find_map(|(key, val)| { + if key == "subdirectory" { + Some(val.into_owned()) + } else { + None + } + }); + url.set_query(None); + url.set_fragment(None); + DirectSource { subdirectory } + } +} + /// NOTE: Care should be taken when adding variants to this enum. Namely, new /// variants should be added without changing the relative ordering of other /// variants. Otherwise, this could cause the lock file to have a different diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index ade9d32bd..2afed75e1 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -440,6 +440,22 @@ fn lock_wheel_url() -> Result<()> { "### ); + // Install from the lockfile. + uv_snapshot!(context.install().arg("--unstable-uv-lock-file").arg("anyio"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Downloaded 4 packages in [TIME] + Installed 5 packages in [TIME] + + anyio==4.3.0 (from https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl) + + exceptiongroup==1.2.0 + + idna==3.6 + + sniffio==1.3.1 + + typing-extensions==4.10.0 + "###); + Ok(()) } @@ -573,5 +589,21 @@ fn lock_sdist_url() -> Result<()> { "### ); + // Install from the lockfile. + uv_snapshot!(context.install().arg("--unstable-uv-lock-file").arg("anyio"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Downloaded 4 packages in [TIME] + Installed 5 packages in [TIME] + + anyio==4.3.0 (from https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz) + + exceptiongroup==1.2.0 + + idna==3.6 + + sniffio==1.3.1 + + typing-extensions==4.10.0 + "###); + Ok(()) }