diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index acd48e322..e94633f89 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -13,7 +13,7 @@ use platform_host::Platform; use platform_tags::Tags; use puffin_client::RegistryClientBuilder; use puffin_dispatch::BuildDispatch; -use puffin_distribution::Distribution; +use puffin_distribution::{AnyDistribution, BaseDistribution}; use puffin_installer::{Builder, InstallPlan}; use puffin_interpreter::Virtualenv; @@ -308,11 +308,11 @@ pub(crate) async fn sync_requirements( for event in extraneous .into_iter() .map(|distribution| ChangeEvent { - distribution: Distribution::from(distribution), + distribution: AnyDistribution::from(distribution), kind: ChangeEventKind::Remove, }) .chain(wheels.into_iter().map(|distribution| ChangeEvent { - distribution: Distribution::from(distribution), + distribution: AnyDistribution::from(distribution), kind: ChangeEventKind::Add, })) .sorted_unstable_by_key(|event| event.distribution.name().clone()) @@ -352,6 +352,6 @@ enum ChangeEventKind { #[derive(Debug)] struct ChangeEvent { - distribution: Distribution, + distribution: AnyDistribution, kind: ChangeEventKind, } diff --git a/crates/puffin-cli/src/commands/pip_uninstall.rs b/crates/puffin-cli/src/commands/pip_uninstall.rs index 82da955ec..a9f1d0d72 100644 --- a/crates/puffin-cli/src/commands/pip_uninstall.rs +++ b/crates/puffin-cli/src/commands/pip_uninstall.rs @@ -6,6 +6,7 @@ use colored::Colorize; use tracing::debug; use platform_host::Platform; +use puffin_distribution::BaseDistribution; use puffin_interpreter::Virtualenv; use crate::commands::{elapsed, ExitStatus}; diff --git a/crates/puffin-cli/src/commands/reporters.rs b/crates/puffin-cli/src/commands/reporters.rs index 376338653..f806cb1b1 100644 --- a/crates/puffin-cli/src/commands/reporters.rs +++ b/crates/puffin-cli/src/commands/reporters.rs @@ -6,8 +6,9 @@ use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use url::Url; use puffin_distribution::{ - CachedDistribution, RemoteDistribution, RemoteDistributionRef, VersionOrUrl, + BaseDistribution, CachedDistribution, Distribution, SourceDistribution, VersionOrUrl, }; +use puffin_installer::Download; use puffin_normalize::ExtraName; use puffin_normalize::PackageName; @@ -38,7 +39,7 @@ impl FinderReporter { } impl puffin_resolver::FinderReporter for FinderReporter { - fn on_progress(&self, wheel: &RemoteDistribution) { + fn on_progress(&self, wheel: &Distribution) { self.progress.set_message(format!("{wheel}")); self.progress.inc(1); } @@ -73,7 +74,7 @@ impl UnzipReporter { } impl puffin_installer::UnzipReporter for UnzipReporter { - fn on_unzip_progress(&self, wheel: &RemoteDistribution) { + fn on_unzip_progress(&self, wheel: &Distribution) { self.progress.set_message(format!("{wheel}")); self.progress.inc(1); } @@ -108,8 +109,8 @@ impl DownloadReporter { } impl puffin_installer::DownloadReporter for DownloadReporter { - fn on_download_progress(&self, wheel: &RemoteDistribution) { - self.progress.set_message(format!("{wheel}")); + fn on_download_progress(&self, download: &Download) { + self.progress.set_message(format!("{download}")); self.progress.inc(1); } @@ -178,7 +179,7 @@ impl BuildReporter { } impl puffin_installer::BuildReporter for BuildReporter { - fn on_progress(&self, wheel: &RemoteDistribution) { + fn on_progress(&self, wheel: &Distribution) { self.progress.set_message(format!("{wheel}")); self.progress.inc(1); } @@ -247,7 +248,7 @@ impl puffin_resolver::ResolverReporter for ResolverReporter { self.progress.finish_and_clear(); } - fn on_build_start(&self, distribution: &RemoteDistributionRef<'_>) -> usize { + fn on_build_start(&self, distribution: &SourceDistribution) -> usize { let progress = self.multi_progress.insert_before( &self.progress, ProgressBar::with_draw_target(None, self.printer.target()), @@ -265,7 +266,7 @@ impl puffin_resolver::ResolverReporter for ResolverReporter { bars.len() - 1 } - fn on_build_complete(&self, distribution: &RemoteDistributionRef<'_>, index: usize) { + fn on_build_complete(&self, distribution: &SourceDistribution, index: usize) { let bars = self.bars.lock().unwrap(); let progress = &bars[index]; progress.finish_with_message(format!( @@ -312,15 +313,10 @@ trait ColorDisplay { fn to_color_string(&self) -> String; } -impl ColorDisplay for &RemoteDistributionRef<'_> { +impl ColorDisplay for &SourceDistribution { fn to_color_string(&self) -> String { - match self { - RemoteDistributionRef::Registry(name, version, _file) => { - format!("{}{}", name, format!("=={version}").dimmed()) - } - RemoteDistributionRef::Url(name, url) => { - format!("{}{}", name, format!(" @ {url}").dimmed()) - } - } + let name = self.name(); + let version_or_url = self.version_or_url(); + format!("{}{}", name, version_or_url.to_string().dimmed()) } } diff --git a/crates/puffin-client/src/client.rs b/crates/puffin-client/src/client.rs index f51bfc07f..1973cabf5 100644 --- a/crates/puffin-client/src/client.rs +++ b/crates/puffin-client/src/client.rs @@ -1,5 +1,6 @@ use std::fmt::Debug; use std::path::PathBuf; +use std::str::FromStr; use async_http_range_reader::{ AsyncHttpRangeReader, AsyncHttpRangeReaderError, CheckSupportMethod, @@ -199,11 +200,7 @@ impl RegistryClient { } /// Fetch the metadata from a wheel file. - pub async fn wheel_metadata( - &self, - file: File, - filename: WheelFilename, - ) -> Result { + pub async fn wheel_metadata(&self, file: File) -> Result { if self.no_index { return Err(Error::NoIndex(file.filename)); } @@ -226,6 +223,7 @@ impl RegistryClient { // `.dist-info/METADATA` file from the zip, and if that also fails, download the whole wheel // into the cache and read from there } else { + let filename = WheelFilename::from_str(&file.filename)?; self.wheel_metadata_no_index(&filename, &url).await } } diff --git a/crates/puffin-client/src/error.rs b/crates/puffin-client/src/error.rs index a65541e7d..e913acb6f 100644 --- a/crates/puffin-client/src/error.rs +++ b/crates/puffin-client/src/error.rs @@ -4,7 +4,7 @@ use async_http_range_reader::AsyncHttpRangeReaderError; use async_zip::error::ZipError; use thiserror::Error; -use distribution_filename::WheelFilename; +use distribution_filename::{WheelFilename, WheelFilenameError}; #[derive(Debug, Error)] pub enum Error { @@ -52,6 +52,9 @@ pub enum Error { #[error("Expected a single .dist-info directory in {0}, found {1}")] InvalidDistInfo(WheelFilename, String), + #[error("{0} is not a valid wheel filename")] + WheelFilename(#[from] WheelFilenameError), + #[error("The wheel {0} is not a valid zip file")] Zip(WheelFilename, #[source] ZipError), diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index 93a6cc354..f8e04c7f9 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -15,6 +15,7 @@ use pep508_rs::Requirement; use platform_tags::Tags; use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_client::RegistryClient; +use puffin_distribution::BaseDistribution; use puffin_installer::{Builder, Downloader, InstallPlan, Installer, Unzipper}; use puffin_interpreter::{InterpreterInfo, Virtualenv}; use puffin_resolver::{DistributionFinder, Manifest, PreReleaseMode, ResolutionMode, Resolver}; diff --git a/crates/puffin-distribution/src/any.rs b/crates/puffin-distribution/src/any.rs new file mode 100644 index 000000000..4b7838e7f --- /dev/null +++ b/crates/puffin-distribution/src/any.rs @@ -0,0 +1,50 @@ +use puffin_normalize::PackageName; + +use crate::cached::CachedDistribution; +use crate::installed::InstalledDistribution; +use crate::traits::BaseDistribution; +use crate::{Distribution, VersionOrUrl}; + +/// A distribution which either exists remotely or locally. +#[derive(Debug, Clone)] +pub enum AnyDistribution { + Remote(Distribution), + Cached(CachedDistribution), + Installed(InstalledDistribution), +} + +impl BaseDistribution for AnyDistribution { + fn name(&self) -> &PackageName { + match self { + Self::Remote(dist) => dist.name(), + Self::Cached(dist) => dist.name(), + Self::Installed(dist) => dist.name(), + } + } + + fn version_or_url(&self) -> VersionOrUrl { + match self { + Self::Remote(dist) => dist.version_or_url(), + Self::Cached(dist) => dist.version_or_url(), + Self::Installed(dist) => dist.version_or_url(), + } + } +} + +impl From for AnyDistribution { + fn from(dist: Distribution) -> Self { + Self::Remote(dist) + } +} + +impl From for AnyDistribution { + fn from(dist: CachedDistribution) -> Self { + Self::Cached(dist) + } +} + +impl From for AnyDistribution { + fn from(dist: InstalledDistribution) -> Self { + Self::Installed(dist) + } +} diff --git a/crates/puffin-distribution/src/cached.rs b/crates/puffin-distribution/src/cached.rs new file mode 100644 index 000000000..058f811bb --- /dev/null +++ b/crates/puffin-distribution/src/cached.rs @@ -0,0 +1,161 @@ +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{anyhow, Result}; +use url::Url; + +use crate::traits::BaseDistribution; +use crate::{BuiltDistribution, Distribution, SourceDistribution, VersionOrUrl}; +use pep440_rs::Version; +use puffin_normalize::PackageName; + +use crate::direct_url::DirectUrl; + +/// A built distribution (wheel) that exists in a local cache. +#[derive(Debug, Clone)] +pub enum CachedDistribution { + /// The distribution exists in a registry, like `PyPI`. + Registry(CachedRegistryDistribution), + /// The distribution exists at an arbitrary URL. + Url(CachedDirectUrlDistribution), +} + +#[derive(Debug, Clone)] +pub struct CachedRegistryDistribution { + pub name: PackageName, + pub version: Version, + pub path: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct CachedDirectUrlDistribution { + pub name: PackageName, + pub url: Url, + pub path: PathBuf, +} + +impl BaseDistribution for CachedRegistryDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Version(&self.version) + } +} + +impl BaseDistribution for CachedDirectUrlDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Url(&self.url) + } +} + +impl BaseDistribution for CachedDistribution { + fn name(&self) -> &PackageName { + match self { + Self::Registry(dist) => dist.name(), + Self::Url(dist) => dist.name(), + } + } + + fn version_or_url(&self) -> VersionOrUrl { + match self { + Self::Registry(dist) => dist.version_or_url(), + Self::Url(dist) => dist.version_or_url(), + } + } +} + +impl CachedDistribution { + /// Initialize a [`CachedDistribution`] from a [`Distribution`]. + pub fn from_remote(remote: Distribution, path: PathBuf) -> Self { + match remote { + Distribution::Built(BuiltDistribution::Registry(dist)) => { + Self::Registry(CachedRegistryDistribution { + name: dist.name, + version: dist.version, + path, + }) + } + Distribution::Built(BuiltDistribution::DirectUrl(dist)) => { + Self::Url(CachedDirectUrlDistribution { + name: dist.name, + url: dist.url, + path, + }) + } + Distribution::Source(SourceDistribution::Registry(dist)) => { + Self::Registry(CachedRegistryDistribution { + name: dist.name, + version: dist.version, + path, + }) + } + Distribution::Source(SourceDistribution::DirectUrl(dist)) => { + Self::Url(CachedDirectUrlDistribution { + name: dist.name, + url: dist.url, + path, + }) + } + Distribution::Source(SourceDistribution::Git(dist)) => { + Self::Url(CachedDirectUrlDistribution { + name: dist.name, + url: dist.url, + path, + }) + } + } + } + + /// Return the [`Path`] at which the distribution is stored on-disk. + pub fn path(&self) -> &Path { + match self { + Self::Registry(dist) => &dist.path, + Self::Url(dist) => &dist.path, + } + } + + /// Return the [`DirectUrl`] of the distribution, if it exists. + pub fn direct_url(&self) -> Result> { + match self { + CachedDistribution::Registry(_) => Ok(None), + CachedDistribution::Url(dist) => DirectUrl::try_from(&dist.url).map(Some), + } + } +} + +impl CachedDirectUrlDistribution { + pub fn from_url(name: PackageName, url: Url, path: PathBuf) -> Self { + Self { name, url, path } + } +} + +impl CachedRegistryDistribution { + /// Try to parse a distribution from a cached directory name (like `django-5.0a1`). + pub fn try_from_path(path: &Path) -> Result> { + let Some(file_name) = path.file_name() else { + return Ok(None); + }; + let Some(file_name) = file_name.to_str() else { + return Ok(None); + }; + let Some((name, version)) = file_name.rsplit_once('-') else { + return Ok(None); + }; + + let name = PackageName::from_str(name)?; + let version = Version::from_str(version).map_err(|err| anyhow!(err))?; + let path = path.to_path_buf(); + + Ok(Some(Self { + name, + version, + path, + })) + } +} diff --git a/crates/puffin-distribution/src/direct_url.rs b/crates/puffin-distribution/src/direct_url.rs new file mode 100644 index 000000000..2476df4e7 --- /dev/null +++ b/crates/puffin-distribution/src/direct_url.rs @@ -0,0 +1,188 @@ +use std::path::PathBuf; + +use anyhow::{Context, Error, Result}; +use url::Url; + +use puffin_git::GitUrl; + +#[derive(Debug)] +pub enum DirectUrl { + Git(DirectGitUrl), + Archive(DirectArchiveUrl), +} + +#[derive(Debug)] +pub struct DirectGitUrl { + pub url: GitUrl, + pub subdirectory: Option, +} + +#[derive(Debug)] +pub struct DirectArchiveUrl { + pub url: Url, + pub subdirectory: Option, +} + +impl TryFrom<&Url> for DirectGitUrl { + type Error = Error; + + fn try_from(url: &Url) -> Result { + // If the URL points to a subdirectory, extract it, as in: + // `https://git.example.com/MyProject.git@v1.0#subdirectory=pkg_dir` + // `https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` + let subdirectory = url.fragment().and_then(|fragment| { + fragment + .split('&') + .find_map(|fragment| fragment.strip_prefix("subdirectory=").map(PathBuf::from)) + }); + + let url = url + .as_str() + .strip_prefix("git+") + .context("Missing git+ prefix for Git URL")?; + let url = Url::parse(url)?; + let url = GitUrl::try_from(url)?; + Ok(Self { url, subdirectory }) + } +} + +impl From<&Url> for DirectArchiveUrl { + fn from(url: &Url) -> Self { + // If the URL points to a subdirectory, extract it, as in: + // `https://git.example.com/MyProject.git@v1.0#subdirectory=pkg_dir` + // `https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` + let subdirectory = url.fragment().and_then(|fragment| { + fragment + .split('&') + .find_map(|fragment| fragment.strip_prefix("subdirectory=").map(PathBuf::from)) + }); + + let url = url.clone(); + Self { url, subdirectory } + } +} + +impl TryFrom<&Url> for DirectUrl { + type Error = Error; + + fn try_from(url: &Url) -> Result { + if let Some((prefix, ..)) = url.scheme().split_once('+') { + match prefix { + "git" => Ok(Self::Git(DirectGitUrl::try_from(url)?)), + _ => Err(Error::msg(format!( + "Unsupported URL prefix `{prefix}` in URL: {url}", + ))), + } + } else { + Ok(Self::Archive(DirectArchiveUrl::from(url))) + } + } +} + +impl TryFrom<&DirectUrl> for pypi_types::DirectUrl { + type Error = Error; + + fn try_from(value: &DirectUrl) -> std::result::Result { + match value { + DirectUrl::Git(value) => pypi_types::DirectUrl::try_from(value), + DirectUrl::Archive(value) => pypi_types::DirectUrl::try_from(value), + } + } +} + +impl TryFrom<&DirectArchiveUrl> for pypi_types::DirectUrl { + type Error = Error; + + fn try_from(value: &DirectArchiveUrl) -> Result { + Ok(pypi_types::DirectUrl::ArchiveUrl { + url: value.url.to_string(), + archive_info: pypi_types::ArchiveInfo { + hash: None, + hashes: None, + }, + subdirectory: value.subdirectory.clone(), + }) + } +} + +impl TryFrom<&DirectGitUrl> for pypi_types::DirectUrl { + type Error = Error; + + fn try_from(value: &DirectGitUrl) -> Result { + Ok(pypi_types::DirectUrl::VcsUrl { + url: value.url.repository().to_string(), + vcs_info: pypi_types::VcsInfo { + vcs: pypi_types::VcsKind::Git, + commit_id: value.url.precise().map(|oid| oid.to_string()), + requested_revision: value.url.reference().map(ToString::to_string), + }, + subdirectory: value.subdirectory.clone(), + }) + } +} + +impl From for Url { + fn from(value: DirectUrl) -> Self { + match value { + DirectUrl::Git(value) => value.into(), + DirectUrl::Archive(value) => value.into(), + } + } +} + +impl From for Url { + fn from(value: DirectArchiveUrl) -> Self { + let mut url = value.url; + if let Some(subdirectory) = value.subdirectory { + url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); + } + url + } +} + +impl From for Url { + fn from(value: DirectGitUrl) -> Self { + let mut url = Url::parse(&format!("{}{}", "git+", Url::from(value.url).as_str())) + .expect("Git URL is invalid"); + if let Some(subdirectory) = value.subdirectory { + url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); + } + url + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use url::Url; + + use crate::direct_url::DirectUrl; + + #[test] + fn direct_url_from_url() -> Result<()> { + let expected = Url::parse("git+https://github.com/pallets/flask.git")?; + let actual = Url::from(DirectUrl::try_from(&expected)?); + assert_eq!(expected, actual); + + let expected = Url::parse("git+https://github.com/pallets/flask.git#subdirectory=pkg_dir")?; + let actual = Url::from(DirectUrl::try_from(&expected)?); + assert_eq!(expected, actual); + + let expected = Url::parse("git+https://github.com/pallets/flask.git@2.0.0")?; + let actual = Url::from(DirectUrl::try_from(&expected)?); + assert_eq!(expected, actual); + + let expected = + Url::parse("git+https://github.com/pallets/flask.git@2.0.0#subdirectory=pkg_dir")?; + let actual = Url::from(DirectUrl::try_from(&expected)?); + assert_eq!(expected, actual); + + // TODO(charlie): Preserve other fragments. + let expected = + Url::parse("git+https://github.com/pallets/flask.git#egg=flask&subdirectory=pkg_dir")?; + let actual = Url::from(DirectUrl::try_from(&expected)?); + assert_ne!(expected, actual); + + Ok(()) + } +} diff --git a/crates/puffin-distribution/src/installed.rs b/crates/puffin-distribution/src/installed.rs new file mode 100644 index 000000000..f3def1b01 --- /dev/null +++ b/crates/puffin-distribution/src/installed.rs @@ -0,0 +1,133 @@ +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{anyhow, Result}; + +use pep440_rs::Version; +use puffin_normalize::PackageName; +use pypi_types::DirectUrl; + +use crate::{BaseDistribution, VersionOrUrl}; + +/// A built distribution (wheel) that exists in a virtual environment. +#[derive(Debug, Clone)] +pub enum InstalledDistribution { + /// The distribution was derived from a registry, like `PyPI`. + Registry(InstalledRegistryDistribution), + /// The distribution was derived from an arbitrary URL. + Url(InstalledDirectUrlDistribution), +} + +#[derive(Debug, Clone)] +pub struct InstalledRegistryDistribution { + pub name: PackageName, + pub version: Version, + pub path: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct InstalledDirectUrlDistribution { + pub name: PackageName, + pub version: Version, + pub url: DirectUrl, + pub path: PathBuf, +} + +impl BaseDistribution for InstalledRegistryDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Version(&self.version) + } +} + +impl BaseDistribution for InstalledDirectUrlDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + // TODO(charlie): Convert a `DirectUrl` to `Url`. + VersionOrUrl::Version(&self.version) + } +} + +impl BaseDistribution for InstalledDistribution { + fn name(&self) -> &PackageName { + match self { + Self::Registry(dist) => dist.name(), + Self::Url(dist) => dist.name(), + } + } + + fn version_or_url(&self) -> VersionOrUrl { + match self { + Self::Registry(dist) => dist.version_or_url(), + Self::Url(dist) => dist.version_or_url(), + } + } +} + +impl InstalledDistribution { + /// Try to parse a distribution from a `.dist-info` directory name (like `django-5.0a1.dist-info`). + /// + /// See: + pub fn try_from_path(path: &Path) -> Result> { + if path.extension().is_some_and(|ext| ext == "dist-info") { + let Some(file_stem) = path.file_stem() else { + return Ok(None); + }; + let Some(file_stem) = file_stem.to_str() else { + return Ok(None); + }; + let Some((name, version)) = file_stem.split_once('-') else { + return Ok(None); + }; + + let name = PackageName::from_str(name)?; + let version = Version::from_str(version).map_err(|err| anyhow!(err))?; + return if let Some(direct_url) = Self::direct_url(path)? { + Ok(Some(Self::Url(InstalledDirectUrlDistribution { + name, + version, + url: direct_url, + path: path.to_path_buf(), + }))) + } else { + Ok(Some(Self::Registry(InstalledRegistryDistribution { + name, + version, + path: path.to_path_buf(), + }))) + }; + } + Ok(None) + } + + /// Return the [`Path`] at which the distribution is stored on-disk. + pub fn path(&self) -> &Path { + match self { + Self::Registry(dist) => &dist.path, + Self::Url(dist) => &dist.path, + } + } + + pub fn version(&self) -> &Version { + match self { + Self::Registry(dist) => &dist.version, + Self::Url(dist) => &dist.version, + } + } + + /// Read the `direct_url.json` file from a `.dist-info` directory. + fn direct_url(path: &Path) -> Result> { + let path = path.join("direct_url.json"); + let Ok(file) = fs_err::File::open(path) else { + return Ok(None); + }; + let direct_url = serde_json::from_reader::(file)?; + Ok(Some(direct_url)) + } +} diff --git a/crates/puffin-distribution/src/lib.rs b/crates/puffin-distribution/src/lib.rs index 36aa2f25f..aa5c85a26 100644 --- a/crates/puffin-distribution/src/lib.rs +++ b/crates/puffin-distribution/src/lib.rs @@ -1,63 +1,22 @@ -use std::borrow::Cow; -use std::path::{Path, PathBuf}; -use std::str::FromStr; +use std::path::Path; -use anyhow::{anyhow, Result}; +use anyhow::{Context, Result}; use url::Url; use pep440_rs::Version; -use puffin_cache::CanonicalUrl; use puffin_normalize::PackageName; -use pypi_types::{DirectUrl, File}; +use pypi_types::File; -pub mod source; +pub use crate::any::*; +pub use crate::cached::*; +pub use crate::installed::*; +pub use crate::traits::*; -/// A built distribution (wheel), which either exists remotely or locally. -#[derive(Debug, Clone)] -pub enum Distribution { - Remote(RemoteDistribution), - Cached(CachedDistribution), - Installed(InstalledDistribution), -} - -impl Distribution { - /// Return the normalized [`PackageName`] of the distribution. - pub fn name(&self) -> &PackageName { - match self { - Self::Remote(dist) => dist.name(), - Self::Cached(dist) => dist.name(), - Self::Installed(dist) => dist.name(), - } - } - - /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based - /// distributions. - pub fn version_or_url(&self) -> VersionOrUrl { - match self { - Self::Remote(dist) => dist.version_or_url(), - Self::Cached(dist) => dist.version_or_url(), - Self::Installed(dist) => dist.version_or_url(), - } - } -} - -impl From for Distribution { - fn from(dist: RemoteDistribution) -> Self { - Self::Remote(dist) - } -} - -impl From for Distribution { - fn from(dist: CachedDistribution) -> Self { - Self::Cached(dist) - } -} - -impl From for Distribution { - fn from(dist: InstalledDistribution) -> Self { - Self::Installed(dist) - } -} +mod any; +mod cached; +pub mod direct_url; +mod installed; +mod traits; #[derive(Debug, Clone)] pub enum VersionOrUrl<'a> { @@ -76,361 +35,446 @@ impl std::fmt::Display for VersionOrUrl<'_> { } } -/// A built distribution (wheel) that exists as a remote file (e.g., on `PyPI`). +#[derive(Debug, Clone)] +pub enum Distribution { + Built(BuiltDistribution), + Source(SourceDistribution), +} + #[derive(Debug, Clone)] #[allow(clippy::large_enum_variant)] -pub enum RemoteDistribution { - /// The distribution exists in a registry, like `PyPI`. - Registry(PackageName, Version, File), - /// The distribution exists at an arbitrary URL. - Url(PackageName, Url), +pub enum BuiltDistribution { + Registry(RegistryBuiltDistribution), + DirectUrl(DirectUrlBuiltDistribution), } -impl RemoteDistribution { - /// Create a [`RemoteDistribution`] for a registry-based distribution. +#[derive(Debug, Clone)] +#[allow(clippy::large_enum_variant)] +pub enum SourceDistribution { + Registry(RegistrySourceDistribution), + DirectUrl(DirectUrlSourceDistribution), + Git(GitSourceDistribution), +} + +/// A built distribution (wheel) that exists in a registry, like `PyPI`. +#[derive(Debug, Clone)] +pub struct RegistryBuiltDistribution { + pub name: PackageName, + pub version: Version, + pub file: File, +} + +/// A built distribution (wheel) that exists at an arbitrary URL. +#[derive(Debug, Clone)] +pub struct DirectUrlBuiltDistribution { + pub name: PackageName, + pub url: Url, +} + +/// A source distribution that exists in a registry, like `PyPI`. +#[derive(Debug, Clone)] +pub struct RegistrySourceDistribution { + pub name: PackageName, + pub version: Version, + pub file: File, +} + +/// A source distribution that exists at an arbitrary URL. +#[derive(Debug, Clone)] +pub struct DirectUrlSourceDistribution { + pub name: PackageName, + pub url: Url, +} + +/// A source distribution that exists in a Git repository. +#[derive(Debug, Clone)] +pub struct GitSourceDistribution { + pub name: PackageName, + pub url: Url, +} + +impl Distribution { + /// Create a [`Distribution`] for a registry-based distribution. pub fn from_registry(name: PackageName, version: Version, file: File) -> Self { - Self::Registry(name, version, file) - } - - /// Create a [`RemoteDistribution`] for a URL-based distribution. - pub fn from_url(name: PackageName, url: Url) -> Self { - Self::Url(name, url) - } - - /// Return the URL of the distribution. - pub fn url(&self) -> Result> { - match self { - Self::Registry(_, _, file) => { - let url = Url::parse(&file.url)?; - Ok(Cow::Owned(url)) - } - Self::Url(_, url) => Ok(Cow::Borrowed(url)), - } - } - - /// Return the filename of the distribution. - pub fn filename(&self) -> Result> { - match self { - Self::Registry(_, _, file) => Ok(Cow::Borrowed(&file.filename)), - Self::Url(_, url) => { - let filename = url - .path_segments() - .and_then(Iterator::last) - .ok_or_else(|| anyhow!("Could not parse filename from URL: {}", url))?; - Ok(Cow::Owned(filename.to_owned())) - } - } - } - - /// Return the normalized [`PackageName`] of the distribution. - pub fn name(&self) -> &PackageName { - match self { - Self::Registry(name, _, _) => name, - Self::Url(name, _) => name, - } - } - - /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based - /// distributions. - pub fn version_or_url(&self) -> VersionOrUrl { - match self { - Self::Registry(_, version, _) => VersionOrUrl::Version(version), - Self::Url(_, url) => VersionOrUrl::Url(url), - } - } - - /// Returns a unique identifier for this distribution. - pub fn id(&self) -> String { - match self { - Self::Registry(name, version, _) => { - // https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory - // `version` is normalized by its `ToString` impl - format!( - "{}-{}", - PackageName::from(name).as_dist_info_name(), - version - ) - } - Self::Url(_name, url) => puffin_cache::digest(&CanonicalUrl::new(url)), - } - } - - /// Returns `true` if this distribution is a wheel. - pub fn is_wheel(&self) -> bool { - let filename = match self { - Self::Registry(_name, _version, file) => &file.filename, - Self::Url(_name, url) => url.path(), - }; - Path::new(filename) + if Path::new(&file.filename) .extension() .is_some_and(|ext| ext.eq_ignore_ascii_case("whl")) - } -} - -impl std::fmt::Display for RemoteDistribution { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Registry(name, version, _file) => { - write!(f, "{name}=={version}") - } - Self::Url(name, url) => { - write!(f, "{name} @ {url}") - } - } - } -} - -/// A built distribution (wheel) that exists in a local cache. -#[derive(Debug, Clone)] -pub enum CachedDistribution { - /// The distribution exists in a registry, like `PyPI`. - Registry(PackageName, Version, PathBuf), - /// The distribution exists at an arbitrary URL. - Url(PackageName, Url, PathBuf), -} - -impl CachedDistribution { - /// Initialize a [`CachedDistribution`] from a [`RemoteDistribution`]. - pub fn from_remote(remote: RemoteDistribution, path: PathBuf) -> Self { - match remote { - RemoteDistribution::Registry(name, version, _file) => { - Self::Registry(name, version, path) - } - RemoteDistribution::Url(name, url) => Self::Url(name, url, path), - } - } - - /// Try to parse a distribution from a cached directory name (like `django-5.0a1`). - pub fn try_from_path(path: &Path) -> Result> { - let Some(file_name) = path.file_name() else { - return Ok(None); - }; - let Some(file_name) = file_name.to_str() else { - return Ok(None); - }; - let Some((name, version)) = file_name.split_once('-') else { - return Ok(None); - }; - - let name = PackageName::from_str(name)?; - let version = Version::from_str(version).map_err(|err| anyhow!(err))?; - let path = path.to_path_buf(); - - Ok(Some(Self::Registry(name, version, path))) - } - - /// Return the normalized [`PackageName`] of the distribution. - pub fn name(&self) -> &PackageName { - match self { - Self::Registry(name, _, _) => name, - Self::Url(name, _, _) => name, - } - } - - /// Return the [`Path`] at which the distribution is stored on-disk. - pub fn path(&self) -> &Path { - match self { - Self::Registry(_, _, path) => path, - Self::Url(_, _, path) => path, - } - } - - /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based - /// distributions. - pub fn version_or_url(&self) -> VersionOrUrl { - match self { - Self::Registry(_, version, _) => VersionOrUrl::Version(version), - Self::Url(_, url, _) => VersionOrUrl::Url(url), - } - } -} - -impl std::fmt::Display for CachedDistribution { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Registry(name, version, _file) => { - write!(f, "{name}=={version}") - } - Self::Url(name, url, _path) => { - write!(f, "{name} @ {url}") - } - } - } -} - -/// A built distribution (wheel) that exists in a virtual environment. -#[derive(Debug, Clone)] -pub struct InstalledDistribution { - name: PackageName, - version: Version, - path: PathBuf, -} - -impl InstalledDistribution { - /// Try to parse a distribution from a `.dist-info` directory name (like `django-5.0a1.dist-info`). - /// - /// See: - pub fn try_from_path(path: &Path) -> Result> { - if path.extension().is_some_and(|ext| ext == "dist-info") { - let Some(file_stem) = path.file_stem() else { - return Ok(None); - }; - let Some(file_stem) = file_stem.to_str() else { - return Ok(None); - }; - let Some((name, version)) = file_stem.split_once('-') else { - return Ok(None); - }; - - let name = PackageName::from_str(name)?; - let version = Version::from_str(version).map_err(|err| anyhow!(err))?; - let path = path.to_path_buf(); - - return Ok(Some(Self { + { + Self::Built(BuiltDistribution::Registry(RegistryBuiltDistribution { name, version, - path, - })); + file, + })) + } else { + Self::Source(SourceDistribution::Registry(RegistrySourceDistribution { + name, + version, + file, + })) } - - Ok(None) } - /// Return the normalized [`PackageName`] of the distribution. - pub fn name(&self) -> &PackageName { + /// Create a [`Distribution`] for a URL-based distribution. + pub fn from_url(name: PackageName, url: Url) -> Self { + if url.scheme().starts_with("git+") { + Self::Source(SourceDistribution::Git(GitSourceDistribution { name, url })) + } else if Path::new(url.path()) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("whl")) + { + Self::Built(BuiltDistribution::DirectUrl(DirectUrlBuiltDistribution { + name, + url, + })) + } else { + Self::Source(SourceDistribution::DirectUrl(DirectUrlSourceDistribution { + name, + url, + })) + } + } +} + +impl BaseDistribution for RegistryBuiltDistribution { + fn name(&self) -> &PackageName { &self.name } - /// Return the [`Version`] of the distribution. - pub fn version(&self) -> &Version { - &self.version - } - - /// Return the [`Path`] at which the distribution is stored on-disk. - pub fn path(&self) -> &Path { - &self.path - } - - /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based - /// distributions. - pub fn version_or_url(&self) -> VersionOrUrl { - // TODO(charlie): If this dependency was installed via a direct URL, return it here, rather - // than the version. + fn version_or_url(&self) -> VersionOrUrl { VersionOrUrl::Version(&self.version) } +} - /// Return the [`DirectUrl`] metadata for this distribution, if it exists. - pub fn direct_url(&self) -> Result> { - let path = self.path.join("direct_url.json"); - let Ok(file) = fs_err::File::open(path) else { - return Ok(None); - }; - let direct_url = serde_json::from_reader::(file)?; - Ok(Some(direct_url)) +impl BaseDistribution for DirectUrlBuiltDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Url(&self.url) } } -impl std::fmt::Display for InstalledDistribution { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}=={}", self.name(), self.version()) +impl BaseDistribution for RegistrySourceDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Version(&self.version) } } -/// Unowned reference to a [`RemoteDistribution`]. -#[derive(Debug, Clone)] -pub enum RemoteDistributionRef<'a> { - /// The distribution exists in a registry, like `PyPI`. - Registry(&'a PackageName, &'a Version, &'a File), - /// The distribution exists at an arbitrary URL. - Url(&'a PackageName, &'a Url), +impl BaseDistribution for DirectUrlSourceDistribution { + fn name(&self) -> &PackageName { + &self.name + } + + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Url(&self.url) + } } -impl<'a> RemoteDistributionRef<'a> { - /// Create a [`RemoteDistribution`] for a registry-based distribution. - pub fn from_registry(name: &'a PackageName, version: &'a Version, file: &'a File) -> Self { - Self::Registry(name, version, file) +impl BaseDistribution for GitSourceDistribution { + fn name(&self) -> &PackageName { + &self.name } - /// Create a [`RemoteDistribution`] for a URL-based distribution. - pub fn from_url(name: &'a PackageName, url: &'a Url) -> Self { - Self::Url(name, url) + fn version_or_url(&self) -> VersionOrUrl { + VersionOrUrl::Url(&self.url) } +} - /// Return the URL of the distribution. - pub fn url(&self) -> Result> { +impl BaseDistribution for SourceDistribution { + fn name(&self) -> &PackageName { match self { - Self::Registry(_, _, file) => { - let url = Url::parse(&file.url)?; - Ok(Cow::Owned(url)) - } - Self::Url(_, url) => Ok(Cow::Borrowed(url)), + Self::Registry(dist) => dist.name(), + Self::DirectUrl(dist) => dist.name(), + Self::Git(dist) => dist.name(), } } - /// Return the filename of the distribution. - pub fn filename(&self) -> Result> { + fn version_or_url(&self) -> VersionOrUrl { match self { - Self::Registry(_, _, file) => Ok(Cow::Borrowed(&file.filename)), - Self::Url(_, url) => { - let filename = url - .path_segments() - .and_then(std::iter::Iterator::last) - .ok_or_else(|| anyhow!("Could not parse filename from URL: {}", url))?; - Ok(Cow::Owned(filename.to_owned())) - } - } - } - - /// Return the normalized [`PackageName`] of the distribution. - pub fn name(&self) -> &PackageName { - match self { - Self::Registry(name, _, _) => name, - Self::Url(name, _) => name, - } - } - - /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based - /// distributions. - pub fn version_or_url(&self) -> VersionOrUrl { - match self { - Self::Registry(_, version, _) => VersionOrUrl::Version(version), - Self::Url(_, url) => VersionOrUrl::Url(url), - } - } - - /// Returns a unique identifier for this distribution. - pub fn id(&self) -> String { - match self { - Self::Registry(name, version, _) => { - // https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory - // `version` is normalized by its `ToString` impl - format!("{}-{}", PackageName::from(*name), version) - } - Self::Url(_name, url) => puffin_cache::digest(&CanonicalUrl::new(url)), + Self::Registry(dist) => dist.version_or_url(), + Self::DirectUrl(dist) => dist.version_or_url(), + Self::Git(dist) => dist.version_or_url(), } } } -impl std::fmt::Display for RemoteDistributionRef<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl BaseDistribution for BuiltDistribution { + fn name(&self) -> &PackageName { match self { - Self::Registry(name, version, _file) => { - write!(f, "{name}=={version}") - } - Self::Url(name, url) => { - write!(f, "{name} @ {url}") - } + Self::Registry(dist) => dist.name(), + Self::DirectUrl(dist) => dist.name(), + } + } + + fn version_or_url(&self) -> VersionOrUrl { + match self { + Self::Registry(dist) => dist.version_or_url(), + Self::DirectUrl(dist) => dist.version_or_url(), } } } -impl<'a> From<&'a RemoteDistribution> for RemoteDistributionRef<'a> { - fn from(dist: &'a RemoteDistribution) -> Self { - match dist { - RemoteDistribution::Registry(name, version, file) => { - Self::Registry(name, version, file) - } - RemoteDistribution::Url(name, url) => Self::Url(name, url), +impl BaseDistribution for Distribution { + fn name(&self) -> &PackageName { + match self { + Self::Built(dist) => dist.name(), + Self::Source(dist) => dist.name(), + } + } + + fn version_or_url(&self) -> VersionOrUrl { + match self { + Self::Built(dist) => dist.version_or_url(), + Self::Source(dist) => dist.version_or_url(), + } + } +} + +impl RemoteDistribution for RegistryBuiltDistribution { + fn filename(&self) -> Result<&str> { + Ok(&self.file.filename) + } + + fn size(&self) -> Option { + Some(self.file.size) + } +} + +impl RemoteDistribution for RegistrySourceDistribution { + fn filename(&self) -> Result<&str> { + Ok(&self.file.filename) + } + + fn size(&self) -> Option { + Some(self.file.size) + } +} + +impl RemoteDistribution for DirectUrlBuiltDistribution { + fn filename(&self) -> Result<&str> { + self.url + .path_segments() + .and_then(Iterator::last) + .map(|filename| { + filename + .rsplit_once('@') + .map_or(filename, |(_, filename)| filename) + }) + .with_context(|| format!("Could not parse filename from URL: {}", self.url)) + } + + fn size(&self) -> Option { + None + } +} + +impl RemoteDistribution for DirectUrlSourceDistribution { + fn filename(&self) -> Result<&str> { + self.url + .path_segments() + .and_then(Iterator::last) + .map(|filename| { + filename + .rsplit_once('@') + .map_or(filename, |(_, filename)| filename) + }) + .with_context(|| format!("Could not parse filename from URL: {}", self.url)) + } + + fn size(&self) -> Option { + None + } +} + +impl RemoteDistribution for GitSourceDistribution { + fn filename(&self) -> Result<&str> { + self.url + .path_segments() + .and_then(Iterator::last) + .map(|filename| { + filename + .rsplit_once('@') + .map_or(filename, |(_, filename)| filename) + }) + .with_context(|| format!("Could not parse filename from URL: {}", self.url)) + } + + fn size(&self) -> Option { + None + } +} + +impl RemoteDistribution for SourceDistribution { + fn filename(&self) -> Result<&str> { + match self { + Self::Registry(dist) => dist.filename(), + Self::DirectUrl(dist) => dist.filename(), + Self::Git(dist) => dist.filename(), + } + } + + fn size(&self) -> Option { + match self { + Self::Registry(dist) => dist.size(), + Self::DirectUrl(dist) => dist.size(), + Self::Git(dist) => dist.size(), + } + } +} + +impl RemoteDistribution for BuiltDistribution { + fn filename(&self) -> Result<&str> { + match self { + Self::Registry(dist) => dist.filename(), + Self::DirectUrl(dist) => dist.filename(), + } + } + + fn size(&self) -> Option { + match self { + Self::Registry(dist) => dist.size(), + Self::DirectUrl(dist) => dist.size(), + } + } +} + +impl RemoteDistribution for Distribution { + fn filename(&self) -> Result<&str> { + match self { + Self::Built(dist) => dist.filename(), + Self::Source(dist) => dist.filename(), + } + } + + fn size(&self) -> Option { + match self { + Self::Built(dist) => dist.size(), + Self::Source(dist) => dist.size(), + } + } +} + +impl DistributionIdentifier for Url { + fn distribution_id(&self) -> String { + puffin_cache::digest(&puffin_cache::CanonicalUrl::new(self)) + } + + fn resource_id(&self) -> String { + puffin_cache::digest(&puffin_cache::RepositoryUrl::new(self)) + } +} + +impl DistributionIdentifier for File { + fn distribution_id(&self) -> String { + self.hashes.sha256.clone() + } + + fn resource_id(&self) -> String { + self.hashes.sha256.clone() + } +} + +impl DistributionIdentifier for RegistryBuiltDistribution { + fn distribution_id(&self) -> String { + self.file.distribution_id() + } + + fn resource_id(&self) -> String { + self.file.resource_id() + } +} + +impl DistributionIdentifier for RegistrySourceDistribution { + fn distribution_id(&self) -> String { + self.file.distribution_id() + } + + fn resource_id(&self) -> String { + self.file.resource_id() + } +} + +impl DistributionIdentifier for DirectUrlBuiltDistribution { + fn distribution_id(&self) -> String { + self.url.distribution_id() + } + + fn resource_id(&self) -> String { + self.url.resource_id() + } +} + +impl DistributionIdentifier for DirectUrlSourceDistribution { + fn distribution_id(&self) -> String { + self.url.distribution_id() + } + + fn resource_id(&self) -> String { + self.url.resource_id() + } +} + +impl DistributionIdentifier for GitSourceDistribution { + fn distribution_id(&self) -> String { + self.url.distribution_id() + } + + fn resource_id(&self) -> String { + self.url.resource_id() + } +} + +impl DistributionIdentifier for SourceDistribution { + fn distribution_id(&self) -> String { + match self { + Self::Registry(dist) => dist.distribution_id(), + Self::DirectUrl(dist) => dist.distribution_id(), + Self::Git(dist) => dist.distribution_id(), + } + } + + fn resource_id(&self) -> String { + match self { + Self::Registry(dist) => dist.resource_id(), + Self::DirectUrl(dist) => dist.resource_id(), + Self::Git(dist) => dist.resource_id(), + } + } +} + +impl DistributionIdentifier for BuiltDistribution { + fn distribution_id(&self) -> String { + match self { + Self::Registry(dist) => dist.distribution_id(), + Self::DirectUrl(dist) => dist.distribution_id(), + } + } + + fn resource_id(&self) -> String { + match self { + Self::Registry(dist) => dist.resource_id(), + Self::DirectUrl(dist) => dist.resource_id(), + } + } +} + +impl DistributionIdentifier for Distribution { + fn distribution_id(&self) -> String { + match self { + Self::Built(dist) => dist.distribution_id(), + Self::Source(dist) => dist.distribution_id(), + } + } + + fn resource_id(&self) -> String { + match self { + Self::Built(dist) => dist.resource_id(), + Self::Source(dist) => dist.resource_id(), } } } diff --git a/crates/puffin-distribution/src/source.rs b/crates/puffin-distribution/src/source.rs deleted file mode 100644 index 8d55e7647..000000000 --- a/crates/puffin-distribution/src/source.rs +++ /dev/null @@ -1,116 +0,0 @@ -use std::path::PathBuf; - -use anyhow::{anyhow, Error, Result}; -use url::Url; - -use puffin_git::Git; -use pypi_types::{ArchiveInfo, DirectUrl, VcsInfo, VcsKind}; - -use crate::RemoteDistributionRef; - -/// The source of a distribution. -#[derive(Debug)] -pub enum Source<'a> { - /// The distribution is available at a URL in a registry, like PyPI. - RegistryUrl(Url), - /// The distribution is available at an arbitrary remote URL, like a GitHub Release. - RemoteUrl(&'a Url, Option), - /// The distribution is available in a remote Git repository. - Git(Git, Option), -} - -impl<'a> TryFrom<&'a RemoteDistributionRef<'_>> for Source<'a> { - type Error = Error; - - fn try_from(value: &'a RemoteDistributionRef<'_>) -> Result { - match value { - // If a distribution is hosted on a registry, it must be available at a URL. - RemoteDistributionRef::Registry(_, _, file) => { - Ok(Self::RegistryUrl(Url::parse(&file.url)?)) - } - - // If a distribution is specified via a direct URL, it could be a URL to a hosted file, - // or a URL to a Git repository. - RemoteDistributionRef::Url(_, url) => Self::try_from(*url), - } - } -} - -impl<'a> TryFrom<&'a Url> for Source<'a> { - type Error = Error; - - fn try_from(url: &'a Url) -> Result { - // If the URL points to a subdirectory, extract it, as in: - // `https://git.example.com/MyProject.git@v1.0#subdirectory=pkg_dir` - // `https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` - let subdirectory = url.fragment().and_then(|fragment| { - fragment - .split('&') - .find_map(|fragment| fragment.strip_prefix("subdirectory=").map(PathBuf::from)) - }); - - // If a distribution is specified via a direct URL, it could be a URL to a hosted file, - // or a URL to a Git repository. - if let Some(url) = url.as_str().strip_prefix("git+") { - let url = Url::parse(url)?; - let git = Git::try_from(url)?; - Ok(Self::Git(git, subdirectory)) - } else { - Ok(Self::RemoteUrl(url, subdirectory)) - } - } -} - -impl From> for Url { - fn from(value: Source) -> Self { - match value { - Source::RegistryUrl(url) => url, - Source::RemoteUrl(url, subdirectory) => { - if let Some(subdirectory) = subdirectory { - let mut url = (*url).clone(); - url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); - url - } else { - url.clone() - } - } - Source::Git(git, subdirectory) => { - let mut url = Url::parse(&format!("{}{}", "git+", Url::from(git).as_str())) - .expect("git url is valid"); - if let Some(subdirectory) = subdirectory { - url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); - } - url - } - } - } -} - -impl TryFrom> for DirectUrl { - type Error = Error; - - fn try_from(value: Source<'_>) -> Result { - match value { - Source::RegistryUrl(_) => Err(anyhow!("Registry dependencies have no direct URL")), - Source::RemoteUrl(url, subdirectory) => Ok(DirectUrl::ArchiveUrl { - url: url.to_string(), - archive_info: ArchiveInfo { - hash: None, - hashes: None, - }, - subdirectory, - }), - Source::Git(git, subdirectory) => Ok(DirectUrl::VcsUrl { - url: git.url().to_string(), - vcs_info: VcsInfo { - vcs: VcsKind::Git, - // TODO(charlie): In `pip-sync`, we should `.precise` our Git dependencies, - // even though we expect it to be a no-op. - commit_id: git.precise().map(|oid| oid.to_string()), - requested_revision: git.reference().map(ToString::to_string), - }, - subdirectory, - }), - } - } -} diff --git a/crates/puffin-distribution/src/traits.rs b/crates/puffin-distribution/src/traits.rs new file mode 100644 index 000000000..71026bc10 --- /dev/null +++ b/crates/puffin-distribution/src/traits.rs @@ -0,0 +1,149 @@ +use anyhow::Result; +use puffin_cache::CanonicalUrl; +use puffin_normalize::PackageName; + +use crate::{ + AnyDistribution, BuiltDistribution, CachedDirectUrlDistribution, CachedDistribution, + CachedRegistryDistribution, DirectUrlBuiltDistribution, DirectUrlSourceDistribution, + Distribution, GitSourceDistribution, InstalledDirectUrlDistribution, InstalledDistribution, + InstalledRegistryDistribution, RegistryBuiltDistribution, RegistrySourceDistribution, + SourceDistribution, VersionOrUrl, +}; + +pub trait BaseDistribution { + /// Return the normalized [`PackageName`] of the distribution. + fn name(&self) -> &PackageName; + + /// Return a [`Version`], for registry-based distributions, or a [`Url`], for URL-based + /// distributions. + fn version_or_url(&self) -> VersionOrUrl; + + /// Returns a unique identifier for the package. + /// + /// Note that this is not equivalent to a unique identifier for the _distribution_, as multiple + /// registry-based distributions (e.g., different wheels for the same package and version) + /// will return the same package ID, but different distribution IDs. + fn package_id(&self) -> String { + match self.version_or_url() { + VersionOrUrl::Version(version) => { + // https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory + // `version` is normalized by its `ToString` impl + format!("{}-{}", self.name().as_dist_info_name(), version) + } + VersionOrUrl::Url(url) => puffin_cache::digest(&CanonicalUrl::new(url)), + } + } +} + +pub trait RemoteDistribution { + /// Return an appropriate filename for the distribution. + fn filename(&self) -> Result<&str>; + + /// Return the size of the distribution, if known. + fn size(&self) -> Option; +} + +pub trait DistributionIdentifier { + /// Return a unique resource identifier for the distribution, like a SHA-256 hash of the + /// distribution's contents. + fn distribution_id(&self) -> String; + + /// Return a unique resource identifier for the underlying resource backing the distribution. + /// + /// This is often equivalent to the distribution ID, but may differ in some cases. For example, + /// if the same Git repository is used for two different distributions, at two different + /// subdirectories or two different commits, then those distributions would share a resource ID, + /// but have different distribution IDs. + fn resource_id(&self) -> String; +} + +// Implement `Display` for all known types that implement `DistributionIdentifier`. +impl std::fmt::Display for AnyDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for BuiltDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for CachedDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for CachedDirectUrlDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for CachedRegistryDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for DirectUrlBuiltDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for DirectUrlSourceDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for Distribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for GitSourceDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for InstalledDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for InstalledDirectUrlDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for InstalledRegistryDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for RegistryBuiltDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for RegistrySourceDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} + +impl std::fmt::Display for SourceDistribution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.name(), self.version_or_url()) + } +} diff --git a/crates/puffin-git/src/lib.rs b/crates/puffin-git/src/lib.rs index 8638da3d7..8d0b4c353 100644 --- a/crates/puffin-git/src/lib.rs +++ b/crates/puffin-git/src/lib.rs @@ -7,18 +7,18 @@ mod git; mod source; mod util; -/// A reference to a Git repository. +/// A URL reference to a Git repository. #[derive(Debug, Clone)] -pub struct Git { +pub struct GitUrl { /// The URL of the Git repository, with any query parameters and fragments removed. - url: Url, + repository: Url, /// The reference to the commit to use, which could be a branch, tag or revision. reference: GitReference, /// The precise commit to use, if known. precise: Option, } -impl Git { +impl GitUrl { #[must_use] pub(crate) fn with_precise(mut self, precise: git2::Oid) -> Self { self.precise = Some(precise); @@ -26,8 +26,8 @@ impl Git { } /// Return the [`Url`] of the Git repository. - pub fn url(&self) -> &Url { - &self.url + pub fn repository(&self) -> &Url { + &self.repository } /// Return the reference to the commit to use, which could be a branch, tag or revision. @@ -49,10 +49,10 @@ impl Git { } } -impl TryFrom for Git { +impl TryFrom for GitUrl { type Error = anyhow::Error; - /// Initialize a [`Git`] source from a URL. + /// Initialize a [`GitUrl`] source from a URL. fn try_from(mut url: Url) -> Result { // Remove any query parameters and fragments. url.set_fragment(None); @@ -72,16 +72,16 @@ impl TryFrom for Git { }; Ok(Self { - url, + repository: url, reference, precise, }) } } -impl From for Url { - fn from(git: Git) -> Self { - let mut url = git.url; +impl From for Url { + fn from(git: GitUrl) -> Self { + let mut url = git.repository; // If we have a precise commit, add `@` and the commit hash to the URL. if let Some(precise) = git.precise { @@ -105,9 +105,9 @@ impl From for Url { } } -impl std::fmt::Display for Git { +impl std::fmt::Display for GitUrl { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.url) + write!(f, "{}", self.repository) } } diff --git a/crates/puffin-git/src/source.rs b/crates/puffin-git/src/source.rs index 595abbf2d..7383324df 100644 --- a/crates/puffin-git/src/source.rs +++ b/crates/puffin-git/src/source.rs @@ -11,12 +11,12 @@ use url::Url; use puffin_cache::{digest, RepositoryUrl}; use crate::git::GitRemote; -use crate::{FetchStrategy, Git}; +use crate::{FetchStrategy, GitUrl}; /// A remote Git source that can be checked out locally. pub struct GitSource { /// The Git reference from the manifest file. - git: Git, + git: GitUrl, /// The HTTP client to use for fetching. client: Client, /// The fetch strategy to use when cloning. @@ -29,7 +29,7 @@ pub struct GitSource { impl GitSource { /// Initialize a new Git source. - pub fn new(git: Git, cache: impl Into) -> Self { + pub fn new(git: GitUrl, cache: impl Into) -> Self { Self { git, client: Client::new(), @@ -51,10 +51,10 @@ impl GitSource { /// Fetch the underlying Git repository at the given revision. pub fn fetch(self) -> Result { // The path to the repo, within the Git database. - let ident = digest(&RepositoryUrl::new(&self.git.url)); + let ident = digest(&RepositoryUrl::new(&self.git.repository)); let db_path = self.cache.join("db").join(&ident); - let remote = GitRemote::new(&self.git.url); + let remote = GitRemote::new(&self.git.repository); let (db, actual_rev, task) = match (self.git.precise, remote.db_at(&db_path).ok()) { // If we have a locked revision, and we have a preexisting database // which has that revision, then no update needs to happen. @@ -65,7 +65,7 @@ impl GitSource { // situation that we have a locked revision but the database // doesn't have it. (locked_rev, db) => { - debug!("Updating git source `{:?}`", self.git.url); + debug!("Updating git source `{:?}`", self.git.repository); // Report the checkout operation to the reporter. let task = self.reporter.as_ref().map(|reporter| { @@ -114,13 +114,13 @@ impl GitSource { } pub struct Fetch { - /// The [`Git`] reference that was fetched. - git: Git, + /// The [`GitUrl`] reference that was fetched. + git: GitUrl, /// The path to the checked out repository. path: PathBuf, } -impl From for Git { +impl From for GitUrl { fn from(fetch: Fetch) -> Self { fetch.git } diff --git a/crates/puffin-installer/src/builder.rs b/crates/puffin-installer/src/builder.rs index 197e5ede2..f7894ab32 100644 --- a/crates/puffin-installer/src/builder.rs +++ b/crates/puffin-installer/src/builder.rs @@ -8,10 +8,10 @@ use anyhow::Result; use fs_err::tokio as fs; use tracing::debug; -use puffin_distribution::RemoteDistribution; +use puffin_distribution::{BaseDistribution, Distribution, RemoteDistribution}; use puffin_traits::BuildContext; -use crate::downloader::{DiskWheel, SourceDistribution, Wheel}; +use crate::downloader::{DiskWheel, SourceDistributionDownload, WheelDownload}; const BUILT_WHEELS_CACHE: &str = "built-wheels-v0"; @@ -39,12 +39,14 @@ impl<'a, T: BuildContext + Send + Sync> Builder<'a, T> { } /// Build a set of source distributions. - pub async fn build(&self, distributions: Vec) -> Result> { + pub async fn build( + &self, + distributions: Vec, + ) -> Result> { // Sort the distributions by size. let mut distributions = distributions; - distributions.sort_unstable_by_key(|distribution| match &distribution.remote { - RemoteDistribution::Registry(_package, _version, file) => Reverse(file.size), - RemoteDistribution::Url(_, _) => Reverse(usize::MIN), + distributions.sort_unstable_by_key(|distribution| { + Reverse(distribution.remote.size().unwrap_or(usize::MAX)) }); // Build the distributions serially. @@ -71,14 +73,14 @@ impl<'a, T: BuildContext + Send + Sync> Builder<'a, T> { /// Build a source distribution into a wheel. async fn build_sdist( - distribution: SourceDistribution, + distribution: SourceDistributionDownload, build_context: &T, -) -> Result { +) -> Result { // Create a directory for the wheel. let wheel_dir = build_context .cache() .join(BUILT_WHEELS_CACHE) - .join(distribution.remote.id()); + .join(distribution.remote.package_id()); fs::create_dir_all(&wheel_dir).await?; // Build the wheel. @@ -95,7 +97,7 @@ async fn build_sdist( .await?; let wheel_filename = wheel_dir.join(disk_filename); - Ok(Wheel::Disk(DiskWheel { + Ok(WheelDownload::Disk(DiskWheel { remote: distribution.remote, path: wheel_filename, })) @@ -103,7 +105,7 @@ async fn build_sdist( pub trait Reporter: Send + Sync { /// Callback to invoke when a source distribution is built. - fn on_progress(&self, wheel: &RemoteDistribution); + fn on_progress(&self, distribution: &Distribution); /// Callback to invoke when the operation is complete. fn on_complete(&self); diff --git a/crates/puffin-installer/src/cache.rs b/crates/puffin-installer/src/cache.rs index b5b5c5e54..919c81265 100644 --- a/crates/puffin-installer/src/cache.rs +++ b/crates/puffin-installer/src/cache.rs @@ -1,7 +1,8 @@ use std::path::{Path, PathBuf}; use fs_err as fs; -use puffin_distribution::RemoteDistribution; + +use puffin_distribution::{BaseDistribution, BuiltDistribution, Distribution, SourceDistribution}; static WHEEL_CACHE: &str = "wheels-v0"; @@ -23,11 +24,11 @@ impl WheelCache { fs::create_dir_all(&self.root) } - /// Return the path at which a given [`RemoteDistribution`] would be stored. - pub(crate) fn entry(&self, distribution: &RemoteDistribution) -> PathBuf { + /// Return the path at which a given [`Distribution`] would be stored. + pub(crate) fn entry(&self, distribution: &Distribution) -> PathBuf { self.root .join(CacheShard::from(distribution).segment()) - .join(distribution.id()) + .join(distribution.package_id()) } /// Returns a handle to the wheel cache directory. @@ -57,11 +58,14 @@ impl CacheShard { } } -impl From<&RemoteDistribution> for CacheShard { - fn from(distribution: &RemoteDistribution) -> Self { +impl From<&Distribution> for CacheShard { + fn from(distribution: &Distribution) -> Self { match distribution { - RemoteDistribution::Registry(_, _, _) => Self::Registry, - RemoteDistribution::Url(_, _) => Self::Url, + Distribution::Built(BuiltDistribution::Registry(_)) => Self::Registry, + Distribution::Built(BuiltDistribution::DirectUrl(_)) => Self::Url, + Distribution::Source(SourceDistribution::Registry(_)) => Self::Registry, + Distribution::Source(SourceDistribution::DirectUrl(_)) => Self::Url, + Distribution::Source(SourceDistribution::Git(_)) => Self::Url, } } } diff --git a/crates/puffin-installer/src/downloader.rs b/crates/puffin-installer/src/downloader.rs index dab88ee60..da0869a86 100644 --- a/crates/puffin-installer/src/downloader.rs +++ b/crates/puffin-installer/src/downloader.rs @@ -10,8 +10,10 @@ use tracing::debug; use url::Url; use puffin_client::RegistryClient; -use puffin_distribution::source::Source; -use puffin_distribution::{RemoteDistribution, RemoteDistributionRef}; +use puffin_distribution::direct_url::{DirectArchiveUrl, DirectGitUrl}; +use puffin_distribution::{ + BuiltDistribution, Distribution, RemoteDistribution, SourceDistribution, +}; use puffin_git::GitSource; use crate::locks::Locks; @@ -55,19 +57,18 @@ impl<'a> Downloader<'a> { } /// Download a set of distributions. - pub async fn download(&self, distributions: Vec) -> Result> { + pub async fn download(&self, distributions: Vec) -> Result> { // Sort the distributions by size. let mut distributions = distributions; - distributions.sort_unstable_by_key(|wheel| match wheel { - RemoteDistribution::Registry(_package, _version, file) => Reverse(file.size), - RemoteDistribution::Url(_, _) => Reverse(usize::MIN), + distributions.sort_unstable_by_key(|distribution| { + Reverse(distribution.size().unwrap_or(usize::MAX)) }); // Fetch the distributions in parallel. let mut fetches = JoinSet::new(); let mut downloads = Vec::with_capacity(distributions.len()); for distribution in distributions { - if self.no_build && !distribution.is_wheel() { + if self.no_build && matches!(distribution, Distribution::Source(_)) { bail!( "Building source distributions is disabled, not downloading {}", distribution @@ -86,7 +87,7 @@ impl<'a> Downloader<'a> { let result = result?; if let Some(reporter) = self.reporter.as_ref() { - reporter.on_download_progress(result.remote()); + reporter.on_download_progress(&result); } downloads.push(result); @@ -102,132 +103,138 @@ impl<'a> Downloader<'a> { /// Download a built distribution (wheel) or source distribution (sdist). async fn fetch_distribution( - distribution: RemoteDistribution, + distribution: Distribution, client: RegistryClient, cache: PathBuf, locks: Arc, ) -> Result { - let url = distribution.url()?; - let lock = locks.acquire(&url).await; + let lock = locks.acquire(&distribution).await; let _guard = lock.lock().await; - if distribution.is_wheel() { - match &distribution { - RemoteDistribution::Registry(.., file) => { - // Fetch the wheel. - let url = Url::parse(&file.url)?; - let reader = client.stream_external(&url).await?; + match &distribution { + Distribution::Built(BuiltDistribution::Registry(wheel)) => { + // Fetch the wheel. + let url = Url::parse(&wheel.file.url)?; + let reader = client.stream_external(&url).await?; - // If the file is greater than 5MB, write it to disk; otherwise, keep it in memory. - let file_size = ByteSize::b(file.size as u64); - if file_size >= ByteSize::mb(5) { - debug!("Fetching disk-based wheel from registry: {distribution} ({file_size})"); - - // Download the wheel to a temporary file. - let temp_dir = tempfile::tempdir_in(cache)?.into_path(); - let wheel_filename = distribution.filename()?; - let wheel_file = temp_dir.join(wheel_filename.as_ref()); - let mut writer = tokio::fs::File::create(&wheel_file).await?; - tokio::io::copy(&mut reader.compat(), &mut writer).await?; - - Ok(Download::Wheel(Wheel::Disk(DiskWheel { - remote: distribution, - path: wheel_file, - }))) - } else { - debug!("Fetching in-memory wheel from registry: {distribution} ({file_size})"); - - // Read into a buffer. - let mut buffer = Vec::with_capacity(file.size); - let mut reader = tokio::io::BufReader::new(reader.compat()); - tokio::io::copy(&mut reader, &mut buffer).await?; - - Ok(Download::Wheel(Wheel::InMemory(InMemoryWheel { - remote: distribution, - buffer, - }))) - } - } - RemoteDistribution::Url(.., url) => { - debug!("Fetching disk-based wheel from URL: {url}"); - - // Fetch the wheel. - let reader = client.stream_external(url).await?; + // If the file is greater than 5MB, write it to disk; otherwise, keep it in memory. + let file_size = ByteSize::b(wheel.file.size as u64); + if file_size >= ByteSize::mb(5) { + debug!("Fetching disk-based wheel from registry: {distribution} ({file_size})"); // Download the wheel to a temporary file. let temp_dir = tempfile::tempdir_in(cache)?.into_path(); - let wheel_filename = distribution.filename()?; - let wheel_file = temp_dir.join(wheel_filename.as_ref()); + let wheel_filename = &wheel.file.filename; + let wheel_file = temp_dir.join(wheel_filename); let mut writer = tokio::fs::File::create(&wheel_file).await?; tokio::io::copy(&mut reader.compat(), &mut writer).await?; - Ok(Download::Wheel(Wheel::Disk(DiskWheel { + Ok(Download::Wheel(WheelDownload::Disk(DiskWheel { remote: distribution, path: wheel_file, }))) + } else { + debug!("Fetching in-memory wheel from registry: {distribution} ({file_size})"); + + // Read into a buffer. + let mut buffer = Vec::with_capacity(wheel.file.size); + let mut reader = tokio::io::BufReader::new(reader.compat()); + tokio::io::copy(&mut reader, &mut buffer).await?; + + Ok(Download::Wheel(WheelDownload::InMemory(InMemoryWheel { + remote: distribution, + buffer, + }))) } } - } else { - let distribution_ref = RemoteDistributionRef::from(&distribution); - let source = Source::try_from(&distribution_ref)?; - let (sdist_file, subdirectory) = match source { - Source::RegistryUrl(url) => { - debug!("Fetching source distribution from registry: {url}"); - let reader = client.stream_external(&url).await?; - let mut reader = tokio::io::BufReader::new(reader.compat()); + Distribution::Built(BuiltDistribution::DirectUrl(wheel)) => { + debug!("Fetching disk-based wheel from URL: {}", &wheel.url); - // Download the source distribution. - let temp_dir = tempfile::tempdir_in(cache)?.into_path(); - let sdist_filename = distribution.filename()?; - let sdist_file = temp_dir.join(sdist_filename.as_ref()); - let mut writer = tokio::fs::File::create(&sdist_file).await?; - tokio::io::copy(&mut reader, &mut writer).await?; + // Fetch the wheel. + let reader = client.stream_external(&wheel.url).await?; - // Registry dependencies can't specify a subdirectory. - let subdirectory = None; + // Download the wheel to a temporary file. + let temp_dir = tempfile::tempdir_in(cache)?.into_path(); + let wheel_filename = wheel.filename()?; + let wheel_file = temp_dir.join(wheel_filename); + let mut writer = tokio::fs::File::create(&wheel_file).await?; + tokio::io::copy(&mut reader.compat(), &mut writer).await?; - (sdist_file, subdirectory) - } - Source::RemoteUrl(url, subdirectory) => { - debug!("Fetching source distribution from URL: {url}"); + Ok(Download::Wheel(WheelDownload::Disk(DiskWheel { + remote: distribution, + path: wheel_file, + }))) + } - let reader = client.stream_external(url).await?; - let mut reader = tokio::io::BufReader::new(reader.compat()); + Distribution::Source(SourceDistribution::Registry(sdist)) => { + debug!( + "Fetching source distribution from registry: {}", + &sdist.file.url + ); - // Download the source distribution. - let temp_dir = tempfile::tempdir_in(cache)?.into_path(); - let sdist_filename = distribution.filename()?; - let sdist_file = temp_dir.join(sdist_filename.as_ref()); - let mut writer = tokio::fs::File::create(&sdist_file).await?; - tokio::io::copy(&mut reader, &mut writer).await?; + let url = Url::parse(&sdist.file.url)?; + let reader = client.stream_external(&url).await?; - (sdist_file, subdirectory) - } - Source::Git(git, subdirectory) => { - debug!("Fetching source distribution from Git: {git}"); + // Download the source distribution. + let temp_dir = tempfile::tempdir_in(cache)?.into_path(); + let sdist_filename = sdist.filename()?; + let sdist_file = temp_dir.join(sdist_filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader.compat(), &mut writer).await?; - let git_dir = cache.join(GIT_CACHE); - let source = GitSource::new(git, git_dir); - let sdist_file = tokio::task::spawn_blocking(move || source.fetch()) - .await?? - .into(); + Ok(Download::SourceDistribution(SourceDistributionDownload { + remote: distribution, + sdist_file, + subdirectory: None, + })) + } - (sdist_file, subdirectory) - } - }; + Distribution::Source(SourceDistribution::DirectUrl(sdist)) => { + debug!("Fetching source distribution from URL: {}", sdist.url); - Ok(Download::SourceDistribution(SourceDistribution { - remote: distribution, - sdist_file, - subdirectory, - })) + let DirectArchiveUrl { url, subdirectory } = DirectArchiveUrl::from(&sdist.url); + + let reader = client.stream_external(&url).await?; + let mut reader = tokio::io::BufReader::new(reader.compat()); + + // Download the source distribution. + let temp_dir = tempfile::tempdir_in(cache)?.into_path(); + let sdist_filename = sdist.filename()?; + let sdist_file = temp_dir.join(sdist_filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader, &mut writer).await?; + + Ok(Download::SourceDistribution(SourceDistributionDownload { + remote: distribution, + sdist_file, + subdirectory, + })) + } + + Distribution::Source(SourceDistribution::Git(sdist)) => { + debug!("Fetching source distribution from Git: {}", sdist.url); + + let DirectGitUrl { url, subdirectory } = DirectGitUrl::try_from(&sdist.url)?; + + let git_dir = cache.join(GIT_CACHE); + let source = GitSource::new(url, git_dir); + let sdist_file = tokio::task::spawn_blocking(move || source.fetch()) + .await?? + .into(); + + Ok(Download::SourceDistribution(SourceDistributionDownload { + remote: distribution, + sdist_file, + subdirectory, + })) + } } } pub trait Reporter: Send + Sync { /// Callback to invoke when a wheel is downloaded. - fn on_download_progress(&self, wheel: &RemoteDistribution); + fn on_download_progress(&self, download: &Download); /// Callback to invoke when the operation is complete. fn on_download_complete(&self); @@ -236,8 +243,8 @@ pub trait Reporter: Send + Sync { /// A downloaded wheel that's stored in-memory. #[derive(Debug)] pub struct InMemoryWheel { - /// The remote file from which this wheel was downloaded. - pub(crate) remote: RemoteDistribution, + /// The remote distribution from which this wheel was downloaded. + pub(crate) remote: Distribution, /// The contents of the wheel. pub(crate) buffer: Vec, } @@ -245,71 +252,67 @@ pub struct InMemoryWheel { /// A downloaded wheel that's stored on-disk. #[derive(Debug)] pub struct DiskWheel { - /// The remote file from which this wheel was downloaded. - pub(crate) remote: RemoteDistribution, + /// The remote distribution from which this wheel was downloaded. + pub(crate) remote: Distribution, /// The path to the downloaded wheel. pub(crate) path: PathBuf, } /// A downloaded wheel. #[derive(Debug)] -pub enum Wheel { +pub enum WheelDownload { InMemory(InMemoryWheel), Disk(DiskWheel), } -impl Wheel { - /// Return the [`RemoteDistribution`] from which this wheel was downloaded. - pub fn remote(&self) -> &RemoteDistribution { +impl WheelDownload { + /// Return the [`Distribution`] from which this wheel was downloaded. + pub fn remote(&self) -> &Distribution { match self { - Wheel::InMemory(wheel) => &wheel.remote, - Wheel::Disk(wheel) => &wheel.remote, + WheelDownload::InMemory(wheel) => &wheel.remote, + WheelDownload::Disk(wheel) => &wheel.remote, } } } -impl std::fmt::Display for Wheel { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.remote()) - } -} - /// A downloaded source distribution. #[derive(Debug, Clone)] -pub struct SourceDistribution { - /// The remote file from which this wheel was downloaded. - pub(crate) remote: RemoteDistribution, +pub struct SourceDistributionDownload { + /// The remote distribution from which this source distribution was downloaded. + pub(crate) remote: Distribution, /// The path to the downloaded archive or directory. pub(crate) sdist_file: PathBuf, /// The subdirectory within the archive or directory. pub(crate) subdirectory: Option, } -impl std::fmt::Display for SourceDistribution { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.remote) - } -} - /// A downloaded distribution, either a wheel or a source distribution. #[derive(Debug)] pub enum Download { - Wheel(Wheel), - SourceDistribution(SourceDistribution), -} - -impl Download { - /// Return the [`RemoteDistribution`] from which this distribution was downloaded. - pub fn remote(&self) -> &RemoteDistribution { - match self { - Download::Wheel(distribution) => distribution.remote(), - Download::SourceDistribution(distribution) => &distribution.remote, - } - } + Wheel(WheelDownload), + SourceDistribution(SourceDistributionDownload), } impl std::fmt::Display for Download { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.remote()) + match self { + Download::Wheel(wheel) => write!(f, "{wheel}"), + Download::SourceDistribution(sdist) => write!(f, "{sdist}"), + } + } +} + +impl std::fmt::Display for WheelDownload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WheelDownload::InMemory(wheel) => write!(f, "{}", wheel.remote), + WheelDownload::Disk(wheel) => write!(f, "{}", wheel.remote), + } + } +} + +impl std::fmt::Display for SourceDistributionDownload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.remote) } } diff --git a/crates/puffin-installer/src/installer.rs b/crates/puffin-installer/src/installer.rs index 4cf19ea04..99ca27237 100644 --- a/crates/puffin-installer/src/installer.rs +++ b/crates/puffin-installer/src/installer.rs @@ -1,10 +1,8 @@ use anyhow::{Context, Error, Result}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use puffin_distribution::source::Source; use puffin_distribution::CachedDistribution; use puffin_interpreter::Virtualenv; -use pypi_types::DirectUrl; pub struct Installer<'a> { venv: &'a Virtualenv, @@ -49,7 +47,12 @@ impl<'a> Installer<'a> { install_wheel_rs::linker::install_wheel( &location, wheel.path(), - direct_url(wheel)?.as_ref(), + wheel + .direct_url()? + .as_ref() + .map(pypi_types::DirectUrl::try_from) + .transpose()? + .as_ref(), self.link_mode, ) .with_context(|| format!("Failed to install: {wheel}"))?; @@ -64,17 +67,6 @@ impl<'a> Installer<'a> { } } -/// Return the [`DirectUrl`] for a wheel, if applicable. -/// -/// TODO(charlie): This shouldn't be in `puffin-installer`. -fn direct_url(wheel: &CachedDistribution) -> Result> { - let CachedDistribution::Url(_, url, _) = wheel else { - return Ok(None); - }; - let source = Source::try_from(url)?; - DirectUrl::try_from(source).map(Some) -} - pub trait Reporter: Send + Sync { /// Callback to invoke when a dependency is resolved. fn on_install_progress(&self, wheel: &CachedDistribution); diff --git a/crates/puffin-installer/src/locks.rs b/crates/puffin-installer/src/locks.rs index 40cf48671..8f73d4126 100644 --- a/crates/puffin-installer/src/locks.rs +++ b/crates/puffin-installer/src/locks.rs @@ -1,8 +1,9 @@ -use fxhash::FxHashMap; -use puffin_cache::RepositoryUrl; use std::sync::Arc; + +use fxhash::FxHashMap; use tokio::sync::Mutex; -use url::Url; + +use puffin_distribution::DistributionIdentifier; /// A set of locks used to prevent concurrent access to the same resource. #[derive(Debug, Default)] @@ -10,9 +11,12 @@ pub(crate) struct Locks(Mutex>>>); impl Locks { /// Acquire a lock on the given resource. - pub(crate) async fn acquire(&self, url: &Url) -> Arc> { + pub(crate) async fn acquire( + &self, + distribution: &impl DistributionIdentifier, + ) -> Arc> { let mut map = self.0.lock().await; - map.entry(puffin_cache::digest(&RepositoryUrl::new(url))) + map.entry(distribution.resource_id()) .or_insert_with(|| Arc::new(Mutex::new(()))) .clone() } diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs index d00860520..307096186 100644 --- a/crates/puffin-installer/src/plan.rs +++ b/crates/puffin-installer/src/plan.rs @@ -4,10 +4,9 @@ use anyhow::{Context, Result}; use tracing::debug; use pep508_rs::{Requirement, VersionOrUrl}; -use puffin_distribution::source::Source; +use puffin_distribution::direct_url::DirectUrl; use puffin_distribution::{CachedDistribution, InstalledDistribution}; use puffin_interpreter::Virtualenv; -use pypi_types::DirectUrl; use crate::url_index::UrlIndex; use crate::{RegistryIndex, SitePackages}; @@ -63,12 +62,13 @@ impl InstallPlan { // If the requirement comes from a direct URL, check by URL. Some(VersionOrUrl::Url(url)) => { - if let Ok(Some(direct_url)) = distribution.direct_url() { - if let Ok(source) = Source::try_from(url) { - if let Ok(target) = DirectUrl::try_from(source) { + if let InstalledDistribution::Url(distribution) = &distribution { + if let Ok(direct_url) = DirectUrl::try_from(url) { + if let Ok(direct_url) = pypi_types::DirectUrl::try_from(&direct_url) + { // TODO(charlie): These don't need to be strictly equal. We only care // about a subset of the fields. - if target == direct_url { + if direct_url == distribution.url { debug!("Requirement already satisfied: {distribution}"); continue; } @@ -84,23 +84,19 @@ impl InstallPlan { // Identify any locally-available distributions that satisfy the requirement. match requirement.version_or_url.as_ref() { None | Some(VersionOrUrl::VersionSpecifier(_)) => { - if let Some(distribution) = - registry_index.get(&requirement.name).filter(|dist| { - let CachedDistribution::Registry(_name, version, _path) = dist else { - return false; - }; - requirement.is_satisfied_by(version) - }) + if let Some(distribution) = registry_index + .get(&requirement.name) + .filter(|dist| requirement.is_satisfied_by(&dist.version)) { debug!("Requirement already cached: {distribution}"); - local.push(distribution.clone()); + local.push(CachedDistribution::Registry(distribution.clone())); continue; } } Some(VersionOrUrl::Url(url)) => { if let Some(distribution) = url_index.get(&requirement.name, url) { debug!("Requirement already cached: {distribution}"); - local.push(distribution.clone()); + local.push(CachedDistribution::Url(distribution.clone())); continue; } } diff --git a/crates/puffin-installer/src/registry_index.rs b/crates/puffin-installer/src/registry_index.rs index bd0b9eb8d..117d1bc0f 100644 --- a/crates/puffin-installer/src/registry_index.rs +++ b/crates/puffin-installer/src/registry_index.rs @@ -4,14 +4,14 @@ use std::path::Path; use fs_err as fs; use tracing::warn; -use puffin_distribution::CachedDistribution; +use puffin_distribution::{BaseDistribution, CachedRegistryDistribution}; use puffin_normalize::PackageName; use crate::cache::{CacheShard, WheelCache}; /// A local index of distributions that originate from a registry, like `PyPI`. #[derive(Debug, Default)] -pub struct RegistryIndex(HashMap); +pub struct RegistryIndex(HashMap); impl RegistryIndex { /// Build an index of cached distributions from a directory. @@ -37,7 +37,7 @@ impl RegistryIndex { } }; if file_type.is_dir() { - match CachedDistribution::try_from_path(&path) { + match CachedRegistryDistribution::try_from_path(&path) { Ok(None) => {} Ok(Some(dist_info)) => { index.insert(dist_info.name().clone(), dist_info); @@ -60,7 +60,7 @@ impl RegistryIndex { } /// Returns a distribution from the index, if it exists. - pub fn get(&self, name: &PackageName) -> Option<&CachedDistribution> { + pub fn get(&self, name: &PackageName) -> Option<&CachedRegistryDistribution> { self.0.get(name) } } diff --git a/crates/puffin-installer/src/site_packages.rs b/crates/puffin-installer/src/site_packages.rs index 060614b81..cd9b48fc3 100644 --- a/crates/puffin-installer/src/site_packages.rs +++ b/crates/puffin-installer/src/site_packages.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use anyhow::Result; use fs_err as fs; -use puffin_distribution::InstalledDistribution; +use puffin_distribution::{BaseDistribution, InstalledDistribution}; use puffin_interpreter::Virtualenv; use puffin_normalize::PackageName; diff --git a/crates/puffin-installer/src/unzipper.rs b/crates/puffin-installer/src/unzipper.rs index fc6c9ef3b..ceed33ed4 100644 --- a/crates/puffin-installer/src/unzipper.rs +++ b/crates/puffin-installer/src/unzipper.rs @@ -8,10 +8,10 @@ use rayon::iter::ParallelIterator; use tracing::debug; use zip::ZipArchive; -use puffin_distribution::{CachedDistribution, RemoteDistribution}; +use puffin_distribution::{CachedDistribution, Distribution, DistributionIdentifier}; use crate::cache::WheelCache; -use crate::downloader::Wheel; +use crate::downloader::WheelDownload; use crate::vendor::{CloneableSeekableReader, HasLength}; #[derive(Default)] @@ -31,7 +31,7 @@ impl Unzipper { /// Unzip a set of downloaded wheels. pub async fn unzip( &self, - downloads: Vec, + downloads: Vec, target: &Path, ) -> Result> { // Create the wheel cache subdirectory, if necessary. @@ -41,8 +41,8 @@ impl Unzipper { // Sort the wheels by size. let mut downloads = downloads; downloads.sort_unstable_by_key(|wheel| match wheel { - Wheel::Disk(_) => Reverse(usize::MIN), - Wheel::InMemory(wheel) => Reverse(wheel.buffer.len()), + WheelDownload::Disk(_) => Reverse(usize::MIN), + WheelDownload::InMemory(wheel) => Reverse(wheel.buffer.len()), }); let staging = tempfile::tempdir_in(wheel_cache.root())?; @@ -56,7 +56,7 @@ impl Unzipper { // Unzip the wheel. tokio::task::spawn_blocking({ - let target = staging.path().join(remote.id()); + let target = staging.path().join(remote.distribution_id()); move || unzip_wheel(download, &target) }) .await??; @@ -66,7 +66,8 @@ impl Unzipper { if let Some(parent) = target.parent() { fs_err::create_dir_all(parent)?; } - let result = fs_err::tokio::rename(staging.path().join(remote.id()), target).await; + let result = + fs_err::tokio::rename(staging.path().join(remote.distribution_id()), target).await; if let Err(err) = result { // If the renaming failed because another instance was faster, that's fine @@ -93,10 +94,10 @@ impl Unzipper { } /// Unzip a wheel into the target directory. -fn unzip_wheel(wheel: Wheel, target: &Path) -> Result<()> { +fn unzip_wheel(wheel: WheelDownload, target: &Path) -> Result<()> { match wheel { - Wheel::InMemory(wheel) => unzip_archive(std::io::Cursor::new(wheel.buffer), target), - Wheel::Disk(wheel) => unzip_archive(fs_err::File::open(wheel.path)?, target), + WheelDownload::InMemory(wheel) => unzip_archive(std::io::Cursor::new(wheel.buffer), target), + WheelDownload::Disk(wheel) => unzip_archive(fs_err::File::open(wheel.path)?, target), } } @@ -148,7 +149,7 @@ fn unzip_archive(reader: R, target: &Path) -> pub trait Reporter: Send + Sync { /// Callback to invoke when a wheel is unzipped. - fn on_unzip_progress(&self, wheel: &RemoteDistribution); + fn on_unzip_progress(&self, distribution: &Distribution); /// Callback to invoke when the operation is complete. fn on_unzip_complete(&self); diff --git a/crates/puffin-installer/src/url_index.rs b/crates/puffin-installer/src/url_index.rs index f8db051c9..438d420ca 100644 --- a/crates/puffin-installer/src/url_index.rs +++ b/crates/puffin-installer/src/url_index.rs @@ -4,7 +4,7 @@ use fxhash::FxHashMap; use tracing::warn; use url::Url; -use puffin_distribution::{CachedDistribution, RemoteDistributionRef}; +use puffin_distribution::{CachedDirectUrlDistribution, DistributionIdentifier}; use puffin_normalize::PackageName; use crate::cache::{CacheShard, WheelCache}; @@ -49,10 +49,12 @@ impl UrlIndex { } /// Returns a distribution from the index, if it exists. - pub(crate) fn get(&self, name: &PackageName, url: &Url) -> Option { - let distribution = RemoteDistributionRef::from_url(name, url); - let path = self.0.get(&distribution.id())?; - Some(CachedDistribution::Url( + pub(crate) fn get(&self, name: &PackageName, url: &Url) -> Option { + // TODO(charlie): This takes advantage of the fact that for URL dependencies, the package ID + // and distribution ID are identical. We should either change the cache layout to use + // distribution IDs, or implement package ID for URL. + let path = self.0.get(&url.distribution_id())?; + Some(CachedDirectUrlDistribution::from_url( name.clone(), url.clone(), path.clone(), diff --git a/crates/puffin-resolver/src/distribution/wheel.rs b/crates/puffin-resolver/src/distribution/built_distribution.rs similarity index 69% rename from crates/puffin-resolver/src/distribution/wheel.rs rename to crates/puffin-resolver/src/distribution/built_distribution.rs index c623eace2..b54f75e42 100644 --- a/crates/puffin-resolver/src/distribution/wheel.rs +++ b/crates/puffin-resolver/src/distribution/built_distribution.rs @@ -3,14 +3,13 @@ use std::str::FromStr; use anyhow::{Context, Result}; use fs_err::tokio as fs; - use tokio_util::compat::FuturesAsyncReadCompatExt; use tracing::debug; use distribution_filename::WheelFilename; use platform_tags::Tags; use puffin_client::RegistryClient; -use puffin_distribution::RemoteDistributionRef; +use puffin_distribution::{DirectUrlBuiltDistribution, DistributionIdentifier, RemoteDistribution}; use pypi_types::Metadata21; use crate::distribution::cached_wheel::CachedWheel; @@ -18,10 +17,10 @@ use crate::distribution::cached_wheel::CachedWheel; const REMOTE_WHEELS_CACHE: &str = "remote-wheels-v0"; /// Fetch a built distribution from a remote source, or from a local cache. -pub(crate) struct WheelFetcher<'a>(&'a Path); +pub(crate) struct BuiltDistributionFetcher<'a>(&'a Path); -impl<'a> WheelFetcher<'a> { - /// Initialize a [`WheelFetcher`] from a [`BuildContext`]. +impl<'a> BuiltDistributionFetcher<'a> { + /// Initialize a [`BuiltDistributionFetcher`] from a [`BuildContext`]. pub(crate) fn new(cache: &'a Path) -> Self { Self(cache) } @@ -29,7 +28,7 @@ impl<'a> WheelFetcher<'a> { /// Read the [`Metadata21`] from a wheel, if it exists in the cache. pub(crate) fn find_dist_info( &self, - distribution: &RemoteDistributionRef<'_>, + distribution: &DirectUrlBuiltDistribution, tags: &Tags, ) -> Result> { CachedWheel::find_in_cache(distribution, tags, self.0.join(REMOTE_WHEELS_CACHE)) @@ -41,26 +40,27 @@ impl<'a> WheelFetcher<'a> { /// Download a wheel, storing it in the cache. pub(crate) async fn download_wheel( &self, - distribution: &RemoteDistributionRef<'_>, + distribution: &DirectUrlBuiltDistribution, client: &RegistryClient, ) -> Result { debug!("Downloading: {distribution}"); - let url = distribution.url()?; - let reader = client.stream_external(&url).await?; - let mut reader = tokio::io::BufReader::new(reader.compat()); + let reader = client.stream_external(&distribution.url).await?; // Create a directory for the wheel. - let wheel_dir = self.0.join(REMOTE_WHEELS_CACHE).join(distribution.id()); + let wheel_dir = self + .0 + .join(REMOTE_WHEELS_CACHE) + .join(distribution.distribution_id()); fs::create_dir_all(&wheel_dir).await?; // Download the wheel. let wheel_filename = distribution.filename()?; - let wheel_file = wheel_dir.join(wheel_filename.as_ref()); + let wheel_file = wheel_dir.join(wheel_filename); let mut writer = tokio::fs::File::create(&wheel_file).await?; - tokio::io::copy(&mut reader, &mut writer).await?; + tokio::io::copy(&mut reader.compat(), &mut writer).await?; // Read the metadata from the wheel. - let wheel = CachedWheel::new(wheel_file, WheelFilename::from_str(&wheel_filename)?); + let wheel = CachedWheel::new(wheel_file, WheelFilename::from_str(wheel_filename)?); let metadata21 = wheel.read_dist_info()?; debug!("Finished downloading: {distribution}"); diff --git a/crates/puffin-resolver/src/distribution/cached_wheel.rs b/crates/puffin-resolver/src/distribution/cached_wheel.rs index 97ee31582..16538862b 100644 --- a/crates/puffin-resolver/src/distribution/cached_wheel.rs +++ b/crates/puffin-resolver/src/distribution/cached_wheel.rs @@ -7,7 +7,7 @@ use zip::ZipArchive; use distribution_filename::WheelFilename; use install_wheel_rs::find_dist_info; use platform_tags::Tags; -use puffin_distribution::RemoteDistributionRef; +use puffin_distribution::DistributionIdentifier; use pypi_types::Metadata21; /// A cached wheel built from a remote source. @@ -23,12 +23,12 @@ impl CachedWheel { } /// Search for a wheel matching the tags that was built from the given distribution. - pub(super) fn find_in_cache( - distribution: &RemoteDistributionRef<'_>, + pub(super) fn find_in_cache( + distribution: &T, tags: &Tags, cache: impl AsRef, ) -> Option { - let wheel_dir = cache.as_ref().join(distribution.id()); + let wheel_dir = cache.as_ref().join(distribution.distribution_id()); let Ok(read_dir) = fs_err::read_dir(wheel_dir) else { return None; }; diff --git a/crates/puffin-resolver/src/distribution/mod.rs b/crates/puffin-resolver/src/distribution/mod.rs index 458c92dd5..4b59c884d 100644 --- a/crates/puffin-resolver/src/distribution/mod.rs +++ b/crates/puffin-resolver/src/distribution/mod.rs @@ -1,8 +1,8 @@ +pub(crate) use built_distribution::BuiltDistributionFetcher; pub(crate) use source_distribution::{ Reporter as SourceDistributionReporter, SourceDistributionFetcher, }; -pub(crate) use wheel::WheelFetcher; +mod built_distribution; mod cached_wheel; mod source_distribution; -mod wheel; diff --git a/crates/puffin-resolver/src/distribution/source_distribution.rs b/crates/puffin-resolver/src/distribution/source_distribution.rs index f80765d29..c49cd60de 100644 --- a/crates/puffin-resolver/src/distribution/source_distribution.rs +++ b/crates/puffin-resolver/src/distribution/source_distribution.rs @@ -7,8 +7,6 @@ use std::sync::Arc; use anyhow::{bail, Result}; use fs_err::tokio as fs; - -use tempfile::tempdir_in; use tokio_util::compat::FuturesAsyncReadCompatExt; use tracing::debug; use url::Url; @@ -16,9 +14,9 @@ use url::Url; use distribution_filename::WheelFilename; use platform_tags::Tags; use puffin_client::RegistryClient; -use puffin_distribution::source::Source; -use puffin_distribution::RemoteDistributionRef; -use puffin_git::{Git, GitSource}; +use puffin_distribution::direct_url::{DirectArchiveUrl, DirectGitUrl}; +use puffin_distribution::{DistributionIdentifier, RemoteDistribution, SourceDistribution}; +use puffin_git::{GitSource, GitUrl}; use puffin_traits::BuildContext; use pypi_types::Metadata21; @@ -55,7 +53,7 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { /// Read the [`Metadata21`] from a built source distribution, if it exists in the cache. pub(crate) fn find_dist_info( &self, - distribution: &RemoteDistributionRef<'_>, + distribution: &SourceDistribution, tags: &Tags, ) -> Result> { CachedWheel::find_in_cache( @@ -71,7 +69,7 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { /// Download and build a source distribution, storing the built wheel in the cache. pub(crate) async fn download_and_build_sdist( &self, - distribution: &RemoteDistributionRef<'_>, + distribution: &SourceDistribution, client: &RegistryClient, ) -> Result { debug!("Building: {distribution}"); @@ -80,50 +78,54 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { bail!("Building source distributions is disabled"); } - // This could extract the subdirectory. - let source = Source::try_from(distribution)?; - let (sdist_file, subdirectory) = match source { - Source::RegistryUrl(url) => { - debug!("Fetching source distribution from registry: {url}"); + let (sdist_file, subdirectory) = match distribution { + SourceDistribution::Registry(sdist) => { + debug!( + "Fetching source distribution from registry: {}", + sdist.file.url + ); + + let url = Url::parse(&sdist.file.url)?; + let reader = client.stream_external(&url).await?; + + // Download the source distribution. + let temp_dir = tempfile::tempdir_in(self.build_context.cache())?.into_path(); + let sdist_filename = sdist.filename()?; + let sdist_file = temp_dir.join(sdist_filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader.compat(), &mut writer).await?; + + (sdist_file, None) + } + + SourceDistribution::DirectUrl(sdist) => { + debug!("Fetching source distribution from URL: {}", sdist.url); + + let DirectArchiveUrl { url, subdirectory } = DirectArchiveUrl::from(&sdist.url); let reader = client.stream_external(&url).await?; let mut reader = tokio::io::BufReader::new(reader.compat()); // Download the source distribution. - let temp_dir = tempdir_in(self.build_context.cache())?.into_path(); - let sdist_filename = distribution.filename()?; - let sdist_file = temp_dir.join(sdist_filename.as_ref()); - let mut writer = tokio::fs::File::create(&sdist_file).await?; - tokio::io::copy(&mut reader, &mut writer).await?; - - // Registry dependencies can't specify a subdirectory. - let subdirectory = None; - - (sdist_file, subdirectory) - } - Source::RemoteUrl(url, subdirectory) => { - debug!("Fetching source distribution from URL: {url}"); - - let reader = client.stream_external(url).await?; - let mut reader = tokio::io::BufReader::new(reader.compat()); - - // Download the source distribution. - let temp_dir = tempdir_in(self.build_context.cache())?.into_path(); - let sdist_filename = distribution.filename()?; - let sdist_file = temp_dir.join(sdist_filename.as_ref()); + let temp_dir = tempfile::tempdir_in(self.build_context.cache())?.into_path(); + let sdist_filename = sdist.filename()?; + let sdist_file = temp_dir.join(sdist_filename); let mut writer = tokio::fs::File::create(&sdist_file).await?; tokio::io::copy(&mut reader, &mut writer).await?; (sdist_file, subdirectory) } - Source::Git(git, subdirectory) => { - debug!("Fetching source distribution from Git: {git}"); + + SourceDistribution::Git(sdist) => { + debug!("Fetching source distribution from Git: {}", sdist.url); + + let DirectGitUrl { url, subdirectory } = DirectGitUrl::try_from(&sdist.url)?; let git_dir = self.build_context.cache().join(GIT_CACHE); let source = if let Some(reporter) = &self.reporter { - GitSource::new(git, git_dir).with_reporter(Facade::from(reporter.clone())) + GitSource::new(url, git_dir).with_reporter(Facade::from(reporter.clone())) } else { - GitSource::new(git, git_dir) + GitSource::new(url, git_dir) }; let sdist_file = tokio::task::spawn_blocking(move || source.fetch()) .await?? @@ -138,7 +140,7 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { .build_context .cache() .join(BUILT_WHEELS_CACHE) - .join(distribution.id()); + .join(distribution.distribution_id()); fs::create_dir_all(&wheel_dir).await?; // Build the wheel. @@ -171,17 +173,15 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { /// This method takes into account various normalizations that are independent from the Git /// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+` /// prefix kinds. - pub(crate) async fn precise( - &self, - distribution: &RemoteDistributionRef<'_>, - ) -> Result> { - let source = Source::try_from(distribution)?; - let Source::Git(git, subdirectory) = source else { + pub(crate) async fn precise(&self, distribution: &SourceDistribution) -> Result> { + let SourceDistribution::Git(sdist) = distribution else { return Ok(None); }; + let DirectGitUrl { url, subdirectory } = DirectGitUrl::try_from(&sdist.url)?; + // If the commit already contains a complete SHA, short-circuit. - if git.precise().is_some() { + if url.precise().is_some() { return Ok(None); } @@ -189,16 +189,15 @@ impl<'a, T: BuildContext> SourceDistributionFetcher<'a, T> { // commit, etc.). let git_dir = self.build_context.cache().join(GIT_CACHE); let source = if let Some(reporter) = &self.reporter { - GitSource::new(git, git_dir).with_reporter(Facade::from(reporter.clone())) + GitSource::new(url, git_dir).with_reporter(Facade::from(reporter.clone())) } else { - GitSource::new(git, git_dir) + GitSource::new(url, git_dir) }; let precise = tokio::task::spawn_blocking(move || source.fetch()).await??; - let git = Git::from(precise); + let url = GitUrl::from(precise); // Re-encode as a URL. - let source = Source::Git(git, subdirectory); - Ok(Some(source.into())) + Ok(Some(DirectGitUrl { url, subdirectory }.into())) } } diff --git a/crates/puffin-resolver/src/error.rs b/crates/puffin-resolver/src/error.rs index 272b8e131..39d18db64 100644 --- a/crates/puffin-resolver/src/error.rs +++ b/crates/puffin-resolver/src/error.rs @@ -6,6 +6,7 @@ use thiserror::Error; use url::Url; use pep508_rs::Requirement; +use puffin_distribution::{BuiltDistribution, SourceDistribution}; use puffin_normalize::PackageName; use crate::pubgrub::{PubGrubPackage, PubGrubVersion}; @@ -46,16 +47,32 @@ pub enum ResolveError { #[error("Package `{0}` attempted to resolve via URL: {1}. URL dependencies must be expressed as direct requirements or constraints. Consider adding `{0} @ {1}` to your dependencies or constraints file.")] DisallowedUrl(PackageName, Url), - #[error("Failed to build distribution: {filename}")] - RegistryDistribution { + #[error("Failed to fetch wheel metadata from: {filename}")] + RegistryBuiltDistribution { filename: String, // TODO(konstin): Gives this a proper error type #[source] err: anyhow::Error, }, - #[error("Failed to build distribution: {url}")] - UrlDistribution { + #[error("Failed to fetch wheel metadata from: {url}")] + UrlBuiltDistribution { + url: Url, + // TODO(konstin): Gives this a proper error type + #[source] + err: anyhow::Error, + }, + + #[error("Failed to build distribution: {filename}")] + RegistrySourceDistribution { + filename: String, + // TODO(konstin): Gives this a proper error type + #[source] + err: anyhow::Error, + }, + + #[error("Failed to build distribution from URL: {url}")] + UrlSourceDistribution { url: Url, // TODO(konstin): Gives this a proper error type #[source] @@ -93,3 +110,35 @@ impl From>> f ResolveError::PubGrub(RichPubGrubError { source: value }) } } + +impl ResolveError { + pub fn from_source_distribution(distribution: SourceDistribution, err: anyhow::Error) -> Self { + match distribution { + SourceDistribution::Registry(sdist) => Self::RegistrySourceDistribution { + filename: sdist.file.filename.clone(), + err, + }, + SourceDistribution::DirectUrl(sdist) => Self::UrlSourceDistribution { + url: sdist.url.clone(), + err, + }, + SourceDistribution::Git(sdist) => Self::UrlSourceDistribution { + url: sdist.url.clone(), + err, + }, + } + } + + pub fn from_built_distribution(distribution: BuiltDistribution, err: anyhow::Error) -> Self { + match distribution { + BuiltDistribution::Registry(wheel) => Self::RegistryBuiltDistribution { + filename: wheel.file.filename.clone(), + err, + }, + BuiltDistribution::DirectUrl(wheel) => Self::UrlBuiltDistribution { + url: wheel.url.clone(), + err, + }, + } + } +} diff --git a/crates/puffin-resolver/src/file.rs b/crates/puffin-resolver/src/file.rs index dfbea37f3..bbe87249e 100644 --- a/crates/puffin-resolver/src/file.rs +++ b/crates/puffin-resolver/src/file.rs @@ -1,14 +1,13 @@ -use distribution_filename::{SourceDistributionFilename, WheelFilename}; use std::ops::Deref; use pypi_types::File; /// A distribution can either be a wheel or a source distribution. #[derive(Debug, Clone)] -pub(crate) struct WheelFile(pub(crate) File, pub(crate) WheelFilename); +pub(crate) struct WheelFile(pub(crate) File); #[derive(Debug, Clone)] -pub(crate) struct SdistFile(pub(crate) File, pub(crate) SourceDistributionFilename); +pub(crate) struct SdistFile(pub(crate) File); #[derive(Debug, Clone)] pub(crate) enum DistributionFile { diff --git a/crates/puffin-resolver/src/finder.rs b/crates/puffin-resolver/src/finder.rs index 1745cc865..41ae55621 100644 --- a/crates/puffin-resolver/src/finder.rs +++ b/crates/puffin-resolver/src/finder.rs @@ -13,7 +13,7 @@ use distribution_filename::{SourceDistributionFilename, WheelFilename}; use pep508_rs::{Requirement, VersionOrUrl}; use platform_tags::Tags; use puffin_client::RegistryClient; -use puffin_distribution::RemoteDistribution; +use puffin_distribution::Distribution; use puffin_normalize::PackageName; use pypi_types::{File, SimpleJson}; @@ -66,7 +66,7 @@ impl<'a> DistributionFinder<'a> { .ready_chunks(32); // Resolve the requirements. - let mut resolution: FxHashMap = + let mut resolution: FxHashMap = FxHashMap::with_capacity_and_hasher(requirements.len(), BuildHasherDefault::default()); // Push all the requirements into the package sink. @@ -77,7 +77,7 @@ impl<'a> DistributionFinder<'a> { } Some(VersionOrUrl::Url(url)) => { let package_name = requirement.name.clone(); - let package = RemoteDistribution::from_url(package_name.clone(), url.clone()); + let package = Distribution::from_url(package_name.clone(), url.clone()); resolution.insert(package_name, package); } } @@ -126,7 +126,7 @@ impl<'a> DistributionFinder<'a> { } /// select a version that satisfies the requirement, preferring wheels to source distributions. - fn select(&self, requirement: &Requirement, files: Vec) -> Option { + fn select(&self, requirement: &Requirement, files: Vec) -> Option { let mut fallback = None; for file in files.into_iter().rev() { if let Ok(wheel) = WheelFilename::from_str(file.filename.as_str()) { @@ -134,7 +134,7 @@ impl<'a> DistributionFinder<'a> { continue; } if requirement.is_satisfied_by(&wheel.version) { - return Some(RemoteDistribution::from_registry( + return Some(Distribution::from_registry( wheel.distribution, wheel.version, file, @@ -144,11 +144,7 @@ impl<'a> DistributionFinder<'a> { SourceDistributionFilename::parse(file.filename.as_str(), &requirement.name) { if requirement.is_satisfied_by(&sdist.version) { - fallback = Some(RemoteDistribution::from_registry( - sdist.name, - sdist.version, - file, - )); + fallback = Some(Distribution::from_registry(sdist.name, sdist.version, file)); } } } @@ -157,7 +153,6 @@ impl<'a> DistributionFinder<'a> { } #[derive(Debug)] -#[allow(clippy::large_enum_variant)] enum Request { /// A request to fetch the metadata for a package. Package(Requirement), @@ -171,7 +166,7 @@ enum Response { pub trait Reporter: Send + Sync { /// Callback to invoke when a package is resolved to a specific distribution. - fn on_progress(&self, wheel: &RemoteDistribution); + fn on_progress(&self, wheel: &Distribution); /// Callback to invoke when the resolution is complete. fn on_complete(&self); diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index b2fcc879f..ae0f0508c 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -12,6 +12,7 @@ mod distribution; mod error; mod file; mod finder; +mod locks; mod manifest; mod prerelease_mode; mod pubgrub; diff --git a/crates/puffin-resolver/src/locks.rs b/crates/puffin-resolver/src/locks.rs new file mode 100644 index 000000000..8f73d4126 --- /dev/null +++ b/crates/puffin-resolver/src/locks.rs @@ -0,0 +1,23 @@ +use std::sync::Arc; + +use fxhash::FxHashMap; +use tokio::sync::Mutex; + +use puffin_distribution::DistributionIdentifier; + +/// A set of locks used to prevent concurrent access to the same resource. +#[derive(Debug, Default)] +pub(crate) struct Locks(Mutex>>>); + +impl Locks { + /// Acquire a lock on the given resource. + pub(crate) async fn acquire( + &self, + distribution: &impl DistributionIdentifier, + ) -> Arc> { + let mut map = self.0.lock().await; + map.entry(distribution.resource_id()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } +} diff --git a/crates/puffin-resolver/src/resolution.rs b/crates/puffin-resolver/src/resolution.rs index 947c83ad2..5f93f9f74 100644 --- a/crates/puffin-resolver/src/resolution.rs +++ b/crates/puffin-resolver/src/resolution.rs @@ -11,7 +11,7 @@ use waitmap::WaitMap; use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; use pep508_rs::{Requirement, VersionOrUrl}; -use puffin_distribution::RemoteDistribution; +use puffin_distribution::{BaseDistribution, BuiltDistribution, Distribution, SourceDistribution}; use puffin_normalize::PackageName; use pypi_types::File; @@ -19,21 +19,21 @@ use crate::pubgrub::{PubGrubPackage, PubGrubPriority, PubGrubVersion}; /// A set of packages pinned at specific versions. #[derive(Debug, Default)] -pub struct Resolution(FxHashMap); +pub struct Resolution(FxHashMap); impl Resolution { /// Create a new resolution from the given pinned packages. - pub(crate) fn new(packages: FxHashMap) -> Self { + pub(crate) fn new(packages: FxHashMap) -> Self { Self(packages) } /// Return the distribution for the given package name, if it exists. - pub fn get(&self, package_name: &PackageName) -> Option<&RemoteDistribution> { + pub fn get(&self, package_name: &PackageName) -> Option<&Distribution> { self.0.get(package_name) } - /// Iterate over the [`RemoteDistribution`] entities in this resolution. - pub fn into_distributions(self) -> impl Iterator { + /// Iterate over the [`Distribution`] entities in this resolution. + pub fn into_distributions(self) -> impl Iterator { self.0.into_values() } @@ -51,7 +51,7 @@ impl Resolution { /// A complete resolution graph in which every node represents a pinned package and every edge /// represents a dependency between two pinned packages. #[derive(Debug)] -pub struct Graph(petgraph::graph::Graph); +pub struct Graph(petgraph::graph::Graph); impl Graph { /// Create a new graph from the resolved `PubGrub` state. @@ -78,7 +78,7 @@ impl Graph { .unwrap() .clone(); let pinned_package = - RemoteDistribution::from_registry(package_name.clone(), version, file); + Distribution::from_registry(package_name.clone(), version, file); let index = graph.add_node(pinned_package); inverse.insert(package_name, index); @@ -87,8 +87,7 @@ impl Graph { let url = redirects .get(url) .map_or_else(|| url.clone(), |url| url.value().clone()); - let pinned_package = - RemoteDistribution::from_url(package_name.clone(), url.clone()); + let pinned_package = Distribution::from_url(package_name.clone(), url); let index = graph.add_node(pinned_package); inverse.insert(package_name, index); @@ -144,18 +143,38 @@ impl Graph { self.0 .node_indices() .map(|node| match &self.0[node] { - RemoteDistribution::Registry(name, version, _file) => Requirement { - name: name.clone(), + Distribution::Built(BuiltDistribution::Registry(wheel)) => Requirement { + name: wheel.name.clone(), extras: None, version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from( - VersionSpecifier::equals_version(version.clone()), + VersionSpecifier::equals_version(wheel.version.clone()), ))), marker: None, }, - RemoteDistribution::Url(name, url) => Requirement { - name: name.clone(), + Distribution::Built(BuiltDistribution::DirectUrl(wheel)) => Requirement { + name: wheel.name.clone(), extras: None, - version_or_url: Some(VersionOrUrl::Url(url.clone())), + version_or_url: Some(VersionOrUrl::Url(wheel.url.clone())), + marker: None, + }, + Distribution::Source(SourceDistribution::Registry(sdist)) => Requirement { + name: sdist.name.clone(), + extras: None, + version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from( + VersionSpecifier::equals_version(sdist.version.clone()), + ))), + marker: None, + }, + Distribution::Source(SourceDistribution::DirectUrl(sdist)) => Requirement { + name: sdist.name.clone(), + extras: None, + version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())), + marker: None, + }, + Distribution::Source(SourceDistribution::Git(sdist)) => Requirement { + name: sdist.name.clone(), + extras: None, + version_or_url: Some(VersionOrUrl::Url(sdist.url.clone())), marker: None, }, }) diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver.rs index d1af6e55a..dcdb19a60 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver.rs @@ -13,7 +13,6 @@ use pubgrub::range::Range; use pubgrub::solver::{Incompatibility, State}; use pubgrub::type_aliases::DependencyConstraints; use tokio::select; -use tokio::sync::Mutex; use tracing::{debug, error, trace}; use url::Url; use waitmap::WaitMap; @@ -21,17 +20,23 @@ use waitmap::WaitMap; use distribution_filename::{SourceDistributionFilename, WheelFilename}; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; -use puffin_cache::{CanonicalUrl, RepositoryUrl}; +use puffin_cache::CanonicalUrl; use puffin_client::RegistryClient; -use puffin_distribution::{RemoteDistributionRef, VersionOrUrl}; +use puffin_distribution::{ + BaseDistribution, BuiltDistribution, DirectUrlSourceDistribution, Distribution, + DistributionIdentifier, GitSourceDistribution, SourceDistribution, VersionOrUrl, +}; use puffin_normalize::{ExtraName, PackageName}; use puffin_traits::BuildContext; use pypi_types::{File, Metadata21, SimpleJson}; use crate::candidate_selector::CandidateSelector; -use crate::distribution::{SourceDistributionFetcher, SourceDistributionReporter, WheelFetcher}; +use crate::distribution::{ + BuiltDistributionFetcher, SourceDistributionFetcher, SourceDistributionReporter, +}; use crate::error::ResolveError; use crate::file::{DistributionFile, SdistFile, WheelFile}; +use crate::locks::Locks; use crate::manifest::Manifest; use crate::pubgrub::{ PubGrubDependencies, PubGrubPackage, PubGrubPriorities, PubGrubVersion, MIN_VERSION, @@ -285,18 +290,11 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { } } PubGrubPackage::Package(package_name, _extra, Some(url)) => { - // Emit a request to fetch the metadata for this package. + // Emit a request to fetch the metadata for this distribution. if in_flight.insert_url(url) { priorities.add(package_name.clone()); - if WheelFilename::try_from(url).is_ok() { - // Kick off a request to download the wheel. - request_sink - .unbounded_send(Request::WheelUrl(package_name.clone(), url.clone()))?; - } else { - // Otherwise, assume this is a source distribution. - request_sink - .unbounded_send(Request::SdistUrl(package_name.clone(), url.clone()))?; - } + let distribution = Distribution::from_url(package_name.clone(), url.clone()); + request_sink.unbounded_send(Request::Distribution(distribution))?; } } } @@ -332,24 +330,13 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { }; // Emit a request to fetch the metadata for this version. - match candidate.file { - DistributionFile::Wheel(file) => { - if in_flight.insert_file(&file) { - request_sink.unbounded_send(Request::Wheel( - candidate.package_name.clone(), - file.clone(), - ))?; - } - } - DistributionFile::Sdist(file) => { - if in_flight.insert_file(&file) { - request_sink.unbounded_send(Request::Sdist( - candidate.package_name.clone(), - candidate.version.clone().into(), - file.clone(), - ))?; - } - } + if in_flight.insert_file(&candidate.file) { + let distribution = Distribution::from_registry( + candidate.package_name.clone(), + candidate.version.clone().into(), + candidate.file.clone().into(), + ); + request_sink.unbounded_send(Request::Distribution(distribution))?; } } Ok(()) @@ -389,7 +376,12 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { } } else { // Otherwise, assume this is a source distribution. - let entry = self.index.versions.wait(url.as_str()).await.unwrap(); + let entry = self + .index + .distributions + .wait(&url.distribution_id()) + .await + .unwrap(); let metadata = entry.value(); let version = PubGrubVersion::from(metadata.version.clone()); if range.contains(&version) { @@ -430,24 +422,13 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { ); // Emit a request to fetch the metadata for this version. - match candidate.file { - DistributionFile::Wheel(file) => { - if in_flight.insert_file(&file) { - request_sink.unbounded_send(Request::Wheel( - candidate.package_name.clone(), - file.clone(), - ))?; - } - } - DistributionFile::Sdist(file) => { - if in_flight.insert_file(&file) { - request_sink.unbounded_send(Request::Sdist( - candidate.package_name.clone(), - candidate.version.clone().into(), - file.clone(), - ))?; - } - } + if in_flight.insert_file(&candidate.file) { + let distribution = Distribution::from_registry( + candidate.package_name.clone(), + candidate.version.clone().into(), + candidate.file.clone().into(), + ); + request_sink.unbounded_send(Request::Distribution(distribution))?; } let version = candidate.version.clone(); @@ -490,11 +471,20 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { PubGrubPackage::Package(package_name, extra, url) => { // Wait for the metadata to be available. let entry = match url { - Some(url) => self.index.versions.wait(url.as_str()).await.unwrap(), + Some(url) => self + .index + .distributions + .wait(&url.distribution_id()) + .await + .unwrap(), None => { let versions = pins.get(package_name).unwrap(); let file = versions.get(version.into()).unwrap(); - self.index.versions.wait(&file.hashes.sha256).await.unwrap() + self.index + .distributions + .wait(&file.distribution_id()) + .await + .unwrap() } }; let metadata = entry.value(); @@ -555,15 +545,11 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { std::collections::btree_map::Entry::Occupied(mut entry) => { if matches!(entry.get(), DistributionFile::Sdist(_)) { // Wheels get precedence over source distributions. - entry.insert(DistributionFile::from(WheelFile( - file, filename, - ))); + entry.insert(DistributionFile::from(WheelFile(file))); } } std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(DistributionFile::from(WheelFile( - file, filename, - ))); + entry.insert(DistributionFile::from(WheelFile(file))); } } } @@ -574,7 +560,7 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { if let std::collections::btree_map::Entry::Vacant(entry) = version_map.entry(version) { - entry.insert(DistributionFile::from(SdistFile(file, filename))); + entry.insert(DistributionFile::from(SdistFile(file))); } } } @@ -583,30 +569,27 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { .packages .insert(package_name.clone(), version_map); } - Response::Wheel(file, metadata) => { - trace!("Received wheel metadata for: {}", file.filename); + Response::Distribution(Distribution::Built(distribution), metadata, ..) => { + trace!("Received built distribution metadata for: {distribution}"); self.index - .versions - .insert(file.hashes.sha256.clone(), metadata); + .distributions + .insert(distribution.distribution_id(), metadata); } - Response::Sdist(file, metadata) => { - trace!("Received sdist metadata for: {}", file.filename); + Response::Distribution(Distribution::Source(distribution), metadata, precise) => { + trace!("Received source distribution metadata for: {distribution}"); self.index - .versions - .insert(file.hashes.sha256.clone(), metadata); - } - Response::WheelUrl(url, precise, metadata) => { - trace!("Received remote wheel metadata for: {url}"); - self.index.versions.insert(url.to_string(), metadata); + .distributions + .insert(distribution.distribution_id(), metadata); if let Some(precise) = precise { - self.index.redirects.insert(url, precise); - } - } - Response::SdistUrl(url, precise, metadata) => { - trace!("Received remote source distribution metadata for: {url}"); - self.index.versions.insert(url.to_string(), metadata); - if let Some(precise) = precise { - self.index.redirects.insert(url, precise); + match distribution { + SourceDistribution::DirectUrl(sdist) => { + self.index.redirects.insert(sdist.url.clone(), precise); + } + SourceDistribution::Git(sdist) => { + self.index.redirects.insert(sdist.url.clone(), precise); + } + SourceDistribution::Registry(_) => {} + } } } } @@ -625,63 +608,67 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { .map_err(ResolveError::Client) .await } - // Fetch wheel metadata from the registry. - Request::Wheel(package_name, file) => { - let metadata = self - .client - .wheel_metadata(file.0.clone(), file.1.clone()) - .map_err(ResolveError::Client) - .await?; - if metadata.name != package_name { + // Fetch wheel metadata. + Request::Distribution(Distribution::Built(distribution)) => { + let metadata = + match &distribution { + BuiltDistribution::Registry(wheel) => { + self.client + .wheel_metadata(wheel.file.clone()) + .map_err(ResolveError::Client) + .await? + } + BuiltDistribution::DirectUrl(wheel) => { + let fetcher = BuiltDistributionFetcher::new(self.build_context.cache()); + match fetcher.find_dist_info(wheel, self.tags) { + Ok(Some(metadata)) => { + debug!("Found wheel metadata in cache: {wheel}"); + metadata + } + Ok(None) => { + debug!("Downloading wheel: {wheel}"); + fetcher.download_wheel(wheel, self.client).await.map_err( + |err| { + ResolveError::from_built_distribution( + distribution.clone(), + err, + ) + }, + )? + } + Err(err) => { + error!("Failed to read wheel from cache: {err}"); + fetcher.download_wheel(wheel, self.client).await.map_err( + |err| { + ResolveError::from_built_distribution( + distribution.clone(), + err, + ) + }, + )? + } + } + } + }; + + if metadata.name != *distribution.name() { return Err(ResolveError::NameMismatch { metadata: metadata.name, - given: package_name, + given: distribution.name().clone(), }); } - Ok(Response::Wheel(file, metadata)) + Ok(Response::Distribution( + Distribution::Built(distribution), + metadata, + None, + )) } - // Build a source distribution from the registry, returning its metadata. - Request::Sdist(package_name, version, file) => { - let builder = SourceDistributionFetcher::new(self.build_context); - let distribution = - RemoteDistributionRef::from_registry(&package_name, &version, &file); - let metadata = match builder.find_dist_info(&distribution, self.tags) { - Ok(Some(metadata)) => metadata, - Ok(None) => builder - .download_and_build_sdist(&distribution, self.client) - .await - .map_err(|err| ResolveError::RegistryDistribution { - filename: file.filename.clone(), - err, - })?, - Err(err) => { - error!( - "Failed to read source distribution {distribution} from cache: {err}", - ); - builder - .download_and_build_sdist(&distribution, self.client) - .await - .map_err(|err| ResolveError::RegistryDistribution { - filename: file.filename.clone(), - err, - })? - } - }; - if metadata.name != package_name { - return Err(ResolveError::NameMismatch { - metadata: metadata.name, - given: package_name, - }); - } - - Ok(Response::Sdist(file, metadata)) - } - // Build a source distribution from a remote URL, returning its metadata. - Request::SdistUrl(package_name, url) => { - let lock = self.locks.acquire(&url).await; + // Fetch source distribution metadata. + Request::Distribution(Distribution::Source(sdist)) => { + let lock = self.locks.acquire(&sdist).await; let _guard = lock.lock().await; let fetcher = if let Some(reporter) = &self.reporter { @@ -693,109 +680,77 @@ impl<'a, Context: BuildContext + Sync> Resolver<'a, Context> { }; let precise = fetcher - .precise(&RemoteDistributionRef::from_url(&package_name, &url)) + .precise(&sdist) .await - .map_err(|err| ResolveError::UrlDistribution { - url: url.clone(), - err, - })?; - - let distribution = RemoteDistributionRef::from_url( - &package_name, - precise.as_ref().unwrap_or(&url), - ); + .map_err(|err| ResolveError::from_source_distribution(sdist.clone(), err))?; let task = self .reporter .as_ref() - .map(|reporter| reporter.on_build_start(&distribution)); + .map(|reporter| reporter.on_build_start(&sdist)); - let metadata = match fetcher.find_dist_info(&distribution, self.tags) { - Ok(Some(metadata)) => { - debug!("Found source distribution metadata in cache: {url}"); - metadata - } - Ok(None) => { - debug!("Downloading source distribution from: {url}"); - fetcher - .download_and_build_sdist(&distribution, self.client) - .await - .map_err(|err| ResolveError::UrlDistribution { - url: url.clone(), - err, - })? - } - Err(err) => { - error!( - "Failed to read source distribution {distribution} from cache: {err}", - ); - fetcher - .download_and_build_sdist(&distribution, self.client) - .await - .map_err(|err| ResolveError::UrlDistribution { - url: url.clone(), - err, - })? + let metadata = { + // Insert the `precise`, if it exists. + let sdist = match sdist.clone() { + SourceDistribution::DirectUrl(sdist) => { + SourceDistribution::DirectUrl(DirectUrlSourceDistribution { + url: precise.clone().unwrap_or_else(|| sdist.url.clone()), + ..sdist + }) + } + SourceDistribution::Git(sdist) => { + SourceDistribution::Git(GitSourceDistribution { + url: precise.clone().unwrap_or_else(|| sdist.url.clone()), + ..sdist + }) + } + sdist @ SourceDistribution::Registry(_) => sdist, + }; + + match fetcher.find_dist_info(&sdist, self.tags) { + Ok(Some(metadata)) => { + debug!("Found source distribution metadata in cache: {sdist}"); + metadata + } + Ok(None) => { + debug!("Downloading source distribution: {sdist}"); + fetcher + .download_and_build_sdist(&sdist, self.client) + .await + .map_err(|err| { + ResolveError::from_source_distribution(sdist.clone(), err) + })? + } + Err(err) => { + error!("Failed to read source distribution from cache: {err}",); + fetcher + .download_and_build_sdist(&sdist, self.client) + .await + .map_err(|err| { + ResolveError::from_source_distribution(sdist.clone(), err) + })? + } } }; - if metadata.name != package_name { + if metadata.name != *sdist.name() { return Err(ResolveError::NameMismatch { metadata: metadata.name, - given: package_name, + given: sdist.name().clone(), }); } if let Some(task) = task { if let Some(reporter) = self.reporter.as_ref() { - reporter.on_build_complete(&distribution, task); + reporter.on_build_complete(&sdist, task); } } - Ok(Response::SdistUrl(url, precise, metadata)) - } - // Fetch wheel metadata from a remote URL. - Request::WheelUrl(package_name, url) => { - let lock = self.locks.acquire(&url).await; - let _guard = lock.lock().await; - - let fetcher = WheelFetcher::new(self.build_context.cache()); - let distribution = RemoteDistributionRef::from_url(&package_name, &url); - let metadata = match fetcher.find_dist_info(&distribution, self.tags) { - Ok(Some(metadata)) => { - debug!("Found wheel metadata in cache: {url}"); - metadata - } - Ok(None) => { - debug!("Downloading wheel from: {url}"); - fetcher - .download_wheel(&distribution, self.client) - .await - .map_err(|err| ResolveError::UrlDistribution { - url: url.clone(), - err, - })? - } - Err(err) => { - error!("Failed to read wheel {distribution} from cache: {err}",); - fetcher - .download_wheel(&distribution, self.client) - .await - .map_err(|err| ResolveError::UrlDistribution { - url: url.clone(), - err, - })? - } - }; - - if metadata.name != package_name { - return Err(ResolveError::NameMismatch { - metadata: metadata.name, - given: package_name, - }); - } - - Ok(Response::WheelUrl(url, None, metadata)) + Ok(Response::Distribution( + Distribution::Source(sdist), + metadata, + precise, + )) } } } @@ -835,10 +790,10 @@ pub trait Reporter: Send + Sync { fn on_complete(&self); /// Callback to invoke when a source distribution build is kicked off. - fn on_build_start(&self, distribution: &RemoteDistributionRef<'_>) -> usize; + fn on_build_start(&self, distribution: &SourceDistribution) -> usize; /// Callback to invoke when a source distribution build is complete. - fn on_build_complete(&self, distribution: &RemoteDistributionRef<'_>, id: usize); + fn on_build_complete(&self, distribution: &SourceDistribution, id: usize); /// Callback to invoke when a repository checkout begins. fn on_checkout_start(&self, url: &Url, rev: &str) -> usize; @@ -864,31 +819,21 @@ impl SourceDistributionReporter for Facade { /// Fetch the metadata for an item #[derive(Debug)] +#[allow(clippy::large_enum_variant)] enum Request { /// A request to fetch the metadata for a package. Package(PackageName), - /// A request to fetch wheel metadata from a registry. - Wheel(PackageName, WheelFile), - /// A request to fetch source distribution metadata from a registry. - Sdist(PackageName, pep440_rs::Version, SdistFile), - /// A request to fetch wheel metadata from a remote URL. - WheelUrl(PackageName, Url), - /// A request to fetch source distribution metadata from a remote URL. - SdistUrl(PackageName, Url), + /// A request to fetch the metadata for a built or source distribution. + Distribution(Distribution), } #[derive(Debug)] +#[allow(clippy::large_enum_variant)] enum Response { /// The returned metadata for a package hosted on a registry. Package(PackageName, SimpleJson), - /// The returned metadata for a wheel hosted on a registry. - Wheel(WheelFile, Metadata21), - /// The returned metadata for a source distribution hosted on a registry. - Sdist(SdistFile, Metadata21), - /// The returned metadata for a wheel hosted on a remote URL. - WheelUrl(Url, Option, Metadata21), - /// The returned metadata for a source distribution hosted on a remote URL. - SdistUrl(Url, Option, Metadata21), + /// The returned metadata for a distribution. + Distribution(Distribution, Metadata21, Option), } pub(crate) type VersionMap = BTreeMap; @@ -910,8 +855,11 @@ impl InFlight { self.packages.insert(package_name.clone()) } - fn insert_file(&mut self, file: &File) -> bool { - self.files.insert(file.hashes.sha256.clone()) + fn insert_file(&mut self, file: &DistributionFile) -> bool { + match file { + DistributionFile::Wheel(file) => self.files.insert(file.hashes.sha256.clone()), + DistributionFile::Sdist(file) => self.files.insert(file.hashes.sha256.clone()), + } } fn insert_url(&mut self, url: &Url) -> bool { @@ -919,27 +867,13 @@ impl InFlight { } } -/// A set of locks used to prevent concurrent access to the same resource. -#[derive(Debug, Default)] -struct Locks(Mutex>>>); - -impl Locks { - /// Acquire a lock on the given resource. - async fn acquire(&self, url: &Url) -> Arc> { - let mut map = self.0.lock().await; - map.entry(puffin_cache::digest(&RepositoryUrl::new(url))) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone() - } -} - /// In-memory index of package metadata. struct Index { /// A map from package name to the metadata for that package. packages: WaitMap, - /// A map from wheel SHA or URL to the metadata for that wheel. - versions: WaitMap, + /// A map from distribution SHA to metadata for that distribution. + distributions: WaitMap, /// A map from source URL to precise URL. redirects: WaitMap, @@ -949,7 +883,7 @@ impl Default for Index { fn default() -> Self { Self { packages: WaitMap::new(), - versions: WaitMap::new(), + distributions: WaitMap::new(), redirects: WaitMap::new(), } }