From 656fc427b979802fc963bfb8d7621cd5314ab463 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 10 Jun 2024 19:27:04 -0700 Subject: [PATCH] Add support for local directories with `--index-url` (#4226) ## Summary Closes #4078. --- crates/distribution-types/src/file.rs | 13 +-- crates/distribution-types/src/index_url.rs | 2 + crates/pep508-rs/src/verbatim_url.rs | 1 + crates/uv-client/src/error.rs | 10 +- crates/uv-client/src/registry_client.rs | 109 +++++++++++++++--- .../src/distribution_database.rs | 11 +- crates/uv-distribution/src/error.rs | 3 + crates/uv-distribution/src/source/mod.rs | 39 +++++++ crates/uv/src/cli.rs | 12 ++ crates/uv/tests/pip_install.rs | 50 ++++++++ 10 files changed, 219 insertions(+), 31 deletions(-) diff --git a/crates/distribution-types/src/file.rs b/crates/distribution-types/src/file.rs index 0eb362421..4e0530477 100644 --- a/crates/distribution-types/src/file.rs +++ b/crates/distribution-types/src/file.rs @@ -2,7 +2,6 @@ use std::fmt::{Display, Formatter}; use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use thiserror::Error; use url::Url; use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError}; @@ -10,7 +9,7 @@ use pep508_rs::split_scheme; use pypi_types::{CoreMetadata, HashDigest, Yanked}; /// Error converting [`pypi_types::File`] to [`distribution_type::File`]. -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub enum FileConversionError { #[error("Failed to parse 'requires-python': `{0}`")] RequiresPython(String, #[source] VersionSpecifiersParseError), @@ -57,12 +56,10 @@ impl File { .map_err(|err| FileConversionError::RequiresPython(err.line().clone(), err))?, size: file.size, upload_time_utc_ms: file.upload_time.map(|dt| dt.timestamp_millis()), - url: { - if split_scheme(&file.url).is_some() { - FileLocation::AbsoluteUrl(file.url) - } else { - FileLocation::RelativeUrl(base.to_string(), file.url) - } + url: if split_scheme(&file.url).is_some() { + FileLocation::AbsoluteUrl(file.url) + } else { + FileLocation::RelativeUrl(base.to_string(), file.url) }, yanked: file.yanked, }) diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index 42c105ccd..0b03fd1b2 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -98,6 +98,8 @@ impl FromStr for IndexUrl { let url = VerbatimUrl::from_url(url).with_given(s.to_owned()); if *url.raw() == *PYPI_URL { Ok(Self::Pypi(url)) + } else if url.scheme() == "file" { + Ok(Self::Path(url)) } else { Ok(Self::Url(url)) } diff --git a/crates/pep508-rs/src/verbatim_url.rs b/crates/pep508-rs/src/verbatim_url.rs index 876f6b0cd..9732568e4 100644 --- a/crates/pep508-rs/src/verbatim_url.rs +++ b/crates/pep508-rs/src/verbatim_url.rs @@ -235,6 +235,7 @@ impl Pep508Url for VerbatimUrl { .with_given(url.to_string()), ) } + // Ex) `https://download.pytorch.org/whl/torch_stable.html` Some(_) => { // Ex) `https://download.pytorch.org/whl/torch_stable.html` diff --git a/crates/uv-client/src/error.rs b/crates/uv-client/src/error.rs index 2f88bdf54..d332d1eb4 100644 --- a/crates/uv-client/src/error.rs +++ b/crates/uv-client/src/error.rs @@ -128,15 +128,15 @@ impl From for Error { #[derive(Debug, thiserror::Error)] pub enum ErrorKind { - /// An invalid URL was provided. #[error(transparent)] - UrlParseError(#[from] url::ParseError), + UrlParse(#[from] url::ParseError), - /// A base URL could not be joined with a possibly relative URL. #[error(transparent)] - JoinRelativeError(#[from] pypi_types::JoinRelativeError), + JoinRelativeUrl(#[from] pypi_types::JoinRelativeError), + + #[error("Expected a file URL, but received: {0}")] + NonFileUrl(Url), - /// Dist-info error #[error(transparent)] DistInfo(#[from] install_wheel_rs::Error), diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 199b66e37..b74eb458d 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; use std::fmt::Debug; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use async_http_range_reader::AsyncHttpRangeReader; @@ -20,7 +20,7 @@ use pep440_rs::Version; use pep508_rs::MarkerEnvironment; use platform_tags::Platform; use pypi_types::{Metadata23, SimpleJson}; -use uv_cache::{Cache, CacheBucket, WheelCache}; +use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache}; use uv_configuration::IndexStrategy; use uv_configuration::KeyringProviderType; use uv_normalize::PackageName; @@ -258,6 +258,10 @@ impl RegistryClient { Ok(results) } + /// Fetch the [`SimpleMetadata`] from a single index for a given package. + /// + /// The index can either be a PEP 503-compatible remote repository, or a local directory laid + /// out in the same format. async fn simple_single_index( &self, package_name: &PackageName, @@ -293,6 +297,22 @@ impl RegistryClient { Connectivity::Offline => CacheControl::AllowStale, }; + if matches!(index, IndexUrl::Path(_)) { + self.fetch_local_index(package_name, &url).await.map(Ok) + } else { + self.fetch_remote_index(package_name, &url, &cache_entry, cache_control) + .await + } + } + + /// Fetch the [`SimpleMetadata`] from a remote URL, using the PEP 503 Simple Repository API. + async fn fetch_remote_index( + &self, + package_name: &PackageName, + url: &Url, + cache_entry: &CacheEntry, + cache_control: CacheControl, + ) -> Result, CachedClientError>, Error> { let simple_request = self .uncached_client() .get(url.clone()) @@ -331,10 +351,7 @@ impl RegistryClient { } MediaType::Html => { let text = response.text().await.map_err(ErrorKind::from)?; - let SimpleHtml { base, files } = SimpleHtml::parse(&text, &url) - .map_err(|err| Error::from_html_err(err, url.clone()))?; - - SimpleMetadata::from_files(files, package_name, base.as_url()) + SimpleMetadata::from_html(&text, package_name, &url)? } }; OwnedArchive::from_unarchived(&unarchived) @@ -346,7 +363,7 @@ impl RegistryClient { .cached_client() .get_cacheable( simple_request, - &cache_entry, + cache_entry, cache_control, parse_simple_response, ) @@ -354,6 +371,24 @@ impl RegistryClient { Ok(result) } + /// Fetch the [`SimpleMetadata`] from a local file, using a PEP 503-compatible directory + /// structure. + async fn fetch_local_index( + &self, + package_name: &PackageName, + url: &Url, + ) -> Result, Error> { + let path = url + .to_file_path() + .map_err(|_| ErrorKind::NonFileUrl(url.clone()))? + .join("index.html"); + let text = fs_err::tokio::read_to_string(&path) + .await + .map_err(ErrorKind::from)?; + let metadata = SimpleMetadata::from_html(&text, package_name, url)?; + OwnedArchive::from_unarchived(&metadata) + } + /// Fetch the metadata for a remote wheel file. /// /// For a remote wheel, we try the following ways to fetch the metadata: @@ -364,20 +399,45 @@ impl RegistryClient { pub async fn wheel_metadata(&self, built_dist: &BuiltDist) -> Result { let metadata = match &built_dist { BuiltDist::Registry(wheels) => { + #[derive(Debug, Clone)] + enum WheelLocation { + /// A local file path. + Path(PathBuf), + /// A remote URL. + Url(Url), + } + let wheel = wheels.best_wheel(); - match &wheel.file.url { + + let location = match &wheel.file.url { FileLocation::RelativeUrl(base, url) => { let url = pypi_types::base_url_join_relative(base, url) - .map_err(ErrorKind::JoinRelativeError)?; - self.wheel_metadata_registry(&wheel.index, &wheel.file, &url) - .await? + .map_err(ErrorKind::JoinRelativeUrl)?; + if url.scheme() == "file" { + let path = url + .to_file_path() + .map_err(|_| ErrorKind::NonFileUrl(url.clone()))?; + WheelLocation::Path(path) + } else { + WheelLocation::Url(url) + } } FileLocation::AbsoluteUrl(url) => { - let url = Url::parse(url).map_err(ErrorKind::UrlParseError)?; - self.wheel_metadata_registry(&wheel.index, &wheel.file, &url) - .await? + let url = Url::parse(url).map_err(ErrorKind::UrlParse)?; + if url.scheme() == "file" { + let path = url + .to_file_path() + .map_err(|_| ErrorKind::NonFileUrl(url.clone()))?; + WheelLocation::Path(path) + } else { + WheelLocation::Url(url) + } } - FileLocation::Path(path) => { + FileLocation::Path(path) => WheelLocation::Path(path.clone()), + }; + + match location { + WheelLocation::Path(path) => { let file = fs_err::tokio::File::open(&path) .await .map_err(ErrorKind::Io)?; @@ -385,6 +445,10 @@ impl RegistryClient { read_metadata_async_seek(&wheel.filename, built_dist.to_string(), reader) .await? } + WheelLocation::Url(url) => { + self.wheel_metadata_registry(&wheel.index, &wheel.file, &url) + .await? + } } } BuiltDist::DirectUrl(wheel) => { @@ -599,7 +663,7 @@ impl RegistryClient { std::io::Error::new( std::io::ErrorKind::TimedOut, format!( - "Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: {}s).", self.timeout() + "Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: {}s).", self.timeout() ), ) } else { @@ -772,7 +836,6 @@ impl SimpleMetadata { DistFilename::SourceDistFilename(ref inner) => &inner.version, DistFilename::WheelFilename(ref inner) => &inner.version, }; - let file = match File::try_from(file, base) { Ok(file) => file, Err(err) => { @@ -799,6 +862,18 @@ impl SimpleMetadata { .collect(), ) } + + /// Read the [`SimpleMetadata`] from an HTML index. + fn from_html(text: &str, package_name: &PackageName, url: &Url) -> Result { + let SimpleHtml { base, files } = + SimpleHtml::parse(text, url).map_err(|err| Error::from_html_err(err, url.clone()))?; + + Ok(SimpleMetadata::from_files( + files, + package_name, + base.as_url(), + )) + } } impl IntoIterator for SimpleMetadata { diff --git a/crates/uv-distribution/src/distribution_database.rs b/crates/uv-distribution/src/distribution_database.rs index 0369df32c..b26d2c8f7 100644 --- a/crates/uv-distribution/src/distribution_database.rs +++ b/crates/uv-distribution/src/distribution_database.rs @@ -171,7 +171,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { WheelCache::Index(&wheel.index).wheel_dir(wheel.name().as_ref()), wheel.filename.stem(), ); - return self .load_wheel(path, &wheel.filename, cache_entry, dist, hashes) .await; @@ -185,6 +184,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { wheel.filename.stem(), ); + // If the URL is a file URL, load the wheel directly. + if url.scheme() == "file" { + let path = url + .to_file_path() + .map_err(|()| Error::NonFileUrl(url.clone()))?; + return self + .load_wheel(&path, &wheel.filename, wheel_entry, dist, hashes) + .await; + } + // Download and unzip. match self .stream_wheel( diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index a9fe55670..80ed26c27 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use tokio::task::JoinError; +use url::Url; use zip::result::ZipError; use crate::metadata::MetadataError; @@ -25,6 +26,8 @@ pub enum Error { RelativePath(PathBuf), #[error(transparent)] JoinRelativeUrl(#[from] pypi_types::JoinRelativeError), + #[error("Expected a file URL, but received: {0}")] + NonFileUrl(Url), #[error(transparent)] Git(#[from] uv_git::GitResolverError), #[error(transparent)] diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 0730787b9..a67690d27 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -124,6 +124,26 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } }; + // If the URL is a file URL, use the local path directly. + if url.scheme() == "file" { + let path = url + .to_file_path() + .map_err(|()| Error::NonFileUrl(url.clone()))?; + return self + .archive( + source, + &PathSourceUrl { + url: &url, + path: Cow::Owned(path), + }, + &cache_shard, + tags, + hashes, + ) + .boxed_local() + .await; + } + self.url( source, &dist.file.filename, @@ -281,6 +301,25 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } }; + // If the URL is a file URL, use the local path directly. + if url.scheme() == "file" { + let path = url + .to_file_path() + .map_err(|()| Error::NonFileUrl(url.clone()))?; + return self + .archive_metadata( + source, + &PathSourceUrl { + url: &url, + path: Cow::Owned(path), + }, + &cache_shard, + hashes, + ) + .boxed_local() + .await; + } + self.url_metadata( source, &dist.file.filename, diff --git a/crates/uv/src/cli.rs b/crates/uv/src/cli.rs index c8e080672..9efc08988 100644 --- a/crates/uv/src/cli.rs +++ b/crates/uv/src/cli.rs @@ -1655,6 +1655,9 @@ pub(crate) struct VenvArgs { /// The URL of the Python package index (by default: ). /// + /// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local + /// directory laid out in the same format. + /// /// The index given by this flag is given lower priority than all other /// indexes specified via the `--extra-index-url` flag. /// @@ -1666,6 +1669,9 @@ pub(crate) struct VenvArgs { /// Extra URLs of package indexes to use, in addition to `--index-url`. /// + /// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local + /// directory laid out in the same format. + /// /// All indexes given via this flag take priority over the index /// in `--index-url` (which defaults to PyPI). And when multiple /// `--extra-index-url` flags are given, earlier values take priority. @@ -2018,6 +2024,9 @@ pub(crate) struct ToolchainInstallArgs { pub(crate) struct IndexArgs { /// The URL of the Python package index (by default: ). /// + /// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local + /// directory laid out in the same format. + /// /// The index given by this flag is given lower priority than all other /// indexes specified via the `--extra-index-url` flag. /// @@ -2029,6 +2038,9 @@ pub(crate) struct IndexArgs { /// Extra URLs of package indexes to use, in addition to `--index-url`. /// + /// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local + /// directory laid out in the same format. + /// /// All indexes given via this flag take priority over the index /// in `--index-url` (which defaults to PyPI). And when multiple /// `--extra-index-url` flags are given, earlier values take priority. diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs index aa521852f..4a3d744d5 100644 --- a/crates/uv/tests/pip_install.rs +++ b/crates/uv/tests/pip_install.rs @@ -8,6 +8,7 @@ use assert_fs::prelude::*; use base64::{prelude::BASE64_STANDARD as base64, Engine}; use indoc::indoc; use itertools::Itertools; +use url::Url; use common::{uv_snapshot, TestContext}; use uv_fs::Simplified; @@ -5306,3 +5307,52 @@ fn prefer_editable() -> Result<()> { Ok(()) } + +/// Resolve against a local directory laid out as a PEP 503-compatible index. +#[test] +fn local_index() -> Result<()> { + let context = TestContext::new("3.12"); + + let root = context.temp_dir.child("simple-html"); + fs_err::create_dir_all(&root)?; + + let tqdm = root.child("tqdm"); + fs_err::create_dir_all(&tqdm)?; + + let index = tqdm.child("index.html"); + index.write_str(&indoc::formatdoc! {r#" + + + + + + +

Links for example-a-961b4c22

+ + tqdm-1000.0.0-py3-none-any.whl + + + + "#, Url::from_directory_path(context.workspace_root.join("scripts/links/")).unwrap().as_str()})?; + + uv_snapshot!(context.filters(), context.install_without_exclude_newer() + .arg("tqdm") + .arg("--index-url") + .arg(Url::from_directory_path(root).unwrap().as_str()), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + tqdm==1000.0.0 + "### + ); + + Ok(()) +}