From ce4b9a410a8652f2f6b448a88189a44af5b138a8 Mon Sep 17 00:00:00 2001 From: Zsolt Dollenstein Date: Tue, 3 Mar 2026 12:02:09 +0000 Subject: [PATCH] Fetch CPython from an Astral mirror by default (#18207) --- crates/uv-client/src/base_client.rs | 16 +- crates/uv-client/src/lib.rs | 2 +- crates/uv-python/src/downloads.rs | 250 +++++++++++++++++++++----- crates/uv/src/commands/python/list.rs | 18 +- crates/uv/tests/it/python_list.rs | 9 +- 5 files changed, 235 insertions(+), 60 deletions(-) diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index 2087a3b69..6cf1fe169 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -1189,7 +1189,7 @@ impl RetryableStrategy for UvRetryableStrategy { /// * When streaming a response, a reqwest error may be hidden several layers behind errors /// of different crates processing the stream, including `io::Error` layers /// * Any `h2` error -fn retryable_on_request_failure(err: &(dyn Error + 'static)) -> Option { +pub fn retryable_on_request_failure(err: &(dyn Error + 'static)) -> Option { // First, try to show a nice trace log if let Some((Some(status), Some(url))) = find_source::(&err) .map(|request_err| (request_err.status(), request_err.url())) @@ -1227,19 +1227,19 @@ fn retryable_on_request_failure(err: &(dyn Error + 'static)) -> Option().is_some() { // All h2 errors look like errors that should be retried // https://github.com/astral-sh/uv/issues/15916 - trace!("Retrying nested h2 error"); + trace!("Transient nested h2 error"); return Some(Retryable::Transient); } else if let Some(io_err) = source.downcast_ref::() { has_known_error = true; @@ -1258,12 +1258,12 @@ fn retryable_on_request_failure(err: &(dyn Error + 'static)) -> Option Option bool { + match self { + // There are two primary reasons to try an alternative URL: + // - HTTP/DNS/TCP/etc errors due to a mirror being blocked at various layers + // - HTTP 404s from the mirror, which may mean the next URL still works + // So we catch all network-level errors here. + Self::NetworkError(..) + | Self::NetworkMiddlewareError(..) + | Self::NetworkErrorWithRetries { .. } => true, + // `Io` uses `#[error(transparent)]`, so `source()` delegates to the inner error's + // own source rather than returning the `io::Error` itself. We must unwrap it + // explicitly so that `retryable_on_request_failure` can inspect the io error kind. + Self::Io(err) => retryable_on_request_failure(err).is_some(), + _ => false, + } + } } +/// The URL prefix used by `python-build-standalone` releases on GitHub. +const CPYTHON_DOWNLOADS_URL_PREFIX: &str = + "https://github.com/astral-sh/python-build-standalone/releases/download/"; + +/// The default Astral mirror for `python-build-standalone` releases. +/// +/// This mirror is tried first for CPython downloads when no user-configured mirror is set. +/// If the mirror fails, uv falls back to the canonical GitHub URL. +const CPYTHON_DOWNLOAD_DEFAULT_MIRROR: &str = + "https://releases.astral.sh/github/python-build-standalone/releases/download/"; + #[derive(Debug, PartialEq, Eq, Clone, Hash)] pub struct ManagedPythonDownload { key: PythonInstallationKey, @@ -1092,6 +1127,10 @@ impl ManagedPythonDownload { } /// Download and extract a Python distribution, retrying on failure. + /// + /// For CPython without a user-configured mirror, the default Astral mirror is tried first. + /// Each attempt tries all URLs in sequence without backoff between them; backoff is only + /// applied after all URLs have been exhausted. #[instrument(skip(client, installation_dir, scratch_dir, reporter), fields(download = % self.key()))] pub async fn fetch_with_retry( &self, @@ -1104,40 +1143,54 @@ impl ManagedPythonDownload { pypy_install_mirror: Option<&str>, reporter: Option<&dyn Reporter>, ) -> Result { + let urls = self.download_urls(python_install_mirror, pypy_install_mirror)?; let mut retry_state = RetryState::start( *retry_policy, - self.download_url(python_install_mirror, pypy_install_mirror)?, + // We pass in the last URL because that will trigger backoff if it fails. + urls.last().ok_or(Error::NoPythonDownloadUrlFound)?.clone(), ); - loop { - let result = self - .fetch( - client, - installation_dir, - scratch_dir, - reinstall, - python_install_mirror, - pypy_install_mirror, - reporter, - ) - .await; - match result { - Ok(download_result) => return Ok(download_result), - Err(err) => { - if let Some(backoff) = retry_state.should_retry(&err, err.retries()) { - retry_state.sleep_backoff(backoff).await; - continue; + 'retry: loop { + for (i, url) in urls.iter().enumerate() { + let is_last = i == urls.len() - 1; + match self + .fetch_from_url( + url, + client, + installation_dir, + scratch_dir, + reinstall, + reporter, + ) + .await + { + Ok(download_result) => return Ok(download_result), + Err(err) => { + if !is_last && err.should_try_next_url() { + warn!( + "Failed to download `{}` from {url} ({err}); falling back to {}", + self.key(), + urls[i + 1] + ); + continue; + } + // All URLs exhausted; apply the retry policy. + if let Some(backoff) = retry_state.should_retry(&err, err.retries()) { + retry_state.sleep_backoff(backoff).await; + continue 'retry; + } + return if retry_state.total_retries() > 0 { + Err(Error::NetworkErrorWithRetries { + err: Box::new(err), + retries: retry_state.total_retries(), + }) + } else { + Err(err) + }; } - return if retry_state.total_retries() > 0 { - Err(Error::NetworkErrorWithRetries { - err: Box::new(err), - retries: retry_state.total_retries(), - }) - } else { - Err(err) - }; } - }; + } + unreachable!("download_urls() must return at least one URL"); } } @@ -1153,7 +1206,29 @@ impl ManagedPythonDownload { pypy_install_mirror: Option<&str>, reporter: Option<&dyn Reporter>, ) -> Result { - let url = self.download_url(python_install_mirror, pypy_install_mirror)?; + let urls = self.download_urls(python_install_mirror, pypy_install_mirror)?; + let url = urls.first().ok_or(Error::NoPythonDownloadUrlFound)?; + self.fetch_from_url( + url, + client, + installation_dir, + scratch_dir, + reinstall, + reporter, + ) + .await + } + + /// Download and extract a Python distribution from the given URL. + async fn fetch_from_url( + &self, + url: &DisplaySafeUrl, + client: &BaseClient, + installation_dir: &Path, + scratch_dir: &Path, + reinstall: bool, + reporter: Option<&dyn Reporter>, + ) -> Result { let path = installation_dir.join(self.key().to_string()); // If it is not a reinstall and the dir already exists, return it. @@ -1213,13 +1288,13 @@ impl ManagedPythonDownload { if client.connectivity().is_offline() { return Err(Error::OfflinePythonMissing { file: Box::new(self.key().clone()), - url: Box::new(url), + url: Box::new(url.clone()), python_builds_dir, }); } self.download_archive( - &url, + url, client, reporter, &python_builds_dir, @@ -1255,7 +1330,7 @@ impl ManagedPythonDownload { temp_dir.path().simplified_display() ); - let (reader, size) = read_url(&url, client).await?; + let (reader, size) = read_url(url, client).await?; self.extract_reader( reader, temp_dir.path(), @@ -1431,27 +1506,58 @@ impl ManagedPythonDownload { self.key.version() } - /// Return the [`Url`] to use when downloading the distribution. If a mirror is set via the - /// appropriate environment variable, use it instead. + /// Return the primary [`Url`] to use when downloading the distribution. + /// + /// This is the first URL from [`Self::download_urls`]. For CPython without a user-configured + /// mirror, this is the default Astral mirror URL. Use [`Self::download_urls`] to get all + /// URLs including fallbacks. pub fn download_url( &self, python_install_mirror: Option<&str>, pypy_install_mirror: Option<&str>, ) -> Result { + self.download_urls(python_install_mirror, pypy_install_mirror) + .map(|mut urls| urls.remove(0)) + } + + /// Return the ordered list of [`Url`]s to try when downloading the distribution. + /// + /// For CPython without a user-configured mirror, the default Astral mirror is listed first, + /// followed by the canonical GitHub URL as a fallback. + /// + /// For all other cases (user mirror explicitly set, PyPy, GraalPy, Pyodide), a single URL + /// is returned with no fallback. + pub fn download_urls( + &self, + python_install_mirror: Option<&str>, + pypy_install_mirror: Option<&str>, + ) -> Result, Error> { match self.key.implementation { LenientImplementationName::Known(ImplementationName::CPython) => { if let Some(mirror) = python_install_mirror { - let Some(suffix) = self.url.strip_prefix( - "https://github.com/astral-sh/python-build-standalone/releases/download/", - ) else { + // User-configured mirror: use it exclusively, no automatic fallback. + let Some(suffix) = self.url.strip_prefix(CPYTHON_DOWNLOADS_URL_PREFIX) else { return Err(Error::Mirror( EnvVars::UV_PYTHON_INSTALL_MIRROR, self.url.to_string(), )); }; - return Ok(DisplaySafeUrl::parse( + return Ok(vec![DisplaySafeUrl::parse( format!("{}/{}", mirror.trim_end_matches('/'), suffix).as_str(), - )?); + )?]); + } + // No user mirror: try the default Astral mirror first, fall back to GitHub. + if let Some(suffix) = self.url.strip_prefix(CPYTHON_DOWNLOADS_URL_PREFIX) { + let mirror_url = DisplaySafeUrl::parse( + format!( + "{}/{}", + CPYTHON_DOWNLOAD_DEFAULT_MIRROR.trim_end_matches('/'), + suffix + ) + .as_str(), + )?; + let canonical_url = DisplaySafeUrl::parse(&self.url)?; + return Ok(vec![mirror_url, canonical_url]); } } @@ -1464,16 +1570,16 @@ impl ManagedPythonDownload { self.url.to_string(), )); }; - return Ok(DisplaySafeUrl::parse( + return Ok(vec![DisplaySafeUrl::parse( format!("{}/{}", mirror.trim_end_matches('/'), suffix).as_str(), - )?); + )?]); } } _ => {} } - Ok(DisplaySafeUrl::parse(&self.url)?) + Ok(vec![DisplaySafeUrl::parse(&self.url)?]) } } @@ -2214,4 +2320,60 @@ mod tests { "cpython-3.12.0-linux-x86_64-gnu" ); } + + /// A hash mismatch is a post-download integrity failure — retrying a different URL cannot fix + /// it, so it should not trigger a fallback. + #[test] + fn test_should_try_next_url_hash_mismatch() { + let err = Error::HashMismatch { + installation: "cpython-3.12.0".to_string(), + expected: "abc".to_string(), + actual: "def".to_string(), + }; + assert!(!err.should_try_next_url()); + } + + /// A local filesystem error during extraction (e.g. permission denied writing to disk) is not + /// a network failure — a different URL would produce the same outcome. + #[test] + fn test_should_try_next_url_extract_error_filesystem() { + let err = Error::ExtractError( + "archive.tar.gz".to_string(), + uv_extract::Error::Io(io::Error::new(io::ErrorKind::PermissionDenied, "")), + ); + assert!(!err.should_try_next_url()); + } + + /// A generic IO error from a local filesystem operation (e.g. permission denied on cache + /// directory) should not trigger a fallback to a different URL. + #[test] + fn test_should_try_next_url_io_error_filesystem() { + let err = Error::Io(io::Error::new(io::ErrorKind::PermissionDenied, "")); + assert!(!err.should_try_next_url()); + } + + /// A network IO error (e.g. connection reset mid-download) surfaces as `Error::Io` from + /// `download_archive`. It should trigger a fallback because a different mirror may succeed. + #[test] + fn test_should_try_next_url_io_error_network() { + let err = Error::Io(io::Error::new(io::ErrorKind::ConnectionReset, "")); + assert!(err.should_try_next_url()); + } + + /// A 404 HTTP response from the mirror becomes `Error::NetworkError` — it should trigger a + /// URL fallback, because a 404 on the mirror does not mean the file is absent from GitHub. + #[test] + fn test_should_try_next_url_network_error_404() { + let url = + DisplaySafeUrl::from_str("https://releases.astral.sh/python/cpython-3.12.0.tar.gz") + .unwrap(); + // `NetworkError` wraps a `WrappedReqwestError`; we use a middleware error as a + // stand-in because `should_try_next_url` only inspects the variant, not the contents. + let wrapped = WrappedReqwestError::with_problem_details( + reqwest_middleware::Error::Middleware(anyhow::anyhow!("404 Not Found")), + None, + ); + let err = Error::NetworkError(url, wrapped); + assert!(err.should_try_next_url()); + } } diff --git a/crates/uv/src/commands/python/list.rs b/crates/uv/src/commands/python/list.rs index cf437d2b8..d4ca88c7f 100644 --- a/crates/uv/src/commands/python/list.rs +++ b/crates/uv/src/commands/python/list.rs @@ -12,7 +12,9 @@ use rustc_hash::FxHashSet; use uv_cache::Cache; use uv_client::BaseClientBuilder; use uv_fs::Simplified; -use uv_python::downloads::{ManagedPythonDownloadList, PythonDownloadRequest}; +use uv_python::downloads::{ + Error as PythonDownloadError, ManagedPythonDownloadList, PythonDownloadRequest, +}; use uv_python::{ DiscoveryError, EnvironmentPreference, PythonDownloads, PythonInstallation, PythonNotFound, PythonPreference, PythonRequest, PythonSource, find_python_installations, @@ -123,10 +125,16 @@ pub(crate) async fn list( output.insert(( download.key().clone(), Kind::Download, - Either::Right(download.download_url( - python_install_mirror.as_deref(), - pypy_install_mirror.as_deref(), - )?), + Either::Right( + download + .download_urls( + python_install_mirror.as_deref(), + pypy_install_mirror.as_deref(), + )? + .into_iter() + .next() + .ok_or(PythonDownloadError::NoPythonDownloadUrlFound)?, + ), )); } } diff --git a/crates/uv/tests/it/python_list.rs b/crates/uv/tests/it/python_list.rs index 8e045ae78..df229d687 100644 --- a/crates/uv/tests/it/python_list.rs +++ b/crates/uv/tests/it/python_list.rs @@ -613,6 +613,11 @@ fn python_list_with_mirrors() { .to_string(), "$1[FILE-PATH]".to_string(), )) + .with_filter(( + r"(https://releases\.astral\.sh/github/python-build-standalone/releases/download/).*" + .to_string(), + "$1[FILE-PATH]".to_string(), + )) .with_filter(( r"(https://downloads\.python\.org/pypy/).*".to_string(), "$1[FILE-PATH]".to_string(), @@ -667,7 +672,7 @@ fn python_list_with_mirrors() { ----- stderr ----- "); - // Test without mirrors - verify default URLs are used + // Test without mirrors - verify the default Astral mirror URL is used for CPython uv_snapshot!(context.filters(), context.python_list() .arg("3.10") .arg("--show-urls") @@ -675,7 +680,7 @@ fn python_list_with_mirrors() { success: true exit_code: 0 ----- stdout ----- - cpython-3.10.19-[PLATFORM] https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + cpython-3.10.19-[PLATFORM] https://releases.astral.sh/github/python-build-standalone/releases/download/[FILE-PATH] pypy-3.10.16-[PLATFORM] https://downloads.python.org/pypy/[FILE-PATH] graalpy-3.10.0-[PLATFORM] https://github.com/oracle/graalpython/releases/download/[FILE-PATH]