Files
uv/crates/uv-git/src/source.rs
T
konsti 59b46bc216 Show messages for builds and large downloads in non-interactive mode (#11165)
When stderr is not a tty, we currently don't show any messages for build
or large downloads, since indicatif is hidden. We can improve this by
showing a message for:

* Starting and finishing a large download (>1MB)
* Starting and finishing a build

Downloads are limited to 1MB or unknown size to keep the logs concise
and not scroll the entire terminal away for a download that finishes
almost immediately.

These messages are not captured in the tests since their order is
non-deterministic (downloads and builds race to finish).

There are no "tick" messages for large downloads yet, we could e.g. show
an update on runnning downloads every n seconds.

Part of #11121

**Test Plan**

```
$ uv venv && FORCE_COLOR=1 cargo run -q pip install numpy --no-binary :all: --no-cache 2>&1 | tee a.txt
  Using CPython 3.13.0
  Creating virtual environment at: .venv
  Activate with: source .venv/bin/activate
  Resolved 1 package in 221ms
     Building numpy==2.2.2
        Built numpy==2.2.2
  Prepared 1 package in 2m 34s
  Installed 1 package in 6ms
   + numpy==2.2.2
```


![image](https://github.com/user-attachments/assets/f4b64313-afa7-449f-9e5b-2b1b7026bef3)


```
$ uv venv && FORCE_COLOR=1 cargo run -q pip install torch --no-cache 2>&1 | tee b.txt
  Using CPython 3.13.0
  Creating virtual environment at: .venv
  Activate with: source .venv/bin/activate
  Resolved 24 packages in 648ms
  Downloading setuptools (1.2MiB)
  Downloading nvidia-cuda-cupti-cu12 (13.2MiB)
  Downloading torch (731.1MiB)
  Downloading nvidia-nvjitlink-cu12 (20.1MiB)
  Downloading nvidia-cufft-cu12 (201.7MiB)
  Downloading nvidia-cuda-nvrtc-cu12 (23.5MiB)
  Downloading nvidia-curand-cu12 (53.7MiB)
  Downloading nvidia-nccl-cu12 (179.9MiB)
  Downloading nvidia-cudnn-cu12 (634.0MiB)
  Downloading nvidia-cublas-cu12 (346.6MiB)
  Downloading sympy (5.9MiB)
  Downloading nvidia-cusparse-cu12 (197.8MiB)
  Downloading nvidia-cusparselt-cu12 (143.1MiB)
  Downloading networkx (1.6MiB)
  Downloading nvidia-cusolver-cu12 (122.0MiB)
  Downloading triton (241.4MiB)
   Downloaded setuptools
   Downloaded networkx
   Downloaded sympy
   Downloaded nvidia-cuda-cupti-cu12
   Downloaded nvidia-nvjitlink-cu12
   Downloaded nvidia-cuda-nvrtc-cu12
   Downloaded nvidia-curand-cu12
[...]
```


![image](https://github.com/user-attachments/assets/71918d94-c5c0-44ce-bea8-aaba6cd80ef7)
2025-02-05 17:38:02 -06:00

179 lines
5.6 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Git support is derived from Cargo's implementation.
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/source.rs>
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Result;
use reqwest_middleware::ClientWithMiddleware;
use tracing::{debug, instrument};
use url::Url;
use uv_cache_key::{cache_digest, RepositoryUrl};
use crate::git::GitRemote;
use crate::{GitOid, GitUrl, GIT_STORE};
/// A remote Git source that can be checked out locally.
pub struct GitSource {
/// The Git reference from the manifest file.
git: GitUrl,
/// The HTTP client to use for fetching.
client: ClientWithMiddleware,
/// Whether to disable SSL verification.
disable_ssl: bool,
/// The path to the Git source database.
cache: PathBuf,
/// The reporter to use for this source.
reporter: Option<Arc<dyn Reporter>>,
}
impl GitSource {
/// Initialize a [`GitSource`] with the given Git URL, HTTP client, and cache path.
pub fn new(
git: GitUrl,
client: impl Into<ClientWithMiddleware>,
cache: impl Into<PathBuf>,
) -> Self {
Self {
git,
disable_ssl: false,
client: client.into(),
cache: cache.into(),
reporter: None,
}
}
/// Disable SSL verification for this [`GitSource`].
#[must_use]
pub fn dangerous(self) -> Self {
Self {
disable_ssl: true,
..self
}
}
/// Set the [`Reporter`] to use for the [`GitSource`].
#[must_use]
pub fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self {
Self {
reporter: Some(reporter),
..self
}
}
/// Fetch the underlying Git repository at the given revision.
#[instrument(skip(self), fields(repository = %self.git.repository, rev = ?self.git.precise))]
pub fn fetch(self) -> Result<Fetch> {
// Compute the canonical URL for the repository.
let canonical = RepositoryUrl::new(&self.git.repository);
// The path to the repo, within the Git database.
let ident = cache_digest(&canonical);
let db_path = self.cache.join("db").join(&ident);
// Authenticate the URL, if necessary.
let remote = if let Some(credentials) = GIT_STORE.get(&canonical) {
Cow::Owned(credentials.apply(self.git.repository.clone()))
} else {
Cow::Borrowed(&self.git.repository)
};
let remote = GitRemote::new(&remote);
let (db, actual_rev, task) = match (self.git.precise, remote.db_at(&db_path).ok()) {
// If we have a locked revision, and we have a preexisting database
// which has that revision, then no update needs to happen.
(Some(rev), Some(db)) if db.contains(rev) => {
debug!("Using existing Git source `{}`", self.git.repository);
(db, rev, None)
}
// ... otherwise we use this state to update the git database. Note
// that we still check for being offline here, for example in the
// situation that we have a locked revision but the database
// doesn't have it.
(locked_rev, db) => {
debug!("Updating Git source `{}`", self.git.repository);
// Report the checkout operation to the reporter.
let task = self.reporter.as_ref().map(|reporter| {
reporter.on_checkout_start(remote.url(), self.git.reference.as_rev())
});
let (db, actual_rev) = remote.checkout(
&db_path,
db,
&self.git.reference,
locked_rev.map(GitOid::from),
&self.client,
self.disable_ssl,
)?;
(db, actual_rev, task)
}
};
// Dont use the full hash, in order to contribute less to reaching the
// path length limit on Windows.
let short_id = db.to_short_id(actual_rev)?;
// Check out `actual_rev` from the database to a scoped location on the
// filesystem. This will use hard links and such to ideally make the
// checkout operation here pretty fast.
let checkout_path = self
.cache
.join("checkouts")
.join(&ident)
.join(short_id.as_str());
db.copy_to(actual_rev, &checkout_path)?;
// Report the checkout operation to the reporter.
if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() {
reporter.on_checkout_complete(remote.url(), actual_rev.as_str(), task);
}
}
Ok(Fetch {
git: self.git.with_precise(actual_rev),
path: checkout_path,
})
}
}
pub struct Fetch {
/// The [`GitUrl`] reference that was fetched.
git: GitUrl,
/// The path to the checked out repository.
path: PathBuf,
}
impl Fetch {
pub fn git(&self) -> &GitUrl {
&self.git
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn into_git(self) -> GitUrl {
self.git
}
pub fn into_path(self) -> PathBuf {
self.path
}
}
pub trait Reporter: Send + Sync {
/// Callback to invoke when a repository checkout begins.
fn on_checkout_start(&self, url: &Url, rev: &str) -> usize;
/// Callback to invoke when a repository checkout completes.
fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize);
}