c19a294a48
Prior to this PR, there were numerous places where uv would leak credentials in logs. We had a way to mask credentials by calling methods or a recently-added `redact_url` function, but this was not secure by default. There were a number of other types (like `GitUrl`) that would leak credentials on display. This PR adds a `DisplaySafeUrl` newtype to prevent leaking credentials when logging by default. It takes a maximalist approach, replacing the use of `Url` almost everywhere. This includes when first parsing config files, when storing URLs in types like `GitUrl`, and also when storing URLs in types that in practice will never contain credentials (like `DirectorySourceUrl`). The idea is to make it easy for developers to do the right thing and for the compiler to support this (and to minimize ever having to manually convert back and forth). Displaying credentials now requires an active step. Note that despite this maximalist approach, the use of the newtype should be zero cost. One conspicuous place this PR does not use `DisplaySafeUrl` is in the `uv-auth` crate. That would require new clones since there are calls to `request.url()` that return a `&Url`. One option would have been to make `DisplaySafeUrl` wrap a `Cow`, but this would lead to lifetime annotations all over the codebase. I've created a separate PR based on this one (#13576) that updates `uv-auth` to use `DisplaySafeUrl` with one new clone. We can discuss the tradeoffs there. Most of this PR just replaces `Url` with `DisplaySafeUrl`. The core is `uv_redacted/lib.rs`, where the newtype is implemented. To make it easier to review the rest, here are some points of note: * `DisplaySafeUrl` has a `Display` implementation that masks credentials. Currently, it will still display the username when there is both a username and password. If we think is the wrong choice, it can now be changed in one place. * `DisplaySafeUrl` has a `remove_credentials()` method and also a `.to_string_with_credentials()` method. This allows us to use it in a variety of scenarios. * `IndexUrl::redacted()` was renamed to `IndexUrl::removed_credentials()` to make it clearer that we are not masking. * We convert from a `DisplaySafeUrl` to a `Url` when calling `reqwest` methods like `.get()` and `.head()`. * We convert from a `DisplaySafeUrl` to a `Url` when creating a `uv_auth::Index`. That is because, as mentioned above, I will be updating the `uv_auth` crate to use this newtype in a separate PR. * A number of tests (e.g., in `pip_install.rs`) that formerly used filters to mask tokens in the test output no longer need those filters since tokens in URLs are now masked automatically. * The one place we are still knowingly writing credentials to `pyproject.toml` is when a URL with credentials is passed to `uv add` with `--raw`. Since displaying credentials is no longer automatic, I have added a `to_string_with_credentials()` method to the `Pep508Url` trait. This is used when `--raw` is passed. Adding it to that trait is a bit weird, but it's the simplest way to achieve the goal. I'm open to suggestions on how to improve this, but note that because of the way we're using generic bounds, it's not as simple as just creating a separate trait for that method.
185 lines
5.8 KiB
Rust
185 lines
5.8 KiB
Rust
//! Git support is derived from Cargo's implementation.
|
||
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
|
||
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/source.rs>
|
||
|
||
use std::borrow::Cow;
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::Arc;
|
||
|
||
use anyhow::Result;
|
||
use reqwest_middleware::ClientWithMiddleware;
|
||
use tracing::{debug, instrument};
|
||
|
||
use uv_cache_key::{RepositoryUrl, cache_digest};
|
||
use uv_git_types::GitUrl;
|
||
use uv_redacted::DisplaySafeUrl;
|
||
|
||
use crate::GIT_STORE;
|
||
use crate::git::GitRemote;
|
||
|
||
/// A remote Git source that can be checked out locally.
|
||
pub struct GitSource {
|
||
/// The Git reference from the manifest file.
|
||
git: GitUrl,
|
||
/// The HTTP client to use for fetching.
|
||
client: ClientWithMiddleware,
|
||
/// Whether to disable SSL verification.
|
||
disable_ssl: bool,
|
||
/// Whether to operate without network connectivity.
|
||
offline: bool,
|
||
/// The path to the Git source database.
|
||
cache: PathBuf,
|
||
/// The reporter to use for this source.
|
||
reporter: Option<Arc<dyn Reporter>>,
|
||
}
|
||
|
||
impl GitSource {
|
||
/// Initialize a [`GitSource`] with the given Git URL, HTTP client, and cache path.
|
||
pub fn new(
|
||
git: GitUrl,
|
||
client: impl Into<ClientWithMiddleware>,
|
||
cache: impl Into<PathBuf>,
|
||
offline: bool,
|
||
) -> Self {
|
||
Self {
|
||
git,
|
||
disable_ssl: false,
|
||
offline,
|
||
client: client.into(),
|
||
cache: cache.into(),
|
||
reporter: None,
|
||
}
|
||
}
|
||
|
||
/// Disable SSL verification for this [`GitSource`].
|
||
#[must_use]
|
||
pub fn dangerous(self) -> Self {
|
||
Self {
|
||
disable_ssl: true,
|
||
..self
|
||
}
|
||
}
|
||
|
||
/// Set the [`Reporter`] to use for the [`GitSource`].
|
||
#[must_use]
|
||
pub fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self {
|
||
Self {
|
||
reporter: Some(reporter),
|
||
..self
|
||
}
|
||
}
|
||
|
||
/// Fetch the underlying Git repository at the given revision.
|
||
#[instrument(skip(self), fields(repository = %self.git.repository(), rev = ?self.git.precise()))]
|
||
pub fn fetch(self) -> Result<Fetch> {
|
||
// Compute the canonical URL for the repository.
|
||
let canonical = RepositoryUrl::new(self.git.repository());
|
||
|
||
// The path to the repo, within the Git database.
|
||
let ident = cache_digest(&canonical);
|
||
let db_path = self.cache.join("db").join(&ident);
|
||
|
||
// Authenticate the URL, if necessary.
|
||
let remote = if let Some(credentials) = GIT_STORE.get(&canonical) {
|
||
Cow::Owned(credentials.apply(self.git.repository().clone()))
|
||
} else {
|
||
Cow::Borrowed(self.git.repository())
|
||
};
|
||
|
||
let remote = GitRemote::new(&remote);
|
||
let (db, actual_rev, task) = match (self.git.precise(), remote.db_at(&db_path).ok()) {
|
||
// If we have a locked revision, and we have a preexisting database
|
||
// which has that revision, then no update needs to happen.
|
||
(Some(rev), Some(db)) if db.contains(rev) => {
|
||
debug!("Using existing Git source `{}`", self.git.repository());
|
||
(db, rev, None)
|
||
}
|
||
|
||
// ... otherwise we use this state to update the git database. Note
|
||
// that we still check for being offline here, for example in the
|
||
// situation that we have a locked revision but the database
|
||
// doesn't have it.
|
||
(locked_rev, db) => {
|
||
debug!("Updating Git source `{}`", self.git.repository());
|
||
|
||
// Report the checkout operation to the reporter.
|
||
let task = self.reporter.as_ref().map(|reporter| {
|
||
reporter.on_checkout_start(remote.url(), self.git.reference().as_rev())
|
||
});
|
||
|
||
let (db, actual_rev) = remote.checkout(
|
||
&db_path,
|
||
db,
|
||
self.git.reference(),
|
||
locked_rev,
|
||
&self.client,
|
||
self.disable_ssl,
|
||
self.offline,
|
||
)?;
|
||
|
||
(db, actual_rev, task)
|
||
}
|
||
};
|
||
|
||
// Don’t use the full hash, in order to contribute less to reaching the
|
||
// path length limit on Windows.
|
||
let short_id = db.to_short_id(actual_rev)?;
|
||
|
||
// Check out `actual_rev` from the database to a scoped location on the
|
||
// filesystem. This will use hard links and such to ideally make the
|
||
// checkout operation here pretty fast.
|
||
let checkout_path = self
|
||
.cache
|
||
.join("checkouts")
|
||
.join(&ident)
|
||
.join(short_id.as_str());
|
||
|
||
db.copy_to(actual_rev, &checkout_path)?;
|
||
|
||
// Report the checkout operation to the reporter.
|
||
if let Some(task) = task {
|
||
if let Some(reporter) = self.reporter.as_ref() {
|
||
reporter.on_checkout_complete(remote.url(), actual_rev.as_str(), task);
|
||
}
|
||
}
|
||
|
||
Ok(Fetch {
|
||
git: self.git.with_precise(actual_rev),
|
||
path: checkout_path,
|
||
})
|
||
}
|
||
}
|
||
|
||
pub struct Fetch {
|
||
/// The [`GitUrl`] reference that was fetched.
|
||
git: GitUrl,
|
||
/// The path to the checked out repository.
|
||
path: PathBuf,
|
||
}
|
||
|
||
impl Fetch {
|
||
pub fn git(&self) -> &GitUrl {
|
||
&self.git
|
||
}
|
||
|
||
pub fn path(&self) -> &Path {
|
||
&self.path
|
||
}
|
||
|
||
pub fn into_git(self) -> GitUrl {
|
||
self.git
|
||
}
|
||
|
||
pub fn into_path(self) -> PathBuf {
|
||
self.path
|
||
}
|
||
}
|
||
|
||
pub trait Reporter: Send + Sync {
|
||
/// Callback to invoke when a repository checkout begins.
|
||
fn on_checkout_start(&self, url: &DisplaySafeUrl, rev: &str) -> usize;
|
||
|
||
/// Callback to invoke when a repository checkout completes.
|
||
fn on_checkout_complete(&self, url: &DisplaySafeUrl, rev: &str, index: usize);
|
||
}
|