pioduino/uv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Allow users to incorporate Git tags into dynamic cache keys (#8259)

Browse Source 
## Summary

You can now use `cache-keys = [{ git = { commit = true, tags = true }
}]` to include both the current commit and set of tags in the cache key.

Closes https://github.com/astral-sh/uv/issues/7866.

Closes https://github.com/astral-sh/uv/issues/7997.
This commit is contained in:
Charlie Marsh
2024-10-16 08:13:29 -07:00
committed by GitHub
parent 6a81d302bb
commit 7730861bc5
8 changed files with 290 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/uv-cache-info/src/cache_info.rs
+69 -12
View File
@@ -1,4 +1,4 @@
use crate::commit_info::CacheCommit;
use crate::git_info::{Commit, Tags};
use crate::timestamp::Timestamp;
use serde::Deserialize;
@@ -26,7 +26,9 @@ pub struct CacheInfo {
/// files to timestamp via the `cache-keys` field.
timestamp: Option<Timestamp>,
/// The commit at which the distribution was built.
commit: Option<CacheCommit>,
commit: Option<Commit>,
/// The Git tags present at the time of the build.
tags: Option<Tags>,
}
impl CacheInfo {
@@ -51,6 +53,7 @@ impl CacheInfo {
/// Compute the cache info for a given directory.
pub fn from_directory(directory: &Path) -> Result<Self, CacheInfoError> {
let mut commit = None;
let mut tags = None;
let mut timestamp = None;
// Read the cache keys.
@@ -109,13 +112,37 @@ impl CacheInfo {
}
timestamp = max(timestamp, Some(Timestamp::from_metadata(&metadata)));
}
CacheKey::Git { git: true } => match CacheCommit::from_repository(directory) {
CacheKey::Git {
git: GitPattern::Bool(true),
} => match Commit::from_repository(directory) {
Ok(commit_info) => commit = Some(commit_info),
Err(err) => {
debug!("Failed to read the current commit: {err}");
}
},
CacheKey::Git { git: false } => {}
CacheKey::Git {
git: GitPattern::Set(set),
} => {
if set.commit.unwrap_or(false) {
match Commit::from_repository(directory) {
Ok(commit_info) => commit = Some(commit_info),
Err(err) => {
debug!("Failed to read the current commit: {err}");
}
}
}
if set.tags.unwrap_or(false) {
match Tags::from_repository(directory) {
Ok(tags_info) => tags = Some(tags_info),
Err(err) => {
debug!("Failed to read the current tags: {err}");
}
}
}
}
CacheKey::Git {
git: GitPattern::Bool(false),
} => {}
}
}
@@ -150,7 +177,11 @@ impl CacheInfo {
}
}
Ok(Self { timestamp, commit })
Ok(Self {
timestamp,
commit,
tags,
})
}
/// Compute the cache info for a given file, assumed to be a binary or source distribution
@@ -165,14 +196,18 @@ impl CacheInfo {
}
pub fn is_empty(&self) -> bool {
self.timestamp.is_none() && self.commit.is_none()
self.timestamp.is_none() && self.commit.is_none() && self.tags.is_none()
}
}
#[derive(Debug, serde::Deserialize)]
struct TimestampCommit {
#[serde(default)]
timestamp: Option<Timestamp>,
commit: Option<CacheCommit>,
#[serde(default)]
commit: Option<Commit>,
#[serde(default)]
tags: Option<Tags>,
}
#[derive(Debug, serde::Deserialize)]
@@ -192,9 +227,15 @@ impl From<CacheInfoWire> for CacheInfo {
timestamp: Some(timestamp),
..Self::default()
},
CacheInfoWire::TimestampCommit(TimestampCommit { timestamp, commit }) => {
Self { timestamp, commit }
}
CacheInfoWire::TimestampCommit(TimestampCommit {
timestamp,
commit,
tags,
}) => Self {
timestamp,
commit,
tags,
},
}
}
}
@@ -226,8 +267,24 @@ pub enum CacheKey {
Path(String),
/// Ex) `{ file = "Cargo.lock" }` or `{ file = "**/*.toml" }`
File { file: String },
/// Ex) `{ git = true }`
Git { git: bool },
/// Ex) `{ git = true }` or `{ git = { commit = true, tags = false } }`
Git { git: GitPattern },
}
#[derive(Debug, Clone, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(untagged, rename_all = "kebab-case", deny_unknown_fields)]
pub enum GitPattern {
Bool(bool),
Set(GitSet),
}
#[derive(Debug, Clone, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "kebab-case", deny_unknown_fields)]
pub struct GitSet {
commit: Option<bool>,
tags: Option<bool>,
}
pub enum FilePattern {
crates/uv-cache-info/src/commit_info.rs
-92
View File
@@ -1,92 +0,0 @@
use std::path::{Path, PathBuf};
#[derive(Debug, thiserror::Error)]
pub(crate) enum CacheCommitError {
#[error("The repository at {0} is missing a `.git` directory")]
MissingGitDir(PathBuf),
#[error("The repository at {0} is missing a `HEAD` file")]
MissingHead(PathBuf),
#[error("The repository at {0} has an invalid reference: `{1}`")]
InvalidRef(PathBuf, String),
#[error("The discovered commit has an invalid length (expected 40 characters): `{0}`")]
WrongLength(String),
#[error("The discovered commit has an invalid character (expected hexadecimal): `{0}`")]
WrongDigit(String),
#[error(transparent)]
Io(#[from] std::io::Error),
}
/// The current commit for a repository (i.e., a 40-character hexadecimal string).
#[derive(Default, Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
pub(crate) struct CacheCommit(String);
impl CacheCommit {
/// Return the [`CacheCommit`] for the repository at the given path.
pub(crate) fn from_repository(path: &Path) -> Result<Self, CacheCommitError> {
// Find the `.git` directory, searching through parent directories if necessary.
let git_dir = path
.ancestors()
.map(|ancestor| ancestor.join(".git"))
.find(|git_dir| git_dir.exists())
.ok_or_else(|| CacheCommitError::MissingGitDir(path.to_path_buf()))?;
let git_head_path =
git_head(&git_dir).ok_or_else(|| CacheCommitError::MissingHead(git_dir.clone()))?;
let git_head_contents = fs_err::read_to_string(git_head_path)?;
// The contents are either a commit or a reference in the following formats
// - "<commit>" when the head is detached
// - "ref <ref>" when working on a branch
// If a commit, checking if the HEAD file has changed is sufficient
// If a ref, we need to add the head file for that ref to rebuild on commit
let mut git_ref_parts = git_head_contents.split_whitespace();
let commit_or_ref = git_ref_parts.next().ok_or_else(|| {
CacheCommitError::InvalidRef(git_dir.clone(), git_head_contents.clone())
})?;
let commit = if let Some(git_ref) = git_ref_parts.next() {
let git_ref_path = git_dir.join(git_ref);
let commit = fs_err::read_to_string(git_ref_path)?;
commit.trim().to_string()
} else {
commit_or_ref.to_string()
};
// The commit should be 40 hexadecimal characters.
if commit.len() != 40 {
return Err(CacheCommitError::WrongLength(commit));
}
if commit.chars().any(|c| !c.is_ascii_hexdigit()) {
return Err(CacheCommitError::WrongDigit(commit));
}
Ok(Self(commit))
}
}
/// Return the path to the `HEAD` file of a Git repository, taking worktrees into account.
fn git_head(git_dir: &Path) -> Option<PathBuf> {
// The typical case is a standard git repository.
let git_head_path = git_dir.join("HEAD");
if git_head_path.exists() {
return Some(git_head_path);
}
if !git_dir.is_file() {
return None;
}
// If `.git/HEAD` doesn't exist and `.git` is actually a file,
// then let's try to attempt to read it as a worktree. If it's
// a worktree, then its contents will look like this, e.g.:
//
// gitdir: /home/andrew/astral/uv/main/.git/worktrees/pr2
//
// And the HEAD file we want to watch will be at:
//
// /home/andrew/astral/uv/main/.git/worktrees/pr2/HEAD
let contents = fs_err::read_to_string(git_dir).ok()?;
let (label, worktree_path) = contents.split_once(':')?;
if label != "gitdir" {
return None;
}
let worktree_path = worktree_path.trim();
Some(PathBuf::from(worktree_path))
}
crates/uv-cache-info/src/git_info.rs
+173
View File
@@ -0,0 +1,173 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
#[derive(Debug, thiserror::Error)]
pub(crate) enum GitInfoError {
#[error("The repository at {0} is missing a `.git` directory")]
MissingGitDir(PathBuf),
#[error("The repository at {0} is missing a `HEAD` file")]
MissingHead(PathBuf),
#[error("The repository at {0} is missing a `refs` directory")]
MissingRefs(PathBuf),
#[error("The repository at {0} has an invalid reference: `{1}`")]
InvalidRef(PathBuf, String),
#[error("The discovered commit has an invalid length (expected 40 characters): `{0}`")]
WrongLength(String),
#[error("The discovered commit has an invalid character (expected hexadecimal): `{0}`")]
WrongDigit(String),
#[error(transparent)]
Io(#[from] std::io::Error),
}
/// The current commit for a repository (i.e., a 40-character hexadecimal string).
#[derive(Default, Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
pub(crate) struct Commit(String);
impl Commit {
/// Return the [`Commit`] for the repository at the given path.
pub(crate) fn from_repository(path: &Path) -> Result<Self, GitInfoError> {
// Find the `.git` directory, searching through parent directories if necessary.
let git_dir = path
.ancestors()
.map(|ancestor| ancestor.join(".git"))
.find(|git_dir| git_dir.exists())
.ok_or_else(|| GitInfoError::MissingGitDir(path.to_path_buf()))?;
let git_head_path =
git_head(&git_dir).ok_or_else(|| GitInfoError::MissingHead(git_dir.clone()))?;
let git_head_contents = fs_err::read_to_string(git_head_path)?;
// The contents are either a commit or a reference in the following formats
// - "<commit>" when the head is detached
// - "ref <ref>" when working on a branch
// If a commit, checking if the HEAD file has changed is sufficient
// If a ref, we need to add the head file for that ref to rebuild on commit
let mut git_ref_parts = git_head_contents.split_whitespace();
let commit_or_ref = git_ref_parts
.next()
.ok_or_else(|| GitInfoError::InvalidRef(git_dir.clone(), git_head_contents.clone()))?;
let commit = if let Some(git_ref) = git_ref_parts.next() {
let git_ref_path = git_dir.join(git_ref);
let commit = fs_err::read_to_string(git_ref_path)?;
commit.trim().to_string()
} else {
commit_or_ref.to_string()
};
// The commit should be 40 hexadecimal characters.
if commit.len() != 40 {
return Err(GitInfoError::WrongLength(commit));
}
if commit.chars().any(|c| !c.is_ascii_hexdigit()) {
return Err(GitInfoError::WrongDigit(commit));
}
Ok(Self(commit))
}
}
/// The set of tags visible in a repository.
#[derive(Default, Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
pub(crate) struct Tags(BTreeMap<String, String>);
impl Tags {
/// Return the [`Tags`] for the repository at the given path.
pub(crate) fn from_repository(path: &Path) -> Result<Self, GitInfoError> {
// Find the `.git` directory, searching through parent directories if necessary.
let git_dir = path
.ancestors()
.map(|ancestor| ancestor.join(".git"))
.find(|git_dir| git_dir.exists())
.ok_or_else(|| GitInfoError::MissingGitDir(path.to_path_buf()))?;
let git_refs_path =
git_refs(&git_dir).ok_or_else(|| GitInfoError::MissingRefs(git_dir.clone()))?;
let mut tags = BTreeMap::new();
// Map each tag to its commit.
let read_dir = match fs_err::read_dir(git_refs_path.join("tags")) {
Ok(read_dir) => read_dir,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Self(tags));
}
Err(err) => return Err(err.into()),
};
for entry in read_dir {
let entry = entry?;
let path = entry.path();
if let Some(tag) = path.file_name().and_then(|name| name.to_str()) {
let commit = fs_err::read_to_string(&path)?.trim().to_string();
// The commit should be 40 hexadecimal characters.
if commit.len() != 40 {
return Err(GitInfoError::WrongLength(commit));
}
if commit.chars().any(|c| !c.is_ascii_hexdigit()) {
return Err(GitInfoError::WrongDigit(commit));
}
tags.insert(tag.to_string(), commit);
}
}
Ok(Self(tags))
}
}
/// Return the path to the `HEAD` file of a Git repository, taking worktrees into account.
fn git_head(git_dir: &Path) -> Option<PathBuf> {
// The typical case is a standard git repository.
let git_head_path = git_dir.join("HEAD");
if git_head_path.exists() {
return Some(git_head_path);
}
if !git_dir.is_file() {
return None;
}
// If `.git/HEAD` doesn't exist and `.git` is actually a file,
// then let's try to attempt to read it as a worktree. If it's
// a worktree, then its contents will look like this, e.g.:
//
// gitdir: /home/andrew/astral/uv/main/.git/worktrees/pr2
//
// And the HEAD file we want to watch will be at:
//
// /home/andrew/astral/uv/main/.git/worktrees/pr2/HEAD
let contents = fs_err::read_to_string(git_dir).ok()?;
let (label, worktree_path) = contents.split_once(':')?;
if label != "gitdir" {
return None;
}
let worktree_path = worktree_path.trim();
Some(PathBuf::from(worktree_path))
}
/// Return the path to the `refs` directory of a Git repository, taking worktrees into account.
fn git_refs(git_dir: &Path) -> Option<PathBuf> {
// The typical case is a standard git repository.
let git_head_path = git_dir.join("refs");
if git_head_path.exists() {
return Some(git_head_path);
}
if !git_dir.is_file() {
return None;
}
// If `.git/refs` doesn't exist and `.git` is actually a file,
// then let's try to attempt to read it as a worktree. If it's
// a worktree, then its contents will look like this, e.g.:
//
// gitdir: /home/andrew/astral/uv/main/.git/worktrees/pr2
//
// And the HEAD refs we want to watch will be at:
//
// /home/andrew/astral/uv/main/.git/refs
let contents = fs_err::read_to_string(git_dir).ok()?;
let (label, worktree_path) = contents.split_once(':')?;
if label != "gitdir" {
return None;
}
let worktree_path = PathBuf::from(worktree_path.trim());
let refs_path = worktree_path.parent()?.parent()?.join("refs");
Some(refs_path)
}
crates/uv-cache-info/src/lib.rs
+1 -1
View File
@@ -2,5 +2,5 @@ pub use crate::cache_info::*;
pub use crate::timestamp::*;
mod cache_info;
mod commit_info;
mod git_info;
mod timestamp;
crates/uv-settings/src/settings.rs
+3 -3
View File
@@ -69,9 +69,9 @@ pub struct Options {
/// determine whether any files have changed.
///
/// Cache keys can also include version control information. For example, if a project uses
/// `setuptools_scm` to read its version from a Git tag, you can specify `cache-keys = [{ git = true }, { file = "pyproject.toml" }]`
/// `setuptools_scm` to read its version from a Git commit, you can specify `cache-keys = [{ git = { commit = true }, { file = "pyproject.toml" }]`
/// to include the current Git commit hash in the cache key (in addition to the
/// `pyproject.toml`).
/// `pyproject.toml`). Git tags are also supported via `cache-keys = [{ git = { commit = true, tags = true } }]`.
///
/// Cache keys only affect the project defined by the `pyproject.toml` in which they're
/// specified (as opposed to, e.g., affecting all members in a workspace), and all paths and
@@ -80,7 +80,7 @@ pub struct Options {
default = r#"[{ file = "pyproject.toml" }, { file = "setup.py" }, { file = "setup.cfg" }]"#,
value_type = "list[dict]",
example = r#"
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = true }]
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = { commit = true }]
"#
)]
cache_keys: Option<Vec<CacheKey>>,
docs/concepts/cache.md
+9 -1
View File
@@ -41,7 +41,15 @@ should be rebuilt whenever the commit hash changes, you can add the following to
```toml title="pyproject.toml"
[tool.uv]
cache-keys = [{ git = true }]
cache-keys = [{ git = { commit = true } }]
```
If your dynamic metadata incorporates information from the set of Git tags, you can expand the cache
key to include the tags:
```toml title="pyproject.toml"
[tool.uv]
cache-keys = [{ git = { commit = true, tag = true } }]
```
Similarly, if a project reads from a `requirements.txt` to populate its dependencies, you can add
docs/reference/settings.md
+4 -4
View File
@@ -330,9 +330,9 @@ Note that the use of globs can be expensive, as uv may need to walk the filesyst
determine whether any files have changed.
Cache keys can also include version control information. For example, if a project uses
`setuptools_scm` to read its version from a Git tag, you can specify `cache-keys = [{ git = true }, { file = "pyproject.toml" }]`
`setuptools_scm` to read its version from a Git commit, you can specify `cache-keys = [{ git = { commit = true }, { file = "pyproject.toml" }]`
to include the current Git commit hash in the cache key (in addition to the
`pyproject.toml`).
`pyproject.toml`). Git tags are also supported via `cache-keys = [{ git = { commit = true, tags = true } }]`.
Cache keys only affect the project defined by the `pyproject.toml` in which they're
specified (as opposed to, e.g., affecting all members in a workspace), and all paths and
@@ -348,13 +348,13 @@ globs are interpreted as relative to the project directory.
```toml
[tool.uv]
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = true }]
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = { commit = true }]
```
=== "uv.toml"
```toml
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = true }]
cache-keys = [{ file = "pyproject.toml" }, { file = "requirements.txt" }, { git = { commit = true }]
```
---
uv.schema.json
Generated
+31 -3
View File
@@ -22,7 +22,7 @@
]
},
"cache-keys": {
"description": "The keys to consider when caching builds for the project.\n\nCache keys enable you to specify the files or directories that should trigger a rebuild when modified. By default, uv will rebuild a project whenever the `pyproject.toml`, `setup.py`, or `setup.cfg` files in the project directory are modified, i.e.:\n\n```toml cache-keys = [{ file = \"pyproject.toml\" }, { file = \"setup.py\" }, { file = \"setup.cfg\" }] ```\n\nAs an example: if a project uses dynamic metadata to read its dependencies from a `requirements.txt` file, you can specify `cache-keys = [{ file = \"requirements.txt\" }, { file = \"pyproject.toml\" }]` to ensure that the project is rebuilt whenever the `requirements.txt` file is modified (in addition to watching the `pyproject.toml`).\n\nGlobs are supported, following the syntax of the [`glob`](https://docs.rs/glob/0.3.1/glob/struct.Pattern.html) crate. For example, to invalidate the cache whenever a `.toml` file in the project directory or any of its subdirectories is modified, you can specify `cache-keys = [{ file = \"**/*.toml\" }]`. Note that the use of globs can be expensive, as uv may need to walk the filesystem to determine whether any files have changed.\n\nCache keys can also include version control information. For example, if a project uses `setuptools_scm` to read its version from a Git tag, you can specify `cache-keys = [{ git = true }, { file = \"pyproject.toml\" }]` to include the current Git commit hash in the cache key (in addition to the `pyproject.toml`).\n\nCache keys only affect the project defined by the `pyproject.toml` in which they're specified (as opposed to, e.g., affecting all members in a workspace), and all paths and globs are interpreted as relative to the project directory.",
"description": "The keys to consider when caching builds for the project.\n\nCache keys enable you to specify the files or directories that should trigger a rebuild when modified. By default, uv will rebuild a project whenever the `pyproject.toml`, `setup.py`, or `setup.cfg` files in the project directory are modified, i.e.:\n\n```toml cache-keys = [{ file = \"pyproject.toml\" }, { file = \"setup.py\" }, { file = \"setup.cfg\" }] ```\n\nAs an example: if a project uses dynamic metadata to read its dependencies from a `requirements.txt` file, you can specify `cache-keys = [{ file = \"requirements.txt\" }, { file = \"pyproject.toml\" }]` to ensure that the project is rebuilt whenever the `requirements.txt` file is modified (in addition to watching the `pyproject.toml`).\n\nGlobs are supported, following the syntax of the [`glob`](https://docs.rs/glob/0.3.1/glob/struct.Pattern.html) crate. For example, to invalidate the cache whenever a `.toml` file in the project directory or any of its subdirectories is modified, you can specify `cache-keys = [{ file = \"**/*.toml\" }]`. Note that the use of globs can be expensive, as uv may need to walk the filesystem to determine whether any files have changed.\n\nCache keys can also include version control information. For example, if a project uses `setuptools_scm` to read its version from a Git commit, you can specify `cache-keys = [{ git = { commit = true }, { file = \"pyproject.toml\" }]` to include the current Git commit hash in the cache key (in addition to the `pyproject.toml`). Git tags are also supported via `cache-keys = [{ git = { commit = true, tags = true } }]`.\n\nCache keys only affect the project defined by the `pyproject.toml` in which they're specified (as opposed to, e.g., affecting all members in a workspace), and all paths and globs are interpreted as relative to the project directory.",
"type": [
"array",
"null"
@@ -488,14 +488,14 @@
"additionalProperties": false
},
{
"description": "Ex) `{ git = true }`",
"description": "Ex) `{ git = true }` or `{ git = { commit = true, tags = false } }`",
"type": "object",
"required": [
"git"
],
"properties": {
"git": {
"type": "boolean"
"$ref": "#/definitions/GitPattern"
}
},
"additionalProperties": false
@@ -551,6 +551,34 @@
"description": "The normalized name of an extra dependency.\n\nConverts the name to lowercase and collapses runs of `-`, `_`, and `.` down to a single `-`. For example, `---`, `.`, and `__` are all converted to a single `-`.\n\nSee: - <https://peps.python.org/pep-0685/#specification/> - <https://packaging.python.org/en/latest/specifications/name-normalization/>",
"type": "string"
},
"GitPattern": {
"anyOf": [
{
"type": "boolean"
},
{
"$ref": "#/definitions/GitSet"
}
]
},
"GitSet": {
"type": "object",
"properties": {
"commit": {
"type": [
"boolean",
"null"
]
},
"tags": {
"type": [
"boolean",
"null"
]
}
},
"additionalProperties": false
},
"Index": {
"type": "object",
"required": [