Avoid uv cache clean errors due to Win32 path normalization (#18856)
## Summary Closes #16586 Adds a public function to `uv-fs` crate called `verbatim_path` which is now leveraged by `rm_rf` in `uv-cache` crate for cleaning paths that require verbatim composition to avoid failures seen in #16586. ## Test Plan Tested locally on Windows 10 and 11 by setting `UV_CACHE_DIR` to a local directory and then running `uv init`, `uv add uwsgi`, and `uv cache clean` to ensure there is no failures. Additionally unit and integration tests were added to avoid future regressions.
This commit is contained in:
samypr100
@@ -62,7 +62,9 @@ impl Removal {
|
||||
reporter: Option<&dyn CleanReporter>,
|
||||
skip_locked_file: bool,
|
||||
) -> io::Result<()> {
|
||||
let metadata = match fs_err::symlink_metadata(path) {
|
||||
let path = uv_fs::verbatim_path(path);
|
||||
|
||||
let metadata = match fs_err::symlink_metadata(&path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
@@ -79,18 +81,18 @@ impl Removal {
|
||||
use std::os::windows::fs::FileTypeExt;
|
||||
|
||||
if metadata.file_type().is_symlink_dir() {
|
||||
remove_dir(path)?;
|
||||
remove_dir(&path)?;
|
||||
} else {
|
||||
remove_file(path)?;
|
||||
remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
remove_file(path)?;
|
||||
remove_file(&path)?;
|
||||
}
|
||||
} else {
|
||||
remove_file(path)?;
|
||||
remove_file(&path)?;
|
||||
}
|
||||
|
||||
reporter.map(CleanReporter::on_clean);
|
||||
@@ -98,7 +100,7 @@ impl Removal {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for entry in walkdir::WalkDir::new(path).contents_first(true) {
|
||||
for entry in walkdir::WalkDir::new(&path).contents_first(true) {
|
||||
// If we hit a directory that lacks read permissions, try to make it readable.
|
||||
if let Err(ref err) = entry {
|
||||
if err
|
||||
@@ -109,7 +111,7 @@ impl Removal {
|
||||
if set_readable(dir).unwrap_or(false) {
|
||||
// Retry the operation; if we _just_ `self.rm_rf(dir)` and continue,
|
||||
// `walkdir` may give us duplicate entries for the directory.
|
||||
return self.rm_rf(path, reporter, skip_locked_file);
|
||||
return self.rm_rf(&path, reporter, skip_locked_file);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -122,7 +124,7 @@ impl Removal {
|
||||
&& entry.file_name() == ".lock"
|
||||
&& entry
|
||||
.path()
|
||||
.strip_prefix(path)
|
||||
.strip_prefix(&path)
|
||||
.is_ok_and(|suffix| suffix == Path::new(".lock"))
|
||||
{
|
||||
continue;
|
||||
@@ -143,7 +145,7 @@ impl Removal {
|
||||
remove_dir(entry.path())?;
|
||||
} else if entry.file_type().is_dir() {
|
||||
// Remove the directory with the exclusive lock last.
|
||||
if skip_locked_file && entry.path() == path {
|
||||
if skip_locked_file && entry.path() == path.as_ref() {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
+159
-1
@@ -1,5 +1,6 @@
|
||||
use std::borrow::Cow;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Component, Path, PathBuf, Prefix};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use either::Either;
|
||||
@@ -351,6 +352,107 @@ pub fn try_relative_to_if(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a [`Path`] to a Windows `verbatim` path (prefixed with `\\?\`) when possible to bypass
|
||||
/// Win32 path normalization such as [`MAX_PATH`] and removed trailing characters (dot, space).
|
||||
/// Other characters as defined by [`Path.GetInvalidFileNameChars`] are still prohibited. This
|
||||
/// function will attempt to perform path normalization similar to Win32 default normalization
|
||||
/// without triggering the existing Win32 limitations.
|
||||
///
|
||||
/// Only [`Prefix::UNC`] and [`Prefix::Disk`] conversion compatible components are supported.
|
||||
/// * [`Prefix::UNC`] `\\server\share` becomes `\\?\UNC\server\share`
|
||||
/// * [`Prefix::Disk`] `DriveLetter:` becomes `\\?\DriveLetter:`
|
||||
///
|
||||
/// Other representations do not yield a `verbatim` path. The following cases are returned as-is:
|
||||
/// * Non-Windows systems.
|
||||
/// * Device paths such as those starting with `\\.\`.
|
||||
/// * Paths already prefixed with `\\?\` or `\\?\UNC\`.
|
||||
///
|
||||
/// WARNING: Adding the `\\?\` prefix effectively skips Win32 default path normalization. Even
|
||||
/// though it allows operations on paths that are normally unavailable, it can also be used to
|
||||
/// create entries that can potentially lead to further issues with operations that expect
|
||||
/// normalization such as symbolic links, junctions or reparse points.
|
||||
///
|
||||
/// [`MAX_PATH`]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
|
||||
/// [`Path.GetInvalidFileNameChars`]: https://learn.microsoft.com/en-us/dotnet/api/system.io.path.getinvalidfilenamechars
|
||||
///
|
||||
/// See:
|
||||
/// * <https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file>
|
||||
/// * <https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats>
|
||||
pub fn verbatim_path(path: &Path) -> Cow<'_, Path> {
|
||||
if !cfg!(windows) {
|
||||
return Cow::Borrowed(path);
|
||||
}
|
||||
|
||||
// Attempt to resolve a fully qualified path just like Win32 path normalization would.
|
||||
// std::path::absolute calls GetFullPathNameW which defeats the purpose of this function
|
||||
// as it results in Win32 default path normalization.
|
||||
let resolved_path = if path.is_relative() {
|
||||
Cow::Owned(CWD.join(path))
|
||||
} else {
|
||||
Cow::Borrowed(path)
|
||||
};
|
||||
|
||||
// Fast Path: we only support verbatim conversion for Prefix::UNC and Prefix::Disk
|
||||
if let Some(Component::Prefix(prefix)) = resolved_path.components().next() {
|
||||
match prefix.kind() {
|
||||
Prefix::UNC(..) | Prefix::Disk(_) => {},
|
||||
// return as-is as there's no verbatim equivalent for `\\.\device`
|
||||
Prefix::DeviceNS(_)
|
||||
// return as-is as its already verbatim
|
||||
| Prefix::Verbatim(_)
|
||||
| Prefix::VerbatimDisk(_)
|
||||
| Prefix::VerbatimUNC(..) => return Cow::Borrowed(path)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve relative directory components while avoiding default Win32 path normalization
|
||||
let normalized_path = normalized(&resolved_path);
|
||||
|
||||
let mut components = normalized_path.components();
|
||||
let Some(Component::Prefix(prefix)) = components.next() else {
|
||||
return Cow::Borrowed(path);
|
||||
};
|
||||
|
||||
match prefix.kind() {
|
||||
// `DriveLetter:` -> `\\?\DriveLetter:`
|
||||
Prefix::Disk(_) => {
|
||||
let mut result = OsString::from(r"\\?\");
|
||||
result.push(normalized_path.as_os_str()); // e.g. "C:"
|
||||
Cow::Owned(PathBuf::from(result))
|
||||
}
|
||||
// `\\server\share` -> `\\?\UNC\server\share`
|
||||
Prefix::UNC(server, share) => {
|
||||
let mut result = OsString::from(r"\\?\UNC\");
|
||||
result.push(server);
|
||||
result.push(r"\");
|
||||
result.push(share);
|
||||
for component in components {
|
||||
match component {
|
||||
Component::RootDir => {} // being cautious
|
||||
Component::Prefix(_) => {
|
||||
debug_assert!(false, "prefix already consumed");
|
||||
}
|
||||
Component::CurDir | Component::ParentDir => {
|
||||
debug_assert!(false, "path already normalized");
|
||||
}
|
||||
Component::Normal(_) => {
|
||||
result.push(r"\");
|
||||
result.push(component.as_os_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
Cow::Owned(PathBuf::from(result))
|
||||
}
|
||||
Prefix::DeviceNS(_)
|
||||
| Prefix::Verbatim(_)
|
||||
| Prefix::VerbatimDisk(_)
|
||||
| Prefix::VerbatimUNC(..) => {
|
||||
debug_assert!(false, "skipped via fast path");
|
||||
Cow::Borrowed(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A path that can be serialized and deserialized in a portable way by converting Windows-style
|
||||
/// backslashes to forward slashes, and using a `.` for an empty path.
|
||||
///
|
||||
@@ -610,4 +712,60 @@ mod tests {
|
||||
assert_eq!(normalize_path(Path::new(input)), Path::new(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_verbatim_path() {
|
||||
let relative_path = format!(r"\\?\{}\path\to\logging.", CWD.simplified_display());
|
||||
let relative_root = format!(
|
||||
r"\\?\{}\path\to\logging.",
|
||||
CWD.components()
|
||||
.next()
|
||||
.expect("expected a drive letter prefix")
|
||||
.simplified_display()
|
||||
);
|
||||
let cases = [
|
||||
// Non-Verbatim disk
|
||||
(r"C:\path\to\logging.", r"\\?\C:\path\to\logging."),
|
||||
(r"C:\path\to\.\logging.", r"\\?\C:\path\to\logging."),
|
||||
(r"C:\path\to\..\to\logging.", r"\\?\C:\path\to\logging."),
|
||||
(r"C:/path/to/../to/./logging.", r"\\?\C:\path\to\logging."),
|
||||
(r"C:path\to\..\to\logging.", r"\\?\C:path\to\logging."), // @TODO(samypr100) we do not support expanding drive-relative paths
|
||||
(r".\path\to\.\logging.", relative_path.as_str()),
|
||||
(r"path\to\..\to\logging.", relative_path.as_str()),
|
||||
(r"./path/to/logging.", relative_path.as_str()),
|
||||
(r"\path\to\logging.", relative_root.as_str()),
|
||||
// Non-Verbatim UNC
|
||||
(
|
||||
r"\\127.0.0.1\c$\path\to\logging.",
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
),
|
||||
(
|
||||
r"\\127.0.0.1\c$\path\to\.\logging.",
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
),
|
||||
(
|
||||
r"\\127.0.0.1\c$\path\to\..\to\logging.",
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
),
|
||||
(
|
||||
r"//127.0.0.1/c$/path/to/../to/./logging.",
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
),
|
||||
// Verbatim Disk
|
||||
(r"\\?\C:\path\to\logging.", r"\\?\C:\path\to\logging."),
|
||||
// Verbatim UNC
|
||||
(
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
r"\\?\UNC\127.0.0.1\c$\path\to\logging.",
|
||||
),
|
||||
// Device Namespace
|
||||
(r"\\.\PhysicalDrive0", r"\\.\PhysicalDrive0"),
|
||||
(r"\\.\NUL", r"\\.\NUL"),
|
||||
];
|
||||
|
||||
for (input, expected) in cases {
|
||||
assert_eq!(verbatim_path(Path::new(input)), Path::new(expected));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,3 +275,48 @@ async fn cache_timeout() {
|
||||
error: Timeout ([TIME]) when waiting for lock on `[CACHE_DIR]/` at `[CACHE_DIR]/.lock`, is another uv process running? You can set `UV_LOCK_TIMEOUT` to increase the timeout.
|
||||
");
|
||||
}
|
||||
|
||||
/// `cache clean` should handle file paths normally restricted by Win32 path normalization.
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn clean_handles_verbatim_paths() -> Result<()> {
|
||||
let context = uv_test::test_context!("3.12");
|
||||
|
||||
// Clean slate
|
||||
fs_err::remove_dir_all(&context.cache_dir)?;
|
||||
|
||||
// Cached sdist path resembling the uwsgi==2.0.31 build failure.
|
||||
let uwsgi_shard = context
|
||||
.cache_dir
|
||||
.child("sdists-v9")
|
||||
.child("pypi")
|
||||
.child("uwsgi")
|
||||
.child("2.0.31")
|
||||
.child("QxDIp0qpjbsWjWURKmegK")
|
||||
.child("src")
|
||||
.child("core");
|
||||
|
||||
// Attempt to create a file with a trailing dot (we need to make it verbatim to do so)
|
||||
uwsgi_shard.create_dir_all()?;
|
||||
let invalid_path = uwsgi_shard.child("logging.").to_path_buf();
|
||||
let invalid_file = uv_fs::verbatim_path(invalid_path.as_path());
|
||||
fs_err::write(&invalid_file, b"")?;
|
||||
|
||||
// Confirm Win32 normalized path causes an os error when attempting to remove
|
||||
let remove_err = fs_err::remove_file(&invalid_path).expect_err("expected to fail");
|
||||
assert_eq!(remove_err.kind(), std::io::ErrorKind::NotFound);
|
||||
|
||||
// Tests cache clean leverages verbatim conversion
|
||||
uv_snapshot!(context.filters(), context.clean().arg("--verbose"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
----- stderr -----
|
||||
DEBUG uv [VERSION] ([COMMIT] DATE)
|
||||
Clearing cache at: [CACHE_DIR]/
|
||||
Removed 2 files
|
||||
");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user