Files
uv/crates/uv-extract/src/sync.rs
T
konsti c94777fc54 Initialize rayon lazily (#9435)
When performing a noop sync, we don't need the rayon threadpool, yet we
pay for its initialization:

![Screenshot from 2024-11-26
08-59-07](https://github.com/user-attachments/assets/d918f50d-b5b7-4bdd-820d-cbe71b633aaa)

Be making the initialization lazy, we avoid that cost:

![Screenshot from 2024-11-26
09-53-08](https://github.com/user-attachments/assets/193baea0-667f-4b9d-9a75-886a86f0f837)

This code runs every time before user code in `uv run`.

This means that before calling rayon, one now needs to call
`LazyLock::force(&RAYON_INITIALIZE);`.

Performance mode (CPU 0 is a perf core):
```
$ taskset -c 0 hyperfine --warmup 5 -N "/home/konsti/projects/uv/uv-main sync" "/home/konsti/projects/uv/target/profiling/uv sync"
Benchmark 1: /home/konsti/projects/uv/uv-main sync
  Time (mean ± σ):       4.5 ms ±   0.1 ms    [User: 2.7 ms, System: 1.8 ms]
  Range (min … max):     4.4 ms …   6.4 ms    640 runs
 
  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.
 
Benchmark 2: /home/konsti/projects/uv/target/profiling/uv sync
  Time (mean ± σ):       4.4 ms ±   0.1 ms    [User: 2.7 ms, System: 1.6 ms]
  Range (min … max):     4.3 ms …   5.0 ms    679 runs
 
Summary
  /home/konsti/projects/uv/target/profiling/uv sync ran
    1.03 ± 0.04 times faster than /home/konsti/projects/uv/uv-main sync
```

Power saver mode:
```
$ hyperfine --warmup 5 -N "/home/konsti/projects/uv/uv-main sync" "/home/konsti/projects/uv/target/profiling/uv sync"
Benchmark 1: /home/konsti/projects/uv/uv-main sync
  Time (mean ± σ):      28.1 ms ±   1.2 ms    [User: 15.5 ms, System: 20.3 ms]
  Range (min … max):    25.7 ms …  31.9 ms    102 runs
 
Benchmark 2: /home/konsti/projects/uv/target/profiling/uv sync
  Time (mean ± σ):      24.0 ms ±   1.2 ms    [User: 13.8 ms, System: 9.9 ms]
  Range (min … max):    22.2 ms …  28.2 ms    122 runs
 
Summary
  /home/konsti/projects/uv/target/profiling/uv sync ran
    1.17 ± 0.08 times faster than /home/konsti/projects/uv/uv-main sync
```
2024-11-26 14:58:38 +00:00

113 lines
4.2 KiB
Rust

use std::path::{Path, PathBuf};
use std::sync::{LazyLock, Mutex};
use crate::vendor::{CloneableSeekableReader, HasLength};
use crate::Error;
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use tracing::warn;
use uv_configuration::RAYON_INITIALIZE;
use zip::ZipArchive;
/// Unzip a `.zip` archive into the target directory.
pub fn unzip<R: Send + std::io::Read + std::io::Seek + HasLength>(
reader: R,
target: &Path,
) -> Result<(), Error> {
// Unzip in parallel.
let reader = std::io::BufReader::new(reader);
let archive = ZipArchive::new(CloneableSeekableReader::new(reader))?;
let directories = Mutex::new(FxHashSet::default());
// Initialize the threadpool with the user settings.
LazyLock::force(&RAYON_INITIALIZE);
(0..archive.len())
.into_par_iter()
.map(|file_number| {
let mut archive = archive.clone();
let mut file = archive.by_index(file_number)?;
// Determine the path of the file within the wheel.
let Some(enclosed_name) = file.enclosed_name() else {
warn!("Skipping unsafe file name: {}", file.name());
return Ok(());
};
// Create necessary parent directories.
let path = target.join(enclosed_name);
if file.is_dir() {
let mut directories = directories.lock().unwrap();
if directories.insert(path.clone()) {
fs_err::create_dir_all(path)?;
}
return Ok(());
}
if let Some(parent) = path.parent() {
let mut directories = directories.lock().unwrap();
if directories.insert(parent.to_path_buf()) {
fs_err::create_dir_all(parent)?;
}
}
// Copy the file contents.
let outfile = fs_err::File::create(&path)?;
let size = file.size();
if size > 0 {
let mut writer = if let Ok(size) = usize::try_from(size) {
std::io::BufWriter::with_capacity(std::cmp::min(size, 1024 * 1024), outfile)
} else {
std::io::BufWriter::new(outfile)
};
std::io::copy(&mut file, &mut writer)?;
}
// See `uv_extract::stream::unzip`. For simplicity, this is identical with the code there except for being
// sync.
#[cfg(unix)]
{
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
if let Some(mode) = file.unix_mode() {
// https://github.com/pypa/pip/blob/3898741e29b7279e7bffe044ecfbe20f6a438b1e/src/pip/_internal/utils/unpacking.py#L88-L100
let has_any_executable_bit = mode & 0o111;
if has_any_executable_bit != 0 {
let permissions = fs_err::metadata(&path)?.permissions();
if permissions.mode() & 0o111 != 0o111 {
fs_err::set_permissions(
&path,
Permissions::from_mode(permissions.mode() | 0o111),
)?;
}
}
}
}
Ok(())
})
.collect::<Result<_, Error>>()
}
/// Extract the top-level directory from an unpacked archive.
///
/// The specification says:
/// > A .tar.gz source distribution (sdist) contains a single top-level directory called
/// > `{name}-{version}` (e.g. foo-1.0), containing the source files of the package.
///
/// This function returns the path to that top-level directory.
pub fn strip_component(source: impl AsRef<Path>) -> Result<PathBuf, Error> {
// TODO(konstin): Verify the name of the directory.
let top_level =
fs_err::read_dir(source.as_ref())?.collect::<std::io::Result<Vec<fs_err::DirEntry>>>()?;
match top_level.as_slice() {
[root] => Ok(root.path()),
[] => Err(Error::EmptyArchive),
_ => Err(Error::NonSingularArchive(
top_level
.into_iter()
.map(|entry| entry.file_name())
.collect(),
)),
}
}