63f7f65190
This copies the allocator configuration used in the Ruff project. In
particular, this gives us an instant 10% win when resolving the top 1K
PyPI packages:
$ hyperfine \
"./target/profiling/puffin-dev-main resolve-many --cache-dir
cache-docker-no-build --no-build pypi_top_8k_flat.txt --limit 1000 2>
/dev/null" \
"./target/profiling/puffin-dev resolve-many --cache-dir
cache-docker-no-build --no-build pypi_top_8k_flat.txt --limit 1000 2>
/dev/null"
Benchmark 1: ./target/profiling/puffin-dev-main resolve-many --cache-dir
cache-docker-no-build --no-build pypi_top_8k_flat.txt --limit 1000 2>
/dev/null
Time (mean ± σ): 974.2 ms ± 26.4 ms [User: 17503.3 ms, System: 2205.3
ms]
Range (min … max): 943.5 ms … 1015.9 ms 10 runs
Benchmark 2: ./target/profiling/puffin-dev resolve-many --cache-dir
cache-docker-no-build --no-build pypi_top_8k_flat.txt --limit 1000 2>
/dev/null
Time (mean ± σ): 883.1 ms ± 23.3 ms [User: 14626.1 ms, System: 2542.2
ms]
Range (min … max): 849.5 ms … 916.9 ms 10 runs
Summary
'./target/profiling/puffin-dev resolve-many --cache-dir
cache-docker-no-build --no-build pypi_top_8k_flat.txt --limit 1000 2>
/dev/null' ran
1.10 ± 0.04 times faster than './target/profiling/puffin-dev-main
resolve-many --cache-dir cache-docker-no-build --no-build
pypi_top_8k_flat.txt --limit 1000 2> /dev/null'
I was moved to do this because I noticed `malloc`/`free` taking up a
fairly sizeable percentage of time during light profiling.
As is becoming a pattern, it will be easier to review this
commit-by-commit.
Ref #396 (wouldn't call this issue fixed)
-----
I did also try adding a `smallvec` optimization to the
`Version::release` field, but it didn't bare any fruit. I still think
there is more to explore since the results I observed don't quite line
up with what I expect. (So probably either my mental model is off or my
measurement process is flawed.) You can see that attempt with a little
more explanation here:
https://github.com/astral-sh/puffin/commit/f9528b4ecd1b0c260df7e8ad57b9ddc4da09d273
In the course of adding the `smallvec` optimization, I also shrunk the
`Version` fields from a `usize` to a `u32`. They should at least be a
fixed size integer since version numbers aren't used to index memory,
and I shrunk it to `u32` since it seems reasonable to assume that all
version numbers will be smaller than `2^32`.
107 lines
3.0 KiB
Rust
107 lines
3.0 KiB
Rust
#![allow(clippy::print_stdout, clippy::print_stderr)]
|
|
|
|
use std::process::ExitCode;
|
|
use std::time::Instant;
|
|
|
|
use anyhow::Result;
|
|
use clap::Parser;
|
|
use colored::Colorize;
|
|
use tracing::debug;
|
|
use tracing_indicatif::IndicatifLayer;
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
use tracing_subscriber::util::SubscriberInitExt;
|
|
use tracing_subscriber::EnvFilter;
|
|
|
|
use resolve_many::ResolveManyArgs;
|
|
|
|
use crate::build::{build, BuildArgs};
|
|
use crate::resolve_cli::ResolveCliArgs;
|
|
use crate::wheel_metadata::WheelMetadataArgs;
|
|
|
|
#[cfg(target_os = "windows")]
|
|
#[global_allocator]
|
|
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
|
|
|
#[cfg(all(
|
|
not(target_os = "windows"),
|
|
not(target_os = "openbsd"),
|
|
any(
|
|
target_arch = "x86_64",
|
|
target_arch = "aarch64",
|
|
target_arch = "powerpc64"
|
|
)
|
|
))]
|
|
#[global_allocator]
|
|
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
|
|
|
|
mod build;
|
|
mod resolve_cli;
|
|
mod resolve_many;
|
|
mod wheel_metadata;
|
|
|
|
#[derive(Parser)]
|
|
enum Cli {
|
|
/// Build a source distribution into a wheel
|
|
Build(BuildArgs),
|
|
/// Resolve many requirements independently in parallel and report failures and sucesses.
|
|
///
|
|
/// Run `scripts/resolve/get_pypi_top_8k.sh` once, then
|
|
/// ```bash
|
|
/// cargo run --bin puffin-dev -- resolve-many scripts/resolve/pypi_top_8k_flat.txt
|
|
/// ```
|
|
ResolveMany(ResolveManyArgs),
|
|
/// Resolve requirements passed on the CLI
|
|
ResolveCli(ResolveCliArgs),
|
|
WheelMetadata(WheelMetadataArgs),
|
|
}
|
|
|
|
async fn run() -> Result<()> {
|
|
let cli = Cli::parse();
|
|
match cli {
|
|
Cli::Build(args) => {
|
|
let target = build(args).await?;
|
|
println!("Wheel built to {}", target.display());
|
|
}
|
|
Cli::ResolveMany(args) => {
|
|
resolve_many::resolve_many(args).await?;
|
|
}
|
|
Cli::ResolveCli(args) => {
|
|
resolve_cli::resolve_cli(args).await?;
|
|
}
|
|
Cli::WheelMetadata(args) => wheel_metadata::wheel_metadata(args).await?,
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> ExitCode {
|
|
let indicatif_layer = IndicatifLayer::new();
|
|
let indicitif_compatible_writer_layer = tracing_subscriber::fmt::layer()
|
|
.with_writer(indicatif_layer.get_stderr_writer())
|
|
.with_target(false);
|
|
let filter_layer = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
|
|
EnvFilter::builder()
|
|
// Show only the important spans
|
|
.parse("puffin_dev=info,puffin_dispatch=info")
|
|
.unwrap()
|
|
});
|
|
tracing_subscriber::registry()
|
|
.with(filter_layer)
|
|
.with(indicitif_compatible_writer_layer)
|
|
.with(indicatif_layer)
|
|
.init();
|
|
|
|
let start = Instant::now();
|
|
let result = run().await;
|
|
debug!("Took {}ms", start.elapsed().as_millis());
|
|
if let Err(err) = result {
|
|
eprintln!("{}", "puffin-dev failed".red().bold());
|
|
for err in err.chain() {
|
|
eprintln!(" {}: {}", "Caused by".red().bold(), err);
|
|
}
|
|
ExitCode::FAILURE
|
|
} else {
|
|
ExitCode::SUCCESS
|
|
}
|
|
}
|