Run benchmarks in offline mode (#18487)

Currently, we have the simulated benchmarks on codspeed turned of, they
flake to much
(https://codspeed.io/astral-sh/uv/benchmarks/crates/uv-bench/benches/uv.rs::uv::resolve_warm_airflow::resolve_warm_airflow?runnerMode=Simulation&period=1m),
while the walltime benchmarks are kinda stable.

<img width="1819" height="996" alt="image"
src="https://github.com/user-attachments/assets/ffd7b5b5-3bf9-427b-92dc-0bb4fbfbb3d0"
/>

<img width="1819" height="955" alt="image"
src="https://github.com/user-attachments/assets/8d1a7519-4214-4564-b7a5-c899bff4f7b4"
/>

Looking into two of these spurious regressions, we see that they occur
in the network code, and that the CI runs for them took >10min, while
regularly runs take <10min.

*
https://codspeed.io/astral-sh/uv/runs/compare/699738846a7b43e19b4d2e67..699778f1f92bec70ac9b3024?uri=crates%2Fuv-bench%2Fbenches%2Fuv.rs%3A%3Auv%3A%3Aresolve_warm_airflow%3A%3Aresolve_warm_airflow&runnerMode=Simulation
*
https://github.com/astral-sh/uv/actions/runs/22199337000/job/64207958534
*
https://codspeed.io/astral-sh/uv/runs/compare/699caa16bfa44e7f9c0b44c5..699cc8e475e640df09b7f01f?uri=crates%2Fuv-bench%2Fbenches%2Fuv.rs%3A%3Auv%3A%3Aresolve_warm_airflow%3A%3Aresolve_warm_airflow&runnerMode=Simulation
*
https://github.com/astral-sh/uv/actions/runs/22325481213/job/64594783994

<img width="1819" height="973" alt="image"
src="https://github.com/user-attachments/assets/e5c7c260-3a93-4e25-bc00-fdf045edc098"
/>

uv caches PyPI responses for 10min, after that, it has to make
revalidation request. If the build takes >7min, and we prime the caches
before the build, the most likely explanation is that these runs have to
make revalidation requests. This is not a problem for walltime
benchmarks, which run multiple times, where revalidation requests in one
run are just an outlier that gets ignored overall.

The fix is that the benchmark itself primes the cache, and then runs in
offline mode. We keep the cache priming in the GitHub action before the
benchmark builds for the sake of the job logs, while we need the priming
inside the bench for running `cargo bench` locally without extra setup.
Running the benches themselves with an offline client make uv ignore the
10min threshold.

crates/uv-bench/benches/uv.rs

@@ -3,44 +3,48 @@ use std::str::FromStr;
 
 use criterion::{Criterion, criterion_group, criterion_main, measurement::WallTime};
 use uv_cache::Cache;
-use uv_client::{BaseClientBuilder, RegistryClientBuilder};
+use uv_client::{BaseClientBuilder, Connectivity, RegistryClientBuilder};
 use uv_distribution_types::Requirement;
 use uv_python::PythonEnvironment;
 use uv_resolver::Manifest;
 
 fn resolve_warm_jupyter(c: &mut Criterion<WallTime>) {
-    let run = setup(Manifest::simple(vec![Requirement::from(
+    let manifest = Manifest::simple(vec![Requirement::from(
         uv_pep508::Requirement::from_str("jupyter==1.0.0").unwrap(),
-    )]));
-    c.bench_function("resolve_warm_jupyter", |b| b.iter(|| run(false)));
+    )]);
+    let run = setup(manifest, false);
+    c.bench_function("resolve_warm_jupyter", |b| b.iter(&run));
 }
 
 fn resolve_warm_jupyter_universal(c: &mut Criterion<WallTime>) {
-    let run = setup(Manifest::simple(vec![Requirement::from(
+    let manifest = Manifest::simple(vec![Requirement::from(
         uv_pep508::Requirement::from_str("jupyter==1.0.0").unwrap(),
-    )]));
-    c.bench_function("resolve_warm_jupyter_universal", |b| b.iter(|| run(true)));
+    )]);
+    let run = setup(manifest, true);
+    c.bench_function("resolve_warm_jupyter_universal", |b| b.iter(&run));
 }
 
 fn resolve_warm_airflow(c: &mut Criterion<WallTime>) {
-    let run = setup(Manifest::simple(vec![
+    let manifest = Manifest::simple(vec![
         Requirement::from(uv_pep508::Requirement::from_str("apache-airflow[all]==2.9.3").unwrap()),
         Requirement::from(
             uv_pep508::Requirement::from_str("apache-airflow-providers-apache-beam>3.0.0").unwrap(),
         ),
-    ]));
-    c.bench_function("resolve_warm_airflow", |b| b.iter(|| run(false)));
+    ]);
+    let run = setup(manifest, false);
+    c.bench_function("resolve_warm_airflow", |b| b.iter(&run));
 }
 
 // This takes >5m to run in CodSpeed.
 // fn resolve_warm_airflow_universal(c: &mut Criterion<WallTime>) {
-//     let run = setup(Manifest::simple(vec![
+//     let manifest = Manifest::simple(vec![
 //         Requirement::from(uv_pep508::Requirement::from_str("apache-airflow[all]").unwrap()),
 //         Requirement::from(
 //             uv_pep508::Requirement::from_str("apache-airflow-providers-apache-beam>3.0.0").unwrap(),
 //         ),
-//     ]));
-//     c.bench_function("resolve_warm_airflow_universal", |b| b.iter(|| run(true)));
+//     ]);
+//     let run = setup(manifest, true);
+//     c.bench_function("resolve_warm_airflow_universal", |b| b.iter(&run));
 // }
 
 criterion_group!(
@@ -51,7 +55,7 @@ criterion_group!(
 );
 criterion_main!(uv);
 
-fn setup(manifest: Manifest) -> impl Fn(bool) {
+fn setup(manifest: Manifest, universal: bool) -> impl Fn() {
     let runtime = tokio::runtime::Builder::new_current_thread()
         // CodSpeed limits the total number of threads to 500
         .max_blocking_threads(256)
@@ -68,7 +72,29 @@ fn setup(manifest: Manifest) -> impl Fn(bool) {
         .into_interpreter();
     let client = RegistryClientBuilder::new(BaseClientBuilder::default(), cache.clone()).build();
 
-    move |universal| {
+    // Prime the cache: First run for performance the network operation, the second run primes
+    // reading from the cache from the first run. If they are already primed, we only lose ~1s for
+    // the large airflow benchmark.
+    for _ in 0..2 {
+        runtime
+            .block_on(resolver::resolve(
+                black_box(manifest.clone()),
+                black_box(cache.clone()),
+                black_box(&client),
+                &interpreter,
+                universal,
+            ))
+            .unwrap();
+    }
+
+    // No matter how long the benchmarks run, never do fresh network requests
+    let client = RegistryClientBuilder::new(
+        BaseClientBuilder::default().connectivity(Connectivity::Offline),
+        cache.clone(),
+    )
+    .build();
+
+    move || {
         runtime
             .block_on(resolver::resolve(
                 black_box(manifest.clone()),