pioduino/uv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Minor improvements to benchmarking setup (#5843)

Browse Source
This commit is contained in:
Charlie Marsh
2024-08-06 19:47:45 -04:00
committed by GitHub
parent 9b06b3905d
commit f3cc8e4790
6 changed files with 412 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark/src/benchmark/__init__.py
+14 -6
View File
@@ -21,12 +21,15 @@ class Hyperfine(typing.NamedTuple):
commands: list[Command]
"""The commands to benchmark."""
warmup: int
warmup: int | None
"""The number of warmup runs to perform."""
min_runs: int
min_runs: int | None
"""The minimum number of runs to perform."""
runs: int | None
"""The number of runs to perform."""
verbose: bool
"""Whether to print verbose output."""
@@ -45,10 +48,15 @@ class Hyperfine(typing.NamedTuple):
# Preamble: benchmark-wide setup.
if self.verbose:
args.append("--show-output")
args.append("--warmup")
args.append(str(self.warmup))
args.append("--min-runs")
args.append(str(self.min_runs))
if self.warmup is not None:
args.append("--warmup")
args.append(str(self.warmup))
if self.min_runs is not None:
args.append("--min-runs")
args.append(str(self.min_runs))
if self.runs is not None:
args.append("--runs")
args.append(str(self.runs))
# Add all command names,
for command in self.commands:
scripts/benchmark/src/benchmark/resolver.py
+23 -15
View File
@@ -28,13 +28,13 @@ By default, the script will run the resolution benchmarks when a `requirements.i
is provided, and the installation benchmarks when a `requirements.txt` file is provided:
# Run the resolution benchmarks against the Trio project.
uv run benchmark\
uv run resolver \
--uv-path ../../target/release/uv \
--uv-path ../../target/release/baseline \
../requirements/trio.in
# Run the installation benchmarks against the Trio project.
uv run benchmark\
uv run resolver \
--uv-path ../../target/release/uv \
--uv-path ../../target/release/baseline \
../requirements/compiled/trio.txt
@@ -42,7 +42,7 @@ is provided, and the installation benchmarks when a `requirements.txt` file is p
You can also specify the benchmark to run explicitly:
# Run the "uncached install" benchmark against the Trio project.
uv run benchmark\
uv run resolver \
--uv-path ../../target/release/uv \
--uv-path ../../target/release/baseline \
--benchmark install-cold \
@@ -302,7 +302,7 @@ class PipSync(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12.3 {venv_dir}",
command=[
self.path,
os.path.abspath(requirements_file),
@@ -319,7 +319,7 @@ class PipSync(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
command=[
self.path,
os.path.abspath(requirements_file),
@@ -559,7 +559,7 @@ class Poetry(Suite):
f"rm -rf {config_dir} && "
f"rm -rf {cache_dir} && "
f"rm -rf {data_dir} &&"
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
),
command=[
f"POETRY_CONFIG_DIR={config_dir}",
@@ -599,7 +599,7 @@ class Poetry(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
command=[
f"POETRY_CONFIG_DIR={config_dir}",
f"POETRY_CACHE_DIR={cache_dir}",
@@ -636,7 +636,7 @@ class Pdm(Suite):
# Create a PDM project.
subprocess.check_call(
[self.path, "init", "--non-interactive", "--python", "3.12"],
[self.path, "init", "--non-interactive", "--python", "3.12.3"],
cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
@@ -792,7 +792,7 @@ class Pdm(Suite):
prepare=(
f"rm -rf {cache_dir} && "
f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
),
command=[
f"VIRTUAL_ENV={venv_dir}",
@@ -826,7 +826,7 @@ class Pdm(Suite):
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=(
f"{self.path} config cache_dir {cache_dir} && "
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
),
command=[
f"VIRTUAL_ENV={venv_dir}",
@@ -841,7 +841,7 @@ class Pdm(Suite):
class UvPip(Suite):
def __init__(self, *, path: str | None = None) -> Command | None:
"""Initialize a uv benchmark."""
self.name = path or "uv"
self.name = path or "uv pip"
self.path = path or os.path.join(
os.path.dirname(
os.path.dirname(
@@ -967,7 +967,7 @@ class UvPip(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12.3 {venv_dir}",
command=[
f"VIRTUAL_ENV={venv_dir}",
self.path,
@@ -985,7 +985,7 @@ class UvPip(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
prepare=f"virtualenv --clear -p 3.12.3 {venv_dir}",
command=[
f"VIRTUAL_ENV={venv_dir}",
self.path,
@@ -1207,7 +1207,7 @@ class UvProject(Suite):
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=(
f"rm -rf {cache_dir} && "
f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"
),
command=[
f"VIRTUAL_ENV={venv_dir}",
@@ -1243,7 +1243,7 @@ class UvProject(Suite):
return Command(
name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
prepare=(f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"),
prepare=(f"virtualenv --clear -p 3.12.3 {venv_dir} --no-seed"),
command=[
f"VIRTUAL_ENV={venv_dir}",
self.path,
@@ -1287,6 +1287,11 @@ def main():
help="The minimum number of runs to perform.",
default=10,
)
parser.add_argument(
"--runs",
type=int,
help="The number of runs to perform.",
)
parser.add_argument(
"--benchmark",
"-b",
@@ -1373,6 +1378,7 @@ def main():
json = args.json
warmup = args.warmup
min_runs = args.min_runs
runs = args.runs
requirements_file = os.path.abspath(args.file)
if not os.path.exists(requirements_file):
@@ -1411,6 +1417,7 @@ def main():
PipSync(),
PipCompile(),
Poetry(),
Pdm(),
UvPip(),
UvProject(),
]
@@ -1453,6 +1460,7 @@ def main():
commands=commands,
warmup=warmup,
min_runs=min_runs,
runs=runs,
verbose=verbose,
json=json,
)
scripts/benchmark/src/benchmark/tools.py
+7
View File
@@ -245,6 +245,11 @@ def main():
help="The minimum number of runs to perform.",
default=10,
)
parser.add_argument(
"--runs",
type=int,
help="The number of runs to perform.",
)
parser.add_argument(
"--benchmark",
"-b",
@@ -287,6 +292,7 @@ def main():
json = args.json
warmup = args.warmup
min_runs = args.min_runs
runs = args.runs
# Determine the tools to benchmark, based on the user-provided arguments.
suites = []
@@ -328,6 +334,7 @@ def main():
commands=commands,
warmup=warmup,
min_runs=min_runs,
runs=runs,
verbose=verbose,
json=json,
)