33b35f7020
Adds support for disabling installation from pre-built wheels i.e. the
package must be built from source locally.
We will still always use pre-built wheels for metadata during
resolution.
Available via `--no-binary` and `--no-binary-package <name>` flags in
`pip install` and `pip sync`. There is no flag for `pip compile` since
no installation happens there.
```
--no-binary
Don't install pre-built wheels.
When enabled, all installed packages will be installed from a source distribution.
The resolver will still use pre-built wheels for metadata.
--no-binary-package <NO_BINARY_PACKAGE>
Don't install pre-built wheels for a specific package.
When enabled, the specified packages will be installed from a source distribution.
The resolver will still use pre-built wheels for metadata.
```
When packages are already installed, the `--no-binary` flag will have no
affect without the `--reinstall` flag. In the future, I'd like to change
this by tracking if a local distribution is from a pre-built wheel or a
locally-built wheel. However, this is significantly more complex and
different than `pip`'s behavior so deferring for now.
For reference, `pip`'s flag works as follows:
```
--no-binary <format_control>
Do not use binary packages. Can be supplied multiple times, and each time adds to the
existing value. Accepts either ":all:" to disable all binary packages, ":none:" to empty the
set (notice the colons), or one or more package names with commas between them (no colons).
Note that some packages are tricky to compile and may fail to install when this option is
used on them.
```
Note we are not matching the exact `pip` interface here because it seems
complicated to use. I think we may want to consider adjusting our
interface for this behavior since we're not entirely compatible anyway
e.g. I think `--force-build` and `--force-build-package` are clearer
names. We could also consider matching the `pip` interface or only
allowing `--no-binary <package>` for compatibility. We can of course do
whatever we want in our _own_ install interfaces later.
Additionally, we may want to further consider the semantics of
`--no-binary`. For example, if I run `pip install pydantic --no-binary`
I expect _just_ Pydantic to be installed without binaries but by default
we will build all of Pydantic's dependencies too.
This work was prompted by #895, as it is much easier to measure
performance gains from building source distributions if we have a flag
to ensure we actually build source distributions. Additionally, this is
a flag I have used frequently in production to debug packages that ship
Cythonized wheels.
138 lines
4.3 KiB
Rust
138 lines
4.3 KiB
Rust
use std::io::{BufWriter, Write};
|
|
use std::path::PathBuf;
|
|
|
|
use anstream::println;
|
|
use anyhow::{Context, Result};
|
|
use chrono::{DateTime, Utc};
|
|
use clap::{Parser, ValueEnum};
|
|
use fs_err::File;
|
|
use itertools::Itertools;
|
|
use petgraph::dot::{Config as DotConfig, Dot};
|
|
|
|
use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution};
|
|
use pep508_rs::Requirement;
|
|
use platform_host::Platform;
|
|
use puffin_cache::{Cache, CacheArgs};
|
|
use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder};
|
|
use puffin_dispatch::BuildDispatch;
|
|
use puffin_installer::NoBinary;
|
|
use puffin_interpreter::Virtualenv;
|
|
use puffin_resolver::{InMemoryIndex, Manifest, ResolutionOptions, Resolver};
|
|
use puffin_traits::{InFlight, SetupPyStrategy};
|
|
|
|
#[derive(ValueEnum, Default, Clone)]
|
|
pub(crate) enum ResolveCliFormat {
|
|
#[default]
|
|
Compact,
|
|
Expanded,
|
|
}
|
|
|
|
#[derive(Parser)]
|
|
pub(crate) struct ResolveCliArgs {
|
|
requirements: Vec<Requirement>,
|
|
/// Write debug output in DOT format for graphviz to this file
|
|
#[clap(long)]
|
|
graphviz: Option<PathBuf>,
|
|
/// Don't build source distributions. This means resolving will not run arbitrary code. The
|
|
/// cached wheels of already built source distributions will be reused.
|
|
#[clap(long)]
|
|
no_build: bool,
|
|
#[clap(long, default_value = "compact")]
|
|
format: ResolveCliFormat,
|
|
#[command(flatten)]
|
|
cache_args: CacheArgs,
|
|
#[arg(long)]
|
|
exclude_newer: Option<DateTime<Utc>>,
|
|
#[clap(long, short, default_value = IndexUrl::Pypi.as_str(), env = "PUFFIN_INDEX_URL")]
|
|
index_url: IndexUrl,
|
|
#[clap(long)]
|
|
extra_index_url: Vec<IndexUrl>,
|
|
#[clap(long)]
|
|
find_links: Vec<FlatIndexLocation>,
|
|
}
|
|
|
|
pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
|
|
let cache = Cache::try_from(args.cache_args)?;
|
|
|
|
let platform = Platform::current()?;
|
|
let venv = Virtualenv::from_env(platform, &cache)?;
|
|
let index_locations =
|
|
IndexLocations::from_args(args.index_url, args.extra_index_url, args.find_links, false);
|
|
let client = RegistryClientBuilder::new(cache.clone())
|
|
.index_urls(index_locations.index_urls())
|
|
.build();
|
|
let flat_index = {
|
|
let client = FlatIndexClient::new(&client, &cache);
|
|
let entries = client.fetch(index_locations.flat_indexes()).await?;
|
|
FlatIndex::from_entries(entries, venv.interpreter().tags()?)
|
|
};
|
|
let index = InMemoryIndex::default();
|
|
let in_flight = InFlight::default();
|
|
|
|
let build_dispatch = BuildDispatch::new(
|
|
&client,
|
|
&cache,
|
|
venv.interpreter(),
|
|
&index_locations,
|
|
&flat_index,
|
|
&index,
|
|
&in_flight,
|
|
venv.python_executable(),
|
|
SetupPyStrategy::default(),
|
|
args.no_build,
|
|
&NoBinary::None,
|
|
);
|
|
|
|
// Copied from `BuildDispatch`
|
|
let tags = venv.interpreter().tags()?;
|
|
let resolver = Resolver::new(
|
|
Manifest::simple(args.requirements.clone()),
|
|
ResolutionOptions::default(),
|
|
venv.interpreter().markers(),
|
|
venv.interpreter(),
|
|
tags,
|
|
&client,
|
|
&flat_index,
|
|
&index,
|
|
&build_dispatch,
|
|
);
|
|
let resolution_graph = resolver.resolve().await.with_context(|| {
|
|
format!(
|
|
"No solution found when resolving: {}",
|
|
args.requirements.iter().map(ToString::to_string).join(", "),
|
|
)
|
|
})?;
|
|
|
|
if let Some(graphviz) = args.graphviz {
|
|
let mut writer = BufWriter::new(File::create(graphviz)?);
|
|
let graphviz = Dot::with_attr_getters(
|
|
resolution_graph.petgraph(),
|
|
&[DotConfig::NodeNoLabel, DotConfig::EdgeNoLabel],
|
|
&|_graph, edge_ref| format!("label={:?}", edge_ref.weight().to_string()),
|
|
&|_graph, (_node_index, dist)| {
|
|
format!(
|
|
"label={:?}",
|
|
dist.to_string().replace("==", "\n").to_string()
|
|
)
|
|
},
|
|
);
|
|
write!(&mut writer, "{graphviz:?}")?;
|
|
}
|
|
|
|
let requirements = Resolution::from(resolution_graph).requirements();
|
|
|
|
#[allow(clippy::print_stderr)]
|
|
match args.format {
|
|
ResolveCliFormat::Compact => {
|
|
println!("{}", requirements.iter().map(ToString::to_string).join(" "));
|
|
}
|
|
ResolveCliFormat::Expanded => {
|
|
for package in requirements {
|
|
println!("{}", package);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|