2e88bb6f1b
Given requirements like: ``` black==23.1.0 black[colorama] ``` The resolver will (on `main`) add a dependency on Black, and then try to use the most recent version of Black to satisfy `black[colorama]`. For sake of example, assume `black==24.0.0` is the most recent version. Once the selects this most recent version, it'll fetch the metadata, then return the dependencies for `black==24.0.0` with the `colorama` extra enabled. Finally, it will tack on `black==24.0.0` (a dependency on the base package). The resolver will then detect a conflict between `black==23.1.0` and `black==24.0.0`, and throw out `black[colorama]==24.0.0`, trying to next most-recent version. This is both wasteful and can cause problems, since we're fetching metadata for versions that will _never_ satisfy the resolver. In the `apache-airflow[all]` case, I also ran into an issue whereby we were attempting to build very old versions of `apache-airflow` due to `apache-airflow[pandas]`, which in turn led to resolution failures. The solution proposed here is that we create a new proxy package with exactly two dependencies: one on `black` and one of `black[colorama]`. Both of these packages must be at the same version as the proxy package, so the resolver knows much _earlier_ that (in the above example) the extra variant _must_ match `23.1.0`.
129 lines
5.7 KiB
Rust
129 lines
5.7 KiB
Rust
use std::cmp::Reverse;
|
|
|
|
use pubgrub::range::Range;
|
|
use rustc_hash::FxHashMap;
|
|
|
|
use pep440_rs::Version;
|
|
use uv_normalize::PackageName;
|
|
|
|
use crate::pubgrub::package::PubGrubPackage;
|
|
|
|
/// A prioritization map to guide the `PubGrub` resolution process.
|
|
///
|
|
/// During resolution, `PubGrub` needs to decide which package to consider next. The priorities
|
|
/// encoded here are used to guide that decision.
|
|
///
|
|
/// Like `pip`, we prefer packages that are pinned to direct URLs over packages pinned to a single
|
|
/// version over packages that are constrained in some way over packages that are unconstrained.
|
|
///
|
|
/// See: <https://github.com/pypa/pip/blob/ef78c129b1a966dbbbdb8ebfffc43723e89110d1/src/pip/_internal/resolution/resolvelib/provider.py#L120>
|
|
#[derive(Debug, Default)]
|
|
pub(crate) struct PubGrubPriorities(FxHashMap<PackageName, PubGrubPriority>);
|
|
|
|
impl PubGrubPriorities {
|
|
/// Add a [`PubGrubPackage`] to the priority map.
|
|
pub(crate) fn insert(&mut self, package: &PubGrubPackage, version: &Range<Version>) {
|
|
let next = self.0.len();
|
|
match package {
|
|
PubGrubPackage::Root(_) => {}
|
|
PubGrubPackage::Python(_) => {}
|
|
|
|
PubGrubPackage::Extra(name, _, None) | PubGrubPackage::Package(name, _, None) => {
|
|
match self.0.entry(name.clone()) {
|
|
std::collections::hash_map::Entry::Occupied(mut entry) => {
|
|
// Preserve the original index.
|
|
let index = match entry.get() {
|
|
PubGrubPriority::Singleton(Reverse(index)) => *index,
|
|
PubGrubPriority::Unconstrained(Reverse(index)) => *index,
|
|
PubGrubPriority::Constrained(Reverse(index)) => *index,
|
|
PubGrubPriority::DirectUrl(Reverse(index)) => *index,
|
|
PubGrubPriority::Root => next,
|
|
};
|
|
|
|
// Compute the priority.
|
|
let priority = if version.as_singleton().is_some() {
|
|
PubGrubPriority::Singleton(Reverse(index))
|
|
} else if version == &Range::full() {
|
|
PubGrubPriority::Unconstrained(Reverse(index))
|
|
} else {
|
|
PubGrubPriority::Constrained(Reverse(index))
|
|
};
|
|
|
|
// Take the maximum of the new and existing priorities.
|
|
if priority > *entry.get() {
|
|
entry.insert(priority);
|
|
}
|
|
}
|
|
std::collections::hash_map::Entry::Vacant(entry) => {
|
|
// Insert the priority.
|
|
entry.insert(if version.as_singleton().is_some() {
|
|
PubGrubPriority::Singleton(Reverse(next))
|
|
} else if version == &Range::full() {
|
|
PubGrubPriority::Unconstrained(Reverse(next))
|
|
} else {
|
|
PubGrubPriority::Constrained(Reverse(next))
|
|
});
|
|
}
|
|
}
|
|
}
|
|
PubGrubPackage::Extra(name, _, Some(_)) | PubGrubPackage::Package(name, _, Some(_)) => {
|
|
match self.0.entry(name.clone()) {
|
|
std::collections::hash_map::Entry::Occupied(mut entry) => {
|
|
// Preserve the original index.
|
|
let index = match entry.get() {
|
|
PubGrubPriority::Singleton(Reverse(index)) => *index,
|
|
PubGrubPriority::Unconstrained(Reverse(index)) => *index,
|
|
PubGrubPriority::Constrained(Reverse(index)) => *index,
|
|
PubGrubPriority::DirectUrl(Reverse(index)) => *index,
|
|
PubGrubPriority::Root => next,
|
|
};
|
|
|
|
// Compute the priority.
|
|
let priority = PubGrubPriority::DirectUrl(Reverse(index));
|
|
|
|
// Take the maximum of the new and existing priorities.
|
|
if priority > *entry.get() {
|
|
entry.insert(priority);
|
|
}
|
|
}
|
|
std::collections::hash_map::Entry::Vacant(entry) => {
|
|
// Insert the priority.
|
|
entry.insert(PubGrubPriority::DirectUrl(Reverse(next)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Return the [`PubGrubPriority`] of the given package, if it exists.
|
|
pub(crate) fn get(&self, package: &PubGrubPackage) -> Option<PubGrubPriority> {
|
|
match package {
|
|
PubGrubPackage::Root(_) => Some(PubGrubPriority::Root),
|
|
PubGrubPackage::Python(_) => Some(PubGrubPriority::Root),
|
|
PubGrubPackage::Extra(name, _, _) => self.0.get(name).copied(),
|
|
PubGrubPackage::Package(name, _, _) => self.0.get(name).copied(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
pub(crate) enum PubGrubPriority {
|
|
/// The package has no specific priority.
|
|
///
|
|
/// As such, its priority is based on the order in which the packages were added (FIFO), such
|
|
/// that the first package we visit is prioritized over subsequent packages.
|
|
Unconstrained(Reverse<usize>),
|
|
|
|
/// The version range is constrained in some way (e.g., with a `<=` or `>` operator).
|
|
Constrained(Reverse<usize>),
|
|
|
|
/// The version range is constrained to a single version (e.g., with the `==` operator).
|
|
Singleton(Reverse<usize>),
|
|
|
|
/// The package was specified via a direct URL.
|
|
DirectUrl(Reverse<usize>),
|
|
|
|
/// The package is the root package.
|
|
Root,
|
|
}
|