From 8b0fad35604ccea4bef2c816131f4e101049cf43 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Wed, 8 May 2024 11:21:11 -0400 Subject: [PATCH] uv-resolver: make MarkerEnvironment optional This commit touches a lot of code, but the conceptual change here is pretty simple: make it so we can run the resolver without providing a `MarkerEnvironment`. This also indicates that the resolver should run in universal mode. That is, the effect of a missing marker environment is that all marker expressions that reference the marker environment are evaluated to `true`. That is, they are ignored. (The only markers we evaluate in that context are extras, which are the only markers that aren't dependent on the environment.) One interesting change here is that a `Resolver` no longer needs an `Interpreter`. Previously, it had only been using it to construct a `PythonRequirement`, by filling in the installed version from the `Interpreter` state. But we now construct a `PythonRequirement` explicitly since its `target` Python version should no longer be tied to the `MarkerEnvironment`. (Currently, the marker environment is mutated such that its `python_full_version` is derived from multiple sources, including the CLI, which I found a touch confusing.) The change in behavior can now be observed through the `--unstable-uv-lock-file` flag. First, without it: ``` $ cat requirements.in anyio>=4.3.0 ; sys_platform == "linux" anyio<4 ; sys_platform == "darwin" $ cargo run -qp uv -- pip compile -p3.10 requirements.in anyio==4.3.0 exceptiongroup==1.2.1 # via anyio idna==3.7 # via anyio sniffio==1.3.1 # via anyio typing-extensions==4.11.0 # via anyio ``` And now with it: ``` $ cargo run -qp uv -- pip compile -p3.10 requirements.in --unstable-uv-lock-file x No solution found when resolving dependencies: `-> Because you require anyio>=4.3.0 and anyio<4, we can conclude that the requirements are unsatisfiable. ``` This is expected at this point because the marker expressions are being explicitly ignored, *and* there is no forking done yet to account for the conflict. --- crates/bench/benches/uv.rs | 9 +++-- crates/distribution-types/src/requirement.rs | 15 +------- .../src/specified_requirement.rs | 9 ++++- crates/pep508-rs/src/unnamed.rs | 11 +++++- crates/uv-dev/src/resolve_cli.rs | 11 ++++-- crates/uv-dispatch/src/lib.rs | 8 ++-- crates/uv-installer/src/plan.rs | 2 +- crates/uv-installer/src/site_packages.rs | 2 +- crates/uv-requirements/src/lookahead.rs | 7 +++- crates/uv-resolver/src/candidate_selector.rs | 2 +- crates/uv-resolver/src/manifest.rs | 36 +++++++++--------- crates/uv-resolver/src/preferences.rs | 2 +- crates/uv-resolver/src/prerelease_mode.rs | 2 +- .../uv-resolver/src/pubgrub/dependencies.rs | 4 +- crates/uv-resolver/src/python_requirement.rs | 8 +++- crates/uv-resolver/src/resolution_mode.rs | 2 +- crates/uv-resolver/src/resolver/locals.rs | 2 +- crates/uv-resolver/src/resolver/mod.rs | 37 +++++++++++++------ crates/uv-resolver/src/resolver/urls.rs | 2 +- crates/uv-resolver/src/yanks.rs | 2 +- crates/uv-resolver/tests/resolver.rs | 8 ++-- crates/uv-types/src/hash.rs | 7 +++- crates/uv/src/commands/pip_compile.rs | 27 +++++++++----- crates/uv/src/commands/pip_install.rs | 12 +++--- crates/uv/src/commands/pip_sync.rs | 11 ++++-- crates/uv/src/commands/workspace/mod.rs | 10 +++-- 26 files changed, 153 insertions(+), 95 deletions(-) diff --git a/crates/bench/benches/uv.rs b/crates/bench/benches/uv.rs index 54e427072..400993f03 100644 --- a/crates/bench/benches/uv.rs +++ b/crates/bench/benches/uv.rs @@ -50,7 +50,9 @@ mod resolver { use uv_client::RegistryClient; use uv_configuration::{BuildKind, NoBinary, NoBuild, SetupPyStrategy}; use uv_interpreter::{Interpreter, PythonEnvironment}; - use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, ResolutionGraph, Resolver}; + use uv_resolver::{ + FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, ResolutionGraph, Resolver, + }; use uv_types::{ BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, SourceBuildTrait, }; @@ -93,12 +95,13 @@ mod resolver { let build_context = Context::new(cache, interpreter.clone()); let hashes = HashStrategy::None; let installed_packages = EmptyInstalledPackages; + let python_requirement = PythonRequirement::from_marker_environment(&interpreter, &MARKERS); let resolver = Resolver::new( manifest, Options::default(), - &MARKERS, - &interpreter, + &python_requirement, + Some(&MARKERS), &TAGS, client, &flat_index, diff --git a/crates/distribution-types/src/requirement.rs b/crates/distribution-types/src/requirement.rs index d240cd679..caab54720 100644 --- a/crates/distribution-types/src/requirement.rs +++ b/crates/distribution-types/src/requirement.rs @@ -33,24 +33,11 @@ pub struct Requirement { impl Requirement { /// Returns whether the markers apply for the given environment. - pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool { - if let Some(marker) = &self.marker { - marker.evaluate(env, extras) - } else { - true - } - } - - /// Returns whether the markers apply only for the given extras. /// /// When `env` is `None`, this specifically evaluates all marker /// expressions based on the environment to `true`. That is, this provides /// environment independent marker evaluation. - pub fn evaluate_optional_environment( - &self, - env: Option<&MarkerEnvironment>, - extras: &[ExtraName], - ) -> bool { + pub fn evaluate_markers(&self, env: Option<&MarkerEnvironment>, extras: &[ExtraName]) -> bool { if let Some(marker) = &self.marker { marker.evaluate_optional_environment(env, extras) } else { diff --git a/crates/distribution-types/src/specified_requirement.rs b/crates/distribution-types/src/specified_requirement.rs index d750f4381..993619fff 100644 --- a/crates/distribution-types/src/specified_requirement.rs +++ b/crates/distribution-types/src/specified_requirement.rs @@ -45,10 +45,15 @@ impl Display for UnresolvedRequirement { impl UnresolvedRequirement { /// Returns whether the markers apply for the given environment. - pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool { + /// + /// When the environment is not given, this treats all marker expressions + /// that reference the environment as true. In other words, it does + /// environment independent expression evaluation. (Which in turn devolves + /// to "only evaluate marker expressions that reference an extra name.") + pub fn evaluate_markers(&self, env: Option<&MarkerEnvironment>, extras: &[ExtraName]) -> bool { match self { Self::Named(requirement) => requirement.evaluate_markers(env, extras), - Self::Unnamed(requirement) => requirement.evaluate_markers(env, extras), + Self::Unnamed(requirement) => requirement.evaluate_optional_environment(env, extras), } } diff --git a/crates/pep508-rs/src/unnamed.rs b/crates/pep508-rs/src/unnamed.rs index 36dcd26c5..7972ef106 100644 --- a/crates/pep508-rs/src/unnamed.rs +++ b/crates/pep508-rs/src/unnamed.rs @@ -37,8 +37,17 @@ pub struct UnnamedRequirement { impl UnnamedRequirement { /// Returns whether the markers apply for the given environment pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool { + self.evaluate_optional_environment(Some(env), extras) + } + + /// Returns whether the markers apply for the given environment + pub fn evaluate_optional_environment( + &self, + env: Option<&MarkerEnvironment>, + extras: &[ExtraName], + ) -> bool { if let Some(marker) = &self.marker { - marker.evaluate(env, extras) + marker.evaluate_optional_environment(env, extras) } else { true } diff --git a/crates/uv-dev/src/resolve_cli.rs b/crates/uv-dev/src/resolve_cli.rs index 776d30c82..713701f0a 100644 --- a/crates/uv-dev/src/resolve_cli.rs +++ b/crates/uv-dev/src/resolve_cli.rs @@ -15,7 +15,9 @@ use uv_configuration::{ConfigSettings, NoBinary, NoBuild, SetupPyStrategy}; use uv_dispatch::BuildDispatch; use uv_installer::SitePackages; use uv_interpreter::PythonEnvironment; -use uv_resolver::{ExcludeNewer, FlatIndex, InMemoryIndex, Manifest, Options, Resolver}; +use uv_resolver::{ + ExcludeNewer, FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, Resolver, +}; use uv_types::{BuildIsolation, HashStrategy, InFlight}; #[derive(ValueEnum, Default, Clone)] @@ -98,6 +100,9 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { // Copied from `BuildDispatch` let tags = venv.interpreter().tags()?; + let markers = venv.interpreter().markers(); + let python_requirement = + PythonRequirement::from_marker_environment(venv.interpreter(), markers); let resolver = Resolver::new( Manifest::simple( args.requirements @@ -107,8 +112,8 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { .collect::>()?, ), Options::default(), - venv.interpreter().markers(), - venv.interpreter(), + &python_requirement, + Some(venv.interpreter().markers()), tags, &client, &flat_index, diff --git a/crates/uv-dispatch/src/lib.rs b/crates/uv-dispatch/src/lib.rs index 15e1952c6..02a774452 100644 --- a/crates/uv-dispatch/src/lib.rs +++ b/crates/uv-dispatch/src/lib.rs @@ -19,7 +19,7 @@ use uv_client::RegistryClient; use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, Reinstall, SetupPyStrategy}; use uv_installer::{Downloader, Installer, Plan, Planner, SitePackages}; use uv_interpreter::{Interpreter, PythonEnvironment}; -use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, Resolver}; +use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, Resolver}; use uv_types::{BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight}; /// The main implementation of [`BuildContext`], used by the CLI, see [`BuildContext`] @@ -135,12 +135,14 @@ impl<'a> BuildContext for BuildDispatch<'a> { async fn resolve<'data>(&'data self, requirements: &'data [Requirement]) -> Result { let markers = self.interpreter.markers(); + let python_requirement = + PythonRequirement::from_marker_environment(self.interpreter, markers); let tags = self.interpreter.tags()?; let resolver = Resolver::new( Manifest::simple(requirements.to_vec()), self.options, - markers, - self.interpreter, + &python_requirement, + Some(markers), tags, self.client, self.flat_index, diff --git a/crates/uv-installer/src/plan.rs b/crates/uv-installer/src/plan.rs index bdba679c4..80befcfd1 100644 --- a/crates/uv-installer/src/plan.rs +++ b/crates/uv-installer/src/plan.rs @@ -141,7 +141,7 @@ impl<'a> Planner<'a> { for requirement in self.requirements { // Filter out incompatible requirements. - if !requirement.evaluate_markers(venv.interpreter().markers(), &[]) { + if !requirement.evaluate_markers(Some(venv.interpreter().markers()), &[]) { continue; } diff --git a/crates/uv-installer/src/site_packages.rs b/crates/uv-installer/src/site_packages.rs index 63643acee..272793599 100644 --- a/crates/uv-installer/src/site_packages.rs +++ b/crates/uv-installer/src/site_packages.rs @@ -296,7 +296,7 @@ impl<'a> SitePackages<'a> { for entry in requirements { if entry .requirement - .evaluate_markers(self.venv.interpreter().markers(), &[]) + .evaluate_markers(Some(self.venv.interpreter().markers()), &[]) { if seen.insert(entry.clone()) { stack.push(entry.clone()); diff --git a/crates/uv-requirements/src/lookahead.rs b/crates/uv-requirements/src/lookahead.rs index bb96a790c..3cbc3ea18 100644 --- a/crates/uv-requirements/src/lookahead.rs +++ b/crates/uv-requirements/src/lookahead.rs @@ -96,9 +96,14 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { } /// Resolve the requirements from the provided source trees. + /// + /// When the environment is not given, this treats all marker expressions + /// that reference the environment as true. In other words, it does + /// environment independent expression evaluation. (Which in turn devolves + /// to "only evaluate marker expressions that reference an extra name.") pub async fn resolve( self, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, ) -> Result, LookaheadError> { let mut results = Vec::new(); let mut futures = FuturesUnordered::new(); diff --git a/crates/uv-resolver/src/candidate_selector.rs b/crates/uv-resolver/src/candidate_selector.rs index 3d6aac5f5..73975e1e6 100644 --- a/crates/uv-resolver/src/candidate_selector.rs +++ b/crates/uv-resolver/src/candidate_selector.rs @@ -29,7 +29,7 @@ impl CandidateSelector { pub(crate) fn for_resolution( options: Options, manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, ) -> Self { Self { resolution_strategy: ResolutionStrategy::from_mode( diff --git a/crates/uv-resolver/src/manifest.rs b/crates/uv-resolver/src/manifest.rs index 9b194609e..1da842890 100644 --- a/crates/uv-resolver/src/manifest.rs +++ b/crates/uv-resolver/src/manifest.rs @@ -97,42 +97,42 @@ impl Manifest { /// - Determining which requirements should allow local version specifiers (e.g., `torch==2.2.0+cpu`). pub fn requirements<'a>( &'a self, - markers: &'a MarkerEnvironment, + markers: Option<&'a MarkerEnvironment>, mode: DependencyMode, - ) -> impl Iterator { + ) -> impl Iterator + 'a { match mode { // Include all direct and transitive requirements, with constraints and overrides applied. DependencyMode::Transitive => Either::Left( self .lookaheads .iter() - .flat_map(|lookahead| { + .flat_map(move |lookahead| { self.overrides .apply(lookahead.requirements()) - .filter(|requirement| { + .filter(move |requirement| { requirement.evaluate_markers(markers, lookahead.extras()) }) }) - .chain(self.editables.iter().flat_map(|(editable, _metadata, requirements)| { + .chain(self.editables.iter().flat_map(move |(editable, _metadata, requirements)| { self.overrides .apply(&requirements.dependencies) - .filter(|requirement| { + .filter(move |requirement| { requirement.evaluate_markers(markers, &editable.extras) }) })) .chain( self.overrides .apply(&self.requirements) - .filter(|requirement| requirement.evaluate_markers(markers, &[])), + .filter(move |requirement| requirement.evaluate_markers(markers, &[])), ) .chain( self.constraints .requirements() - .filter(|requirement| requirement.evaluate_markers(markers, &[])), + .filter(move |requirement| requirement.evaluate_markers(markers, &[])), ) .chain( self.overrides .requirements() - .filter(|requirement| requirement.evaluate_markers(markers, &[])), + .filter(move |requirement| requirement.evaluate_markers(markers, &[])), )) , @@ -141,7 +141,7 @@ impl Manifest { self.overrides.apply(& self.requirements) .chain(self.constraints.requirements()) .chain(self.overrides.requirements()) - .filter(|requirement| requirement.evaluate_markers(markers, &[]))), + .filter(move |requirement| requirement.evaluate_markers(markers, &[]))), } } @@ -157,9 +157,9 @@ impl Manifest { /// the `lowest-direct` strategy is in use. pub fn user_requirements<'a>( &'a self, - markers: &'a MarkerEnvironment, + markers: Option<&'a MarkerEnvironment>, mode: DependencyMode, - ) -> impl Iterator { + ) -> impl Iterator + 'a { match mode { // Include direct requirements, dependencies of editables, and transitive dependencies // of local packages. @@ -167,17 +167,17 @@ impl Manifest { self.lookaheads .iter() .filter(|lookahead| lookahead.direct()) - .flat_map(|lookahead| { + .flat_map(move |lookahead| { self.overrides .apply(lookahead.requirements()) - .filter(|requirement| { + .filter(move |requirement| { requirement.evaluate_markers(markers, lookahead.extras()) }) }) .chain(self.editables.iter().flat_map( - |(editable, _metadata, uv_requirements)| { + move |(editable, _metadata, uv_requirements)| { self.overrides.apply(&uv_requirements.dependencies).filter( - |requirement| { + move |requirement| { requirement.evaluate_markers(markers, &editable.extras) }, ) @@ -186,7 +186,7 @@ impl Manifest { .chain( self.overrides .apply(&self.requirements) - .filter(|requirement| requirement.evaluate_markers(markers, &[])), + .filter(move |requirement| requirement.evaluate_markers(markers, &[])), ) .map(|requirement| &requirement.name), ), @@ -195,7 +195,7 @@ impl Manifest { DependencyMode::Direct => Either::Right( self.overrides .apply(self.requirements.iter()) - .filter(|requirement| requirement.evaluate_markers(markers, &[])) + .filter(move |requirement| requirement.evaluate_markers(markers, &[])) .map(|requirement| &requirement.name), ), } diff --git a/crates/uv-resolver/src/preferences.rs b/crates/uv-resolver/src/preferences.rs index 97664a5db..d86963126 100644 --- a/crates/uv-resolver/src/preferences.rs +++ b/crates/uv-resolver/src/preferences.rs @@ -79,7 +79,7 @@ impl Preferences { /// to an applicable subset. pub(crate) fn from_iter>( preferences: PreferenceIterator, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, ) -> Self { Self( // TODO(zanieb): We should explicitly ensure that when a package name is seen multiple times diff --git a/crates/uv-resolver/src/prerelease_mode.rs b/crates/uv-resolver/src/prerelease_mode.rs index 3ef5b9c59..20a406b28 100644 --- a/crates/uv-resolver/src/prerelease_mode.rs +++ b/crates/uv-resolver/src/prerelease_mode.rs @@ -56,7 +56,7 @@ impl PreReleaseStrategy { pub(crate) fn from_mode( mode: PreReleaseMode, manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, dependencies: DependencyMode, ) -> Self { match mode { diff --git a/crates/uv-resolver/src/pubgrub/dependencies.rs b/crates/uv-resolver/src/pubgrub/dependencies.rs index 4fb2b46ec..aeeca47e4 100644 --- a/crates/uv-resolver/src/pubgrub/dependencies.rs +++ b/crates/uv-resolver/src/pubgrub/dependencies.rs @@ -28,7 +28,7 @@ impl PubGrubDependencies { source_extra: Option<&ExtraName>, urls: &Urls, locals: &Locals, - env: &MarkerEnvironment, + env: Option<&MarkerEnvironment>, ) -> Result { let mut dependencies = Vec::default(); let mut seen = FxHashSet::default(); @@ -70,7 +70,7 @@ fn add_requirements( source_extra: Option<&ExtraName>, urls: &Urls, locals: &Locals, - env: &MarkerEnvironment, + env: Option<&MarkerEnvironment>, dependencies: &mut Vec<(PubGrubPackage, Range)>, seen: &mut FxHashSet, ) -> Result<(), ResolveError> { diff --git a/crates/uv-resolver/src/python_requirement.rs b/crates/uv-resolver/src/python_requirement.rs index 8c8aa0cfe..b5662cf8f 100644 --- a/crates/uv-resolver/src/python_requirement.rs +++ b/crates/uv-resolver/src/python_requirement.rs @@ -12,13 +12,17 @@ pub struct PythonRequirement { } impl PythonRequirement { - pub fn new(interpreter: &Interpreter, markers: &MarkerEnvironment) -> Self { + pub fn new(interpreter: &Interpreter, target: &StringVersion) -> Self { Self { installed: interpreter.python_full_version().clone(), - target: markers.python_full_version.clone(), + target: target.clone(), } } + pub fn from_marker_environment(interpreter: &Interpreter, env: &MarkerEnvironment) -> Self { + Self::new(interpreter, &env.python_full_version) + } + /// Return the installed version of Python. pub fn installed(&self) -> &StringVersion { &self.installed diff --git a/crates/uv-resolver/src/resolution_mode.rs b/crates/uv-resolver/src/resolution_mode.rs index 67bce2951..dc0b86268 100644 --- a/crates/uv-resolver/src/resolution_mode.rs +++ b/crates/uv-resolver/src/resolution_mode.rs @@ -37,7 +37,7 @@ impl ResolutionStrategy { pub(crate) fn from_mode( mode: ResolutionMode, manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, dependencies: DependencyMode, ) -> Self { match mode { diff --git a/crates/uv-resolver/src/resolver/locals.rs b/crates/uv-resolver/src/resolver/locals.rs index 37bba44fb..dded38a7d 100644 --- a/crates/uv-resolver/src/resolver/locals.rs +++ b/crates/uv-resolver/src/resolver/locals.rs @@ -21,7 +21,7 @@ impl Locals { /// Determine the set of permitted local versions in the [`Manifest`]. pub(crate) fn from_manifest( manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, dependencies: DependencyMode, ) -> Self { let mut required: FxHashMap = FxHashMap::default(); diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index e3c95ec76..aefcae9f2 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -32,7 +32,6 @@ pub(crate) use urls::Urls; use uv_client::RegistryClient; use uv_configuration::{Constraints, Overrides}; use uv_distribution::{ArchiveMetadata, DistributionDatabase}; -use uv_interpreter::Interpreter; use uv_normalize::PackageName; use uv_types::{BuildContext, HashStrategy, InstalledPackagesProvider}; @@ -188,8 +187,9 @@ pub struct Resolver<'a, Provider: ResolverProvider, InstalledPackages: Installed locals: Locals, dependency_mode: DependencyMode, hasher: &'a HashStrategy, - markers: &'a MarkerEnvironment, - python_requirement: PythonRequirement, + /// When not set, the resolver is in "universal" mode. + markers: Option<&'a MarkerEnvironment>, + python_requirement: &'a PythonRequirement, selector: CandidateSelector, index: &'a InMemoryIndex, installed_packages: &'a InstalledPackages, @@ -209,12 +209,27 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider> /// Initialize a new resolver using the default backend doing real requests. /// /// Reads the flat index entries. + /// + /// # Marker environment + /// + /// The marker environment is optional. + /// + /// When a marker environment is not provided, the resolver is said to be + /// in "universal" mode. When in universal mode, the resolution produced + /// may contain multiple versions of the same package. And thus, in order + /// to use the resulting resolution, there must be a "universal"-aware + /// reader of the resolution that knows to exclude distributions that can't + /// be used in the current environment. + /// + /// When a marker environment is provided, the reslver is in + /// "non-universal" mode, which corresponds to standard `pip` behavior that + /// works only for a specific marker environment. #[allow(clippy::too_many_arguments)] pub fn new( manifest: Manifest, options: Options, - markers: &'a MarkerEnvironment, - interpreter: &'a Interpreter, + python_requirement: &'a PythonRequirement, + markers: Option<&'a MarkerEnvironment>, tags: &'a Tags, client: &'a RegistryClient, flat_index: &'a FlatIndex, @@ -228,7 +243,7 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider> DistributionDatabase::new(client, build_context), flat_index, tags, - PythonRequirement::new(interpreter, markers), + python_requirement.clone(), AllowedYanks::from_manifest(&manifest, markers, options.dependency_mode), hasher, options.exclude_newer, @@ -240,7 +255,7 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider> options, hasher, markers, - PythonRequirement::new(interpreter, markers), + python_requirement, index, provider, installed_packages, @@ -257,8 +272,8 @@ impl<'a, Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvide manifest: Manifest, options: Options, hasher: &'a HashStrategy, - markers: &'a MarkerEnvironment, - python_requirement: PythonRequirement, + markers: Option<&'a MarkerEnvironment>, + python_requirement: &'a PythonRequirement, index: &'a InMemoryIndex, provider: Provider, installed_packages: &'a InstalledPackages, @@ -323,12 +338,12 @@ impl<'a, Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvide Err(if let ResolveError::NoSolution(err) = err { ResolveError::NoSolution( err.with_available_versions( - &self.python_requirement, + self.python_requirement, &self.visited, &self.index.packages, ) .with_selector(self.selector.clone()) - .with_python_requirement(&self.python_requirement) + .with_python_requirement(self.python_requirement) .with_index_locations(self.provider.index_locations()) .with_unavailable_packages(&self.unavailable_packages) .with_incomplete_packages(&self.incomplete_packages), diff --git a/crates/uv-resolver/src/resolver/urls.rs b/crates/uv-resolver/src/resolver/urls.rs index f76c64053..765c7fc02 100644 --- a/crates/uv-resolver/src/resolver/urls.rs +++ b/crates/uv-resolver/src/resolver/urls.rs @@ -15,7 +15,7 @@ pub(crate) struct Urls(FxHashMap); impl Urls { pub(crate) fn from_manifest( manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, dependencies: DependencyMode, ) -> Result { let mut urls: FxHashMap = FxHashMap::default(); diff --git a/crates/uv-resolver/src/yanks.rs b/crates/uv-resolver/src/yanks.rs index c06791130..75129260d 100644 --- a/crates/uv-resolver/src/yanks.rs +++ b/crates/uv-resolver/src/yanks.rs @@ -15,7 +15,7 @@ pub struct AllowedYanks(FxHashMap>); impl AllowedYanks { pub fn from_manifest( manifest: &Manifest, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, dependencies: DependencyMode, ) -> Self { let mut allowed_yanks = FxHashMap::>::default(); diff --git a/crates/uv-resolver/tests/resolver.rs b/crates/uv-resolver/tests/resolver.rs index 2b07bc9c1..33493f2c3 100644 --- a/crates/uv-resolver/tests/resolver.rs +++ b/crates/uv-resolver/tests/resolver.rs @@ -19,7 +19,8 @@ use uv_configuration::{BuildKind, Constraints, NoBinary, NoBuild, Overrides, Set use uv_interpreter::{find_default_python, Interpreter, PythonEnvironment}; use uv_resolver::{ DisplayResolutionGraph, ExcludeNewer, Exclusions, FlatIndex, InMemoryIndex, Manifest, Options, - OptionsBuilder, PreReleaseMode, Preference, ResolutionGraph, ResolutionMode, Resolver, + OptionsBuilder, PreReleaseMode, Preference, PythonRequirement, ResolutionGraph, ResolutionMode, + Resolver, }; use uv_types::{ BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, SourceBuildTrait, @@ -126,14 +127,15 @@ async fn resolve( let real_interpreter = find_default_python(&Cache::temp().unwrap()).expect("Expected a python to be installed"); let interpreter = Interpreter::artificial(real_interpreter.platform().clone(), markers.clone()); + let python_requirement = PythonRequirement::from_marker_environment(&interpreter, markers); let build_context = DummyContext::new(Cache::temp()?, interpreter.clone()); let hashes = HashStrategy::None; let installed_packages = EmptyInstalledPackages; let resolver = Resolver::new( manifest, options, - markers, - &interpreter, + &python_requirement, + Some(markers), tags, &client, &flat_index, diff --git a/crates/uv-types/src/hash.rs b/crates/uv-types/src/hash.rs index dc912c945..e00262636 100644 --- a/crates/uv-types/src/hash.rs +++ b/crates/uv-types/src/hash.rs @@ -85,9 +85,14 @@ impl HashStrategy { } /// Generate the required hashes from a set of [`UnresolvedRequirement`] entries. + /// + /// When the environment is not given, this treats all marker expressions + /// that reference the environment as true. In other words, it does + /// environment independent expression evaluation. (Which in turn devolves + /// to "only evaluate marker expressions that reference an extra name.") pub fn from_requirements<'a>( requirements: impl Iterator, - markers: &MarkerEnvironment, + markers: Option<&MarkerEnvironment>, ) -> Result { let mut hashes = FxHashMap::>::default(); diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 00bc2a49f..6bbe95ebc 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -238,6 +238,16 @@ pub(crate) async fn pip_compile( (None, Some(python_version)) => Cow::Owned(python_version.markers(interpreter.markers())), (None, None) => Cow::Borrowed(interpreter.markers()), }; + // The marker environment to use for evaluating requirements. When + // `uv_lock` is enabled, we specifically do environment independent marker + // evaluation. (i.e., Only consider extras.) + let marker_filter = if uv_lock { None } else { Some(&*markers) }; + // The Python requirement in "workspace-aware uv" should, I believe, come + // from the pyproject.toml. For now, we just take it from the markers + // (which does have its Python version set potentially from the CLI, which + // I think is spiritually equivalent to setting the Python version in + // pyproject.toml). + let python_requirement = PythonRequirement::from_marker_environment(&interpreter, &markers); // Generate, but don't enforce hashes for the requirements. let hasher = if generate_hashes { @@ -359,7 +369,7 @@ pub(crate) async fn pip_compile( for requirement in requirements .iter() - .filter(|requirement| requirement.evaluate_markers(&markers, &[])) + .filter(|requirement| requirement.evaluate_markers(marker_filter, &[])) { if let Some(path) = &requirement.path { if path.ends_with("pyproject.toml") { @@ -381,7 +391,7 @@ pub(crate) async fn pip_compile( for requirement in constraints .iter() - .filter(|requirement| requirement.evaluate_markers(&markers, &[])) + .filter(|requirement| requirement.evaluate_markers(marker_filter, &[])) { if let Some(path) = &requirement.path { sources.add( @@ -393,7 +403,7 @@ pub(crate) async fn pip_compile( for requirement in overrides .iter() - .filter(|requirement| requirement.evaluate_markers(&markers, &[])) + .filter(|requirement| requirement.evaluate_markers(marker_filter, &[])) { if let Some(path) = &requirement.path { sources.add(&requirement.name, SourceAnnotation::Override(path.clone())); @@ -469,15 +479,14 @@ pub(crate) async fn pip_compile( .collect::>()?; // Validate that the editables are compatible with the target Python version. - let requirement = PythonRequirement::new(&interpreter, &markers); for (_, metadata, _) in &editables { if let Some(python_requires) = metadata.requires_python.as_ref() { - if !python_requires.contains(requirement.target()) { + if !python_requires.contains(python_requirement.target()) { return Err(anyhow!( "Editable `{}` requires Python {}, but resolution targets Python {}", metadata.name, python_requires, - requirement.target() + python_requirement.target() )); } } @@ -511,7 +520,7 @@ pub(crate) async fn pip_compile( &top_level_index, ) .with_reporter(ResolverReporter::from(printer)) - .resolve(&markers) + .resolve(marker_filter) .await? } DependencyMode::Direct => Vec::new(), @@ -542,8 +551,8 @@ pub(crate) async fn pip_compile( let resolver = Resolver::new( manifest.clone(), options, - &markers, - &interpreter, + &python_requirement, + marker_filter, &tags, &client, &flat_index, diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 34fc7bf56..2d5d72acf 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -45,7 +45,8 @@ use uv_requirements::{ }; use uv_resolver::{ DependencyMode, ExcludeNewer, Exclusions, FlatIndex, InMemoryIndex, Lock, Manifest, Options, - OptionsBuilder, PreReleaseMode, Preference, ResolutionGraph, ResolutionMode, Resolver, + OptionsBuilder, PreReleaseMode, Preference, PythonRequirement, ResolutionGraph, ResolutionMode, + Resolver, }; use uv_types::{BuildIsolation, HashStrategy, InFlight}; use uv_warnings::warn_user; @@ -268,7 +269,7 @@ pub(crate) async fn pip_install( .iter() .chain(overrides.iter()) .map(|entry| (&entry.requirement, entry.hashes.as_slice())), - &markers, + Some(&markers), )? } else { HashStrategy::None @@ -688,6 +689,7 @@ async fn resolve( // Collect constraints and overrides. let constraints = Constraints::from_requirements(constraints); let overrides = Overrides::from_requirements(overrides); + let python_requirement = PythonRequirement::from_marker_environment(interpreter, markers); // Map the editables to their metadata. let editables: Vec<_> = editables @@ -726,7 +728,7 @@ async fn resolve( index, ) .with_reporter(ResolverReporter::from(printer)) - .resolve(markers) + .resolve(Some(markers)) .await? } DependencyMode::Direct => Vec::new(), @@ -748,8 +750,8 @@ async fn resolve( let resolver = Resolver::new( manifest, options, - markers, - interpreter, + &python_requirement, + Some(markers), tags, client, flat_index, diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index 017573f39..3c9b6da76 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -31,7 +31,9 @@ use uv_requirements::{ ExtrasSpecification, NamedRequirementsResolver, RequirementsSource, RequirementsSpecification, SourceTreeResolver, }; -use uv_resolver::{DependencyMode, FlatIndex, InMemoryIndex, Manifest, OptionsBuilder, Resolver}; +use uv_resolver::{ + DependencyMode, FlatIndex, InMemoryIndex, Manifest, OptionsBuilder, PythonRequirement, Resolver, +}; use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight}; use uv_warnings::warn_user; @@ -191,7 +193,7 @@ pub(crate) async fn pip_sync( requirements .iter() .map(|entry| (&entry.requirement, entry.hashes.as_slice())), - &markers, + Some(&markers), )? } else { HashStrategy::None @@ -354,6 +356,7 @@ pub(crate) async fn pip_sync( let interpreter = venv.interpreter(); let tags = interpreter.tags()?; let markers = interpreter.markers(); + let python_requirement = PythonRequirement::from_marker_environment(interpreter, markers); // Resolve with `--no-deps`. let options = OptionsBuilder::new() @@ -367,8 +370,8 @@ pub(crate) async fn pip_sync( let resolver = Resolver::new( Manifest::simple(remote), options, - markers, - interpreter, + &python_requirement, + Some(markers), tags, &client, &flat_index, diff --git a/crates/uv/src/commands/workspace/mod.rs b/crates/uv/src/commands/workspace/mod.rs index b03dac217..90120d390 100644 --- a/crates/uv/src/commands/workspace/mod.rs +++ b/crates/uv/src/commands/workspace/mod.rs @@ -22,7 +22,8 @@ use uv_requirements::{ RequirementsSpecification, SourceTreeResolver, }; use uv_resolver::{ - Exclusions, FlatIndex, InMemoryIndex, Manifest, Options, ResolutionGraph, Resolver, + Exclusions, FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, ResolutionGraph, + Resolver, }; use uv_types::{EmptyInstalledPackages, HashStrategy, InFlight}; @@ -101,6 +102,7 @@ pub(crate) async fn resolve( let preferences = Vec::new(); let constraints = Constraints::default(); let overrides = Overrides::default(); + let python_requirement = PythonRequirement::from_marker_environment(interpreter, markers); let editables = Vec::new(); let installed_packages = EmptyInstalledPackages; @@ -150,7 +152,7 @@ pub(crate) async fn resolve( index, ) .with_reporter(ResolverReporter::from(printer)) - .resolve(markers) + .resolve(Some(markers)) .await?; // Create a manifest of the requirements. @@ -169,8 +171,8 @@ pub(crate) async fn resolve( let resolver = Resolver::new( manifest, options, - markers, - interpreter, + &python_requirement, + Some(markers), tags, client, flat_index,