From b6f470416ebe93aad25a2ca02a2b501344239923 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 22 Jul 2024 10:33:53 -0400 Subject: [PATCH] Match wheel tags against `Requires-Python` major-minor (#5289) ## Summary Given `Requires-Python: ">=3.12.3"`, we were rejecting wheels like `dearpygui-1.11.1-cp312-cp312-win_amd64.whl`, since `3.12.0` is not included in `>=3.12.3`. We instead need to test against the major-minor version of `Requires-Python`. The easiest way to do this, I think, is the use the `RequiresPython` struct, which has a single bound that we can truncate the major-minor. This also means that we now allow `dearpygui-1.11.1-cp312-cp312-win_amd64.whl` for specifiers like `Requires-Python: "==3.10.*"`. This is incorrect on the surface, but it does match our semantics for `Requires-Python` elsewhere: we treat it as a lower bound. Closes https://github.com/astral-sh/uv/issues/5287. --- crates/distribution-filename/src/wheel.rs | 108 +------------- crates/uv-resolver/src/lock.rs | 7 +- crates/uv-resolver/src/requires_python.rs | 164 ++++++++++++++++++++++ crates/uv-resolver/src/version_map.rs | 2 +- crates/uv/tests/lock.rs | 15 ++ 5 files changed, 183 insertions(+), 113 deletions(-) diff --git a/crates/distribution-filename/src/wheel.rs b/crates/distribution-filename/src/wheel.rs index 56efcfb44..5c6a5afef 100644 --- a/crates/distribution-filename/src/wheel.rs +++ b/crates/distribution-filename/src/wheel.rs @@ -5,7 +5,7 @@ use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; use url::Url; -use pep440_rs::{Version, VersionParseError, VersionSpecifiers}; +use pep440_rs::{Version, VersionParseError}; use platform_tags::{TagCompatibility, Tags}; use uv_normalize::{InvalidNameError, PackageName}; @@ -60,71 +60,6 @@ impl WheelFilename { compatible_tags.compatibility(&self.python_tag, &self.abi_tag, &self.platform_tag) } - /// Returns `false` if the wheel's tags state it can't be used in the given Python version - /// range. - /// - /// It is meant to filter out clearly unusable wheels with perfect specificity and acceptable - /// sensitivity, we return `true` if the tags are unknown. - pub fn matches_requires_python(&self, specifiers: &VersionSpecifiers) -> bool { - self.abi_tag.iter().any(|abi_tag| { - if abi_tag == "abi3" { - // Universal tags are allowed. - true - } else if abi_tag == "none" { - self.python_tag.iter().any(|python_tag| { - // Remove `py2-none-any` and `py27-none-any`. - if python_tag.starts_with("py2") { - return false; - } - - // Remove (e.g.) `cp36-none-any` if the specifier is `==3.10.*`. - let Some(minor) = python_tag - .strip_prefix("cp3") - .or_else(|| python_tag.strip_prefix("pp3")) - .or_else(|| python_tag.strip_prefix("py3")) - else { - return true; - }; - let Ok(minor) = minor.parse::() else { - return true; - }; - let version = Version::new([3, minor]); - specifiers.contains(&version) - }) - } else if abi_tag.starts_with("cp2") || abi_tag.starts_with("pypy2") { - // Python 2 is never allowed. - false - } else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("cp3") { - // Remove ABI tags, both old (dmu) and future (t, and all other letters). - let minor_not_dot = minor_no_dot_abi.trim_matches(char::is_alphabetic); - let Ok(minor) = minor_not_dot.parse::() else { - // Unknown version pattern are allowed. - return true; - }; - - let version = Version::new([3, minor]); - specifiers.contains(&version) - } else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("pypy3") { - // Given `pypy39_pp73`, we just removed `pypy3`, now we remove `_pp73` ... - let Some((minor_not_dot, _)) = minor_no_dot_abi.split_once('_') else { - // Unknown version pattern are allowed. - return true; - }; - // ... and get `9`. - let Ok(minor) = minor_not_dot.parse::() else { - // Unknown version pattern are allowed. - return true; - }; - - let version = Version::new([3, minor]); - specifiers.contains(&version) - } else { - // Unknown python tag -> allowed. - true - } - }) - } - /// The wheel filename without the extension. pub fn stem(&self) -> String { format!( @@ -397,45 +332,4 @@ mod tests { ); } } - - #[test] - fn test_requires_python_included() { - let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap(); - let wheel_names = &[ - "bcrypt-4.1.3-cp37-abi3-macosx_10_12_universal2.whl", - "black-24.4.2-cp310-cp310-win_amd64.whl", - "black-24.4.2-cp310-none-win_amd64.whl", - "cbor2-5.6.4-py3-none-any.whl", - "watchfiles-0.22.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", - ]; - for wheel_name in wheel_names { - assert!( - WheelFilename::from_str(wheel_name) - .unwrap() - .matches_requires_python(&version_specifiers), - "{wheel_name}" - ); - } - } - - #[test] - fn test_requires_python_dropped() { - let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap(); - let wheel_names = &[ - "PySocks-1.7.1-py27-none-any.whl", - "black-24.4.2-cp39-cp39-win_amd64.whl", - "psutil-6.0.0-cp36-cp36m-win32.whl", - "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", - "torch-1.10.0-cp36-none-macosx_10_9_x86_64.whl", - "torch-1.10.0-py36-none-macosx_10_9_x86_64.whl", - ]; - for wheel_name in wheel_names { - assert!( - !WheelFilename::from_str(wheel_name) - .unwrap() - .matches_requires_python(&version_specifiers), - "{wheel_name}" - ); - } - } } diff --git a/crates/uv-resolver/src/lock.rs b/crates/uv-resolver/src/lock.rs index 890937567..bd2f5ac9c 100644 --- a/crates/uv-resolver/src/lock.rs +++ b/crates/uv-resolver/src/lock.rs @@ -222,11 +222,8 @@ impl Lock { // Remove wheels that don't match `requires-python` and can't be selected for // installation. if let Some(requires_python) = &requires_python { - dist.wheels.retain(|wheel| { - wheel - .filename - .matches_requires_python(requires_python.specifiers()) - }); + dist.wheels + .retain(|wheel| requires_python.matches_wheel_tag(&wheel.filename)); } } distributions.sort_by(|dist1, dist2| dist1.id.cmp(&dist2.id)); diff --git a/crates/uv-resolver/src/requires_python.rs b/crates/uv-resolver/src/requires_python.rs index 983026f7d..34edaf596 100644 --- a/crates/uv-resolver/src/requires_python.rs +++ b/crates/uv-resolver/src/requires_python.rs @@ -5,6 +5,7 @@ use std::ops::Deref; use itertools::Itertools; use pubgrub::range::Range; +use distribution_filename::WheelFilename; use pep440_rs::{Operator, Version, VersionSpecifier, VersionSpecifiers}; use pep508_rs::{MarkerExpression, MarkerTree, MarkerValueVersion}; @@ -160,6 +161,23 @@ impl RequiresPython { &self.bound } + /// Returns the [`RequiresPythonBound`] truncated to the major and minor version. + pub fn bound_major_minor(&self) -> RequiresPythonBound { + match self.bound.as_ref() { + // Ex) `>=3.10.1` -> `>=3.10` + Bound::Included(version) => RequiresPythonBound(Bound::Included(Version::new( + version.release().iter().take(2), + ))), + // Ex) `>3.10.1` -> `>=3.10` + // This is unintuitive, but `>3.10.1` does indicate that _some_ version of Python 3.10 + // is supported. + Bound::Excluded(version) => RequiresPythonBound(Bound::Included(Version::new( + version.release().iter().take(2), + ))), + Bound::Unbounded => RequiresPythonBound(Bound::Unbounded), + } + } + /// Returns this `Requires-Python` specifier as an equivalent marker /// expression utilizing the `python_version` marker field. /// @@ -217,6 +235,74 @@ impl RequiresPython { MarkerTree::Expression(expr_python_full_version), ]) } + + /// Returns `false` if the wheel's tags state it can't be used in the given Python version + /// range. + /// + /// It is meant to filter out clearly unusable wheels with perfect specificity and acceptable + /// sensitivity, we return `true` if the tags are unknown. + pub fn matches_wheel_tag(&self, wheel: &WheelFilename) -> bool { + wheel.abi_tag.iter().any(|abi_tag| { + if abi_tag == "abi3" { + // Universal tags are allowed. + true + } else if abi_tag == "none" { + wheel.python_tag.iter().any(|python_tag| { + // Remove `py2-none-any` and `py27-none-any`. + if python_tag.starts_with("py2") { + return false; + } + + // Remove (e.g.) `cp36-none-any` if the specifier is `==3.10.*`. + let Some(minor) = python_tag + .strip_prefix("cp3") + .or_else(|| python_tag.strip_prefix("pp3")) + .or_else(|| python_tag.strip_prefix("py3")) + else { + return true; + }; + let Ok(minor) = minor.parse::() else { + return true; + }; + + // Ex) If the wheel bound is `3.6`, then it doesn't match `>=3.10`. + let wheel_bound = + RequiresPythonBound(Bound::Included(Version::new([3, minor]))); + wheel_bound >= self.bound_major_minor() + }) + } else if abi_tag.starts_with("cp2") || abi_tag.starts_with("pypy2") { + // Python 2 is never allowed. + false + } else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("cp3") { + // Remove ABI tags, both old (dmu) and future (t, and all other letters). + let minor_not_dot = minor_no_dot_abi.trim_matches(char::is_alphabetic); + let Ok(minor) = minor_not_dot.parse::() else { + // Unknown version pattern are allowed. + return true; + }; + + let wheel_bound = RequiresPythonBound(Bound::Included(Version::new([3, minor]))); + wheel_bound >= self.bound_major_minor() + } else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("pypy3") { + // Given `pypy39_pp73`, we just removed `pypy3`, now we remove `_pp73` ... + let Some((minor_not_dot, _)) = minor_no_dot_abi.split_once('_') else { + // Unknown version pattern are allowed. + return true; + }; + // ... and get `9`. + let Ok(minor) = minor_not_dot.parse::() else { + // Unknown version pattern are allowed. + return true; + }; + + let wheel_bound = RequiresPythonBound(Bound::Included(Version::new([3, minor]))); + wheel_bound >= self.bound_major_minor() + } else { + // Unknown python tag -> allowed. + true + } + }) + } } impl std::fmt::Display for RequiresPython { @@ -295,3 +381,81 @@ impl Ord for RequiresPythonBound { } } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use distribution_filename::WheelFilename; + use pep440_rs::VersionSpecifiers; + + use crate::RequiresPython; + + #[test] + fn requires_python_included() { + let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap(); + let requires_python = RequiresPython::union(std::iter::once(&version_specifiers)) + .unwrap() + .unwrap(); + let wheel_names = &[ + "bcrypt-4.1.3-cp37-abi3-macosx_10_12_universal2.whl", + "black-24.4.2-cp310-cp310-win_amd64.whl", + "black-24.4.2-cp310-none-win_amd64.whl", + "cbor2-5.6.4-py3-none-any.whl", + "watchfiles-0.22.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", + "dearpygui-1.11.1-cp312-cp312-win_amd64.whl", + ]; + for wheel_name in wheel_names { + assert!( + requires_python.matches_wheel_tag(&WheelFilename::from_str(wheel_name).unwrap()), + "{wheel_name}" + ); + } + + let version_specifiers = VersionSpecifiers::from_str(">=3.12.3").unwrap(); + let requires_python = RequiresPython::union(std::iter::once(&version_specifiers)) + .unwrap() + .unwrap(); + let wheel_names = &["dearpygui-1.11.1-cp312-cp312-win_amd64.whl"]; + for wheel_name in wheel_names { + assert!( + requires_python.matches_wheel_tag(&WheelFilename::from_str(wheel_name).unwrap()), + "{wheel_name}" + ); + } + } + + #[test] + fn requires_python_dropped() { + let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap(); + let requires_python = RequiresPython::union(std::iter::once(&version_specifiers)) + .unwrap() + .unwrap(); + let wheel_names = &[ + "PySocks-1.7.1-py27-none-any.whl", + "black-24.4.2-cp39-cp39-win_amd64.whl", + "psutil-6.0.0-cp36-cp36m-win32.whl", + "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", + "torch-1.10.0-cp36-none-macosx_10_9_x86_64.whl", + "torch-1.10.0-py36-none-macosx_10_9_x86_64.whl", + ]; + for wheel_name in wheel_names { + assert!( + !requires_python.matches_wheel_tag(&WheelFilename::from_str(wheel_name).unwrap()), + "{wheel_name}" + ); + } + + let version_specifiers = VersionSpecifiers::from_str(">=3.12.3").unwrap(); + let requires_python = RequiresPython::union(std::iter::once(&version_specifiers)) + .unwrap() + .unwrap(); + let wheel_names = &["dearpygui-1.11.1-cp310-cp310-win_amd64.whl"]; + for wheel_name in wheel_names { + assert!( + !requires_python.matches_wheel_tag(&WheelFilename::from_str(wheel_name).unwrap()), + "{wheel_name}" + ); + } + } +} diff --git a/crates/uv-resolver/src/version_map.rs b/crates/uv-resolver/src/version_map.rs index 5ed6f0b0b..9355adb0f 100644 --- a/crates/uv-resolver/src/version_map.rs +++ b/crates/uv-resolver/src/version_map.rs @@ -511,7 +511,7 @@ impl VersionMapLazy { // Check if the wheel is compatible with the `requires-python` (i.e., the Python ABI tag // is not less than the `requires-python` minimum version). if let Some(requires_python) = self.requires_python.as_ref() { - if !filename.matches_requires_python(requires_python.specifiers()) { + if !requires_python.matches_wheel_tag(filename) { return WheelCompatibility::Incompatible(IncompatibleWheel::RequiresPython( requires_python.specifiers().clone(), PythonRequirementKind::Target, diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index 19b783221..d20ff215f 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -2840,6 +2840,21 @@ fn lock_requires_python_wheels() -> Result<()> { { url = "https://files.pythonhosted.org/packages/db/1b/6a5b970e55dffc1a7d0bb54f57b184b2a2a2ad0b7bca16a97ca26d73c5b5/frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2", size = 272292 }, { url = "https://files.pythonhosted.org/packages/1a/05/ebad68130e6b6eb9b287dacad08ea357c33849c74550c015b355b75cc714/frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17", size = 44446 }, { url = "https://files.pythonhosted.org/packages/b3/21/c5aaffac47fd305d69df46cfbf118768cdf049a92ee6b0b5cb029d449dcf/frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825", size = 50459 }, + { url = "https://files.pythonhosted.org/packages/b4/db/4cf37556a735bcdb2582f2c3fa286aefde2322f92d3141e087b8aeb27177/frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae", size = 93937 }, + { url = "https://files.pythonhosted.org/packages/46/03/69eb64642ca8c05f30aa5931d6c55e50b43d0cd13256fdd01510a1f85221/frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb", size = 53656 }, + { url = "https://files.pythonhosted.org/packages/3f/ab/c543c13824a615955f57e082c8a5ee122d2d5368e80084f2834e6f4feced/frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b", size = 51868 }, + { url = "https://files.pythonhosted.org/packages/a9/b8/438cfd92be2a124da8259b13409224d9b19ef8f5a5b2507174fc7e7ea18f/frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86", size = 280652 }, + { url = "https://files.pythonhosted.org/packages/54/72/716a955521b97a25d48315c6c3653f981041ce7a17ff79f701298195bca3/frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480", size = 286739 }, + { url = "https://files.pythonhosted.org/packages/65/d8/934c08103637567084568e4d5b4219c1016c60b4d29353b1a5b3587827d6/frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09", size = 289447 }, + { url = "https://files.pythonhosted.org/packages/70/bb/d3b98d83ec6ef88f9bd63d77104a305d68a146fd63a683569ea44c3085f6/frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a", size = 265466 }, + { url = "https://files.pythonhosted.org/packages/0b/f2/b8158a0f06faefec33f4dff6345a575c18095a44e52d4f10c678c137d0e0/frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd", size = 281530 }, + { url = "https://files.pythonhosted.org/packages/ea/a2/20882c251e61be653764038ece62029bfb34bd5b842724fff32a5b7a2894/frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6", size = 281295 }, + { url = "https://files.pythonhosted.org/packages/4c/f9/8894c05dc927af2a09663bdf31914d4fb5501653f240a5bbaf1e88cab1d3/frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1", size = 268054 }, + { url = "https://files.pythonhosted.org/packages/37/ff/a613e58452b60166507d731812f3be253eb1229808e59980f0405d1eafbf/frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b", size = 286904 }, + { url = "https://files.pythonhosted.org/packages/cc/6e/0091d785187f4c2020d5245796d04213f2261ad097e0c1cf35c44317d517/frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e", size = 290754 }, + { url = "https://files.pythonhosted.org/packages/a5/c2/e42ad54bae8bcffee22d1e12a8ee6c7717f7d5b5019261a8c861854f4776/frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8", size = 282602 }, + { url = "https://files.pythonhosted.org/packages/b6/61/56bad8cb94f0357c4bc134acc30822e90e203b5cb8ff82179947de90c17f/frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89", size = 44063 }, + { url = "https://files.pythonhosted.org/packages/3e/dc/96647994a013bc72f3d453abab18340b7f5e222b7b7291e3697ca1fcfbd5/frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5", size = 50452 }, { url = "https://files.pythonhosted.org/packages/83/10/466fe96dae1bff622021ee687f68e5524d6392b0a2f80d05001cd3a451ba/frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7", size = 11552 }, ]