Unify editable handling between sync and install (#3568)

## Summary

Uses the editable handling from `pip sync`, and improves the
abstractions such that we can pass those resolved editables into the
resolver.

---------

Co-authored-by: konstin <konstin@mailbox.org>
This commit is contained in:
Charlie Marsh
2024-05-14 05:18:29 -04:00
committed by GitHub
parent c666e43b90
commit 7c1c5df968
8 changed files with 289 additions and 293 deletions
+45 -1
View File
@@ -8,6 +8,14 @@ use requirements_txt::EditableRequirement;
use uv_normalize::PackageName;
/// An editable distribution that has been installed.
#[derive(Debug, Clone)]
pub struct InstalledEditable {
pub editable: LocalEditable,
pub wheel: InstalledDist,
pub metadata: Metadata23,
}
/// An editable distribution that has been built.
#[derive(Debug, Clone)]
pub struct BuiltEditable {
@@ -21,11 +29,35 @@ pub struct BuiltEditable {
#[allow(clippy::large_enum_variant)]
pub enum ResolvedEditable {
/// The editable is already installed in the environment.
Installed(InstalledDist),
Installed(InstalledEditable),
/// The editable has been built and is ready to be installed.
Built(BuiltEditable),
}
impl ResolvedEditable {
/// Return the [`LocalEditable`] for the distribution.
pub fn local(&self) -> &LocalEditable {
match self {
Self::Installed(dist) => &dist.editable,
Self::Built(dist) => &dist.editable,
}
}
/// Return the [`Metadata23`] for the distribution.
pub fn metadata(&self) -> &Metadata23 {
match self {
Self::Installed(dist) => &dist.metadata,
Self::Built(dist) => &dist.metadata,
}
}
}
impl Name for InstalledEditable {
fn name(&self) -> &PackageName {
&self.metadata.name
}
}
impl Name for BuiltEditable {
fn name(&self) -> &PackageName {
&self.metadata.name
@@ -41,6 +73,12 @@ impl Name for ResolvedEditable {
}
}
impl InstalledMetadata for InstalledEditable {
fn installed_version(&self) -> InstalledVersion {
self.wheel.installed_version()
}
}
impl InstalledMetadata for BuiltEditable {
fn installed_version(&self) -> InstalledVersion {
self.wheel.installed_version()
@@ -56,6 +94,12 @@ impl InstalledMetadata for ResolvedEditable {
}
}
impl std::fmt::Display for InstalledEditable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}{}", self.name(), self.installed_version())
}
}
impl std::fmt::Display for BuiltEditable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}{}", self.name(), self.installed_version())
+1 -1
View File
@@ -1,6 +1,6 @@
pub use compile::{compile_tree, CompileError};
pub use downloader::{Downloader, Reporter as DownloadReporter};
pub use editable::{is_dynamic, BuiltEditable, ResolvedEditable};
pub use editable::{is_dynamic, BuiltEditable, InstalledEditable, ResolvedEditable};
pub use installer::{Installer, Reporter as InstallReporter};
pub use plan::{Plan, Planner};
pub use site_packages::{Diagnostic, SatisfiesResult, SitePackages};
+2 -2
View File
@@ -114,7 +114,7 @@ impl<'a> Planner<'a> {
debug!("Treating editable requirement as immutable: {installed}");
// Remove from the site-packages index, to avoid marking as extraneous.
let Some(editable) = installed.as_editable() else {
let Some(editable) = installed.wheel.as_editable() else {
warn!("Editable requirement is not editable: {installed}");
continue;
};
@@ -127,7 +127,7 @@ impl<'a> Planner<'a> {
ResolvedEditable::Built(built) => {
debug!("Treating editable requirement as mutable: {built}");
// Remove any editable installs.
// Remove any editables.
let existing = site_packages.remove_editables(built.editable.raw());
reinstalls.extend(existing);
+201
View File
@@ -0,0 +1,201 @@
use std::fmt::Write;
use std::ops::Deref;
use anyhow::{anyhow, Context, Result};
use owo_colors::OwoColorize;
use distribution_types::{InstalledDist, LocalEditable, LocalEditables, Name};
use platform_tags::Tags;
use requirements_txt::EditableRequirement;
use uv_cache::{ArchiveTarget, ArchiveTimestamp, Cache};
use uv_client::RegistryClient;
use uv_configuration::{Concurrency, Reinstall};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
use uv_installer::{is_dynamic, Downloader, InstalledEditable, ResolvedEditable, SitePackages};
use uv_interpreter::Interpreter;
use uv_types::HashStrategy;
use crate::commands::elapsed;
use crate::commands::reporters::DownloadReporter;
use crate::printer::Printer;
#[derive(Debug)]
pub(crate) struct ResolvedEditables {
/// The set of resolved editables, including both those that were already installed and those
/// that were built.
pub(crate) editables: Vec<ResolvedEditable>,
/// The temporary directory in which the built editables were stored.
#[allow(dead_code)]
temp_dir: Option<tempfile::TempDir>,
}
impl Deref for ResolvedEditables {
type Target = [ResolvedEditable];
fn deref(&self) -> &Self::Target {
&self.editables
}
}
impl ResolvedEditables {
/// Resolve the set of editables that need to be installed.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn resolve(
editables: Vec<EditableRequirement>,
site_packages: &SitePackages<'_>,
reinstall: &Reinstall,
hasher: &HashStrategy,
interpreter: &Interpreter,
tags: &Tags,
cache: &Cache,
client: &RegistryClient,
build_dispatch: &BuildDispatch<'_>,
concurrency: Concurrency,
printer: Printer,
) -> Result<Self> {
// Partition the editables into those that are already installed, and those that must be built.
let mut installed = Vec::with_capacity(editables.len());
let mut builds = Vec::with_capacity(editables.len());
for editable in editables {
match reinstall {
Reinstall::None => {
if let [dist] = site_packages.get_editables(editable.raw()).as_slice() {
if let Some(editable) = up_to_date(&editable, dist)? {
installed.push(editable);
} else {
builds.push(editable);
}
} else {
builds.push(editable);
}
}
Reinstall::All => {
builds.push(editable);
}
Reinstall::Packages(packages) => {
if let [dist] = site_packages.get_editables(editable.raw()).as_slice() {
if packages.contains(dist.name()) {
builds.push(editable);
} else if let Some(editable) = up_to_date(&editable, dist)? {
installed.push(editable);
} else {
builds.push(editable);
}
} else {
builds.push(editable);
}
}
}
}
// Build any editables.
let (built_editables, temp_dir) = if builds.is_empty() {
(Vec::new(), None)
} else {
let start = std::time::Instant::now();
let downloader = Downloader::new(
cache,
tags,
hasher,
DistributionDatabase::new(client, build_dispatch, concurrency.downloads),
)
.with_reporter(DownloadReporter::from(printer).with_length(builds.len() as u64));
let editables = LocalEditables::from_editables(builds.iter().map(|editable| {
let EditableRequirement {
url,
path,
extras,
origin: _,
} = editable;
LocalEditable {
url: url.clone(),
path: path.clone(),
extras: extras.clone(),
}
}));
let temp_dir = tempfile::tempdir_in(cache.root())?;
let editables: Vec<_> = downloader
.build_editables(editables, temp_dir.path())
.await
.context("Failed to build editables")?
.into_iter()
.collect();
// Validate that the editables are compatible with the target Python version.
for editable in &editables {
if let Some(python_requires) = editable.metadata.requires_python.as_ref() {
if !python_requires.contains(interpreter.python_version()) {
return Err(anyhow!(
"Editable `{}` requires Python {}, but {} is installed",
editable.metadata.name,
python_requires,
interpreter.python_version()
));
}
}
}
let s = if editables.len() == 1 { "" } else { "s" };
writeln!(
printer.stderr(),
"{}",
format!(
"Built {} in {}",
format!("{} editable{}", editables.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
(editables, Some(temp_dir))
};
let editables = installed
.into_iter()
.map(ResolvedEditable::Installed)
.chain(built_editables.into_iter().map(ResolvedEditable::Built))
.collect::<Vec<_>>();
Ok(Self {
editables,
temp_dir,
})
}
}
/// Returns the [`InstalledEditable`] if the installed distribution is up-to-date for the given
/// requirement.
fn up_to_date(
editable: &EditableRequirement,
dist: &InstalledDist,
) -> Result<Option<InstalledEditable>> {
// If the editable isn't up-to-date, don't reuse it.
if !ArchiveTimestamp::up_to_date_with(&editable.path, ArchiveTarget::Install(dist))? {
return Ok(None);
};
// If the editable is dynamic, don't reuse it.
if is_dynamic(editable) {
return Ok(None);
};
// If we can't read the metadata from the installed distribution, don't reuse it.
let Ok(metadata) = dist.metadata() else {
return Ok(None);
};
Ok(Some(InstalledEditable {
editable: LocalEditable {
url: editable.url.clone(),
path: editable.path.clone(),
extras: editable.extras.clone(),
},
wheel: (*dist).clone(),
metadata,
}))
}
+29 -119
View File
@@ -1,6 +1,5 @@
use std::borrow::Cow;
use std::fmt::Write;
use std::path::Path;
use anstream::eprint;
use anyhow::{anyhow, Context, Result};
@@ -8,12 +7,11 @@ use fs_err as fs;
use indexmap::IndexMap;
use itertools::Itertools;
use owo_colors::OwoColorize;
use tempfile::tempdir_in;
use tracing::{debug, enabled, Level};
use distribution_types::{
DistributionMetadata, IndexLocations, InstalledMetadata, InstalledVersion, LocalDist,
LocalEditable, LocalEditables, Name, ParsedUrl, ParsedUrlError, RequirementSource, Resolution,
DistributionMetadata, IndexLocations, InstalledMetadata, InstalledVersion, LocalDist, Name,
ParsedUrl, ParsedUrlError, RequirementSource, Resolution,
};
use distribution_types::{Requirement, Requirements};
use install_wheel_rs::linker::LinkMode;
@@ -21,7 +19,6 @@ use pep440_rs::{VersionSpecifier, VersionSpecifiers};
use pep508_rs::{MarkerEnvironment, VerbatimUrl};
use platform_tags::Tags;
use pypi_types::Yanked;
use requirements_txt::EditableRequirement;
use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{
@@ -35,9 +32,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
use uv_fs::Simplified;
use uv_installer::{
BuiltEditable, Downloader, Plan, Planner, ResolvedEditable, SatisfiesResult, SitePackages,
};
use uv_installer::{Downloader, Plan, Planner, ResolvedEditable, SatisfiesResult, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment, PythonVersion, Target};
use uv_normalize::PackageName;
use uv_requirements::{
@@ -52,6 +47,7 @@ use uv_resolver::{
use uv_types::{BuildIsolation, HashStrategy, InFlight};
use uv_warnings::warn_user;
use crate::commands::pip::editables::ResolvedEditables;
use crate::commands::reporters::{DownloadReporter, InstallReporter, ResolverReporter};
use crate::commands::DryRunEvent;
use crate::commands::{compile_bytecode, elapsed, ChangeEvent, ChangeEventKind, ExitStatus};
@@ -340,27 +336,22 @@ pub(crate) async fn pip_install(
.with_options(OptionsBuilder::new().exclude_newer(exclude_newer).build());
// Build all editable distributions. The editables are shared between resolution and
// installation, and should live for the duration of the command. If an editable is already
// installed in the environment, we'll still re-build it here.
let editable_wheel_dir;
let editables = if editables.is_empty() {
vec![]
} else {
editable_wheel_dir = tempdir_in(cache.root())?;
build_editables(
&editables,
editable_wheel_dir.path(),
&hasher,
&cache,
&interpreter,
&tags,
concurrency,
&client,
&resolve_dispatch,
printer,
)
.await?
};
// installation, and should live for the duration of the command.
// Resolve any editables.
let editables = ResolvedEditables::resolve(
editables,
&site_packages,
&reinstall,
&hasher,
venv.interpreter(),
&tags,
&cache,
&client,
&resolve_dispatch,
concurrency,
printer,
)
.await?;
// Resolve the requirements.
let resolution = if let Some(ref root) = uv_lock {
@@ -488,7 +479,7 @@ pub(crate) async fn pip_install(
// Sync the environment.
install(
&resolution,
editables,
&editables,
site_packages,
&reinstall,
&no_binary,
@@ -569,81 +560,6 @@ async fn read_requirements(
Ok(spec)
}
/// Build a set of editable distributions.
#[allow(clippy::too_many_arguments)]
async fn build_editables(
editables: &[EditableRequirement],
editable_wheel_dir: &Path,
hasher: &HashStrategy,
cache: &Cache,
interpreter: &Interpreter,
tags: &Tags,
concurrency: Concurrency,
client: &RegistryClient,
build_dispatch: &BuildDispatch<'_>,
printer: Printer,
) -> Result<Vec<BuiltEditable>, Error> {
let start = std::time::Instant::now();
let downloader = Downloader::new(
cache,
tags,
hasher,
DistributionDatabase::new(client, build_dispatch, concurrency.downloads),
)
.with_reporter(DownloadReporter::from(printer).with_length(editables.len() as u64));
let editables = LocalEditables::from_editables(editables.iter().map(|editable| {
let EditableRequirement {
url,
extras,
path,
origin: _,
} = editable;
LocalEditable {
url: url.clone(),
extras: extras.clone(),
path: path.clone(),
}
}));
let editables: Vec<_> = downloader
.build_editables(editables, editable_wheel_dir)
.await
.context("Failed to build editables")?
.into_iter()
.collect();
// Validate that the editables are compatible with the target Python version.
for editable in &editables {
if let Some(python_requires) = editable.metadata.requires_python.as_ref() {
if !python_requires.contains(interpreter.python_version()) {
return Err(anyhow!(
"Editable `{}` requires Python {}, but {} is installed",
editable.metadata.name,
python_requires,
interpreter.python_version()
)
.into());
}
}
}
let s = if editables.len() == 1 { "" } else { "s" };
writeln!(
printer.stderr(),
"{}",
format!(
"Built {} in {}",
format!("{} editable{}", editables.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
Ok(editables)
}
/// Resolve a set of requirements, similar to running `pip compile`.
#[allow(clippy::too_many_arguments)]
async fn resolve(
@@ -651,7 +567,7 @@ async fn resolve(
constraints: Vec<Requirement>,
overrides: Vec<Requirement>,
project: Option<PackageName>,
editables: &[BuiltEditable],
editables: &[ResolvedEditable],
hasher: &HashStrategy,
site_packages: &SitePackages<'_>,
reinstall: &Reinstall,
@@ -713,17 +629,17 @@ async fn resolve(
// Map the editables to their metadata.
let editables: Vec<_> = editables
.iter()
.map(|built_editable| {
let dependencies: Vec<_> = built_editable
.metadata
.map(|editable| {
let dependencies: Vec<_> = editable
.metadata()
.requires_dist
.iter()
.cloned()
.map(Requirement::from_pep508)
.collect::<Result<_, _>>()?;
Ok::<_, Box<ParsedUrlError>>((
built_editable.editable.clone(),
built_editable.metadata.clone(),
editable.local().clone(),
editable.metadata().clone(),
Requirements {
dependencies,
optional_dependencies: IndexMap::default(),
@@ -811,7 +727,7 @@ async fn resolve(
#[allow(clippy::too_many_arguments)]
async fn install(
resolution: &Resolution,
built_editables: Vec<BuiltEditable>,
editables: &[ResolvedEditable],
site_packages: SitePackages<'_>,
reinstall: &Reinstall,
no_binary: &NoBinary,
@@ -833,16 +749,10 @@ async fn install(
let requirements = resolution.requirements();
// Map the built editables to their resolved form.
let editables = built_editables
.into_iter()
.map(ResolvedEditable::Built)
.collect::<Vec<_>>();
// Partition into those that should be linked from the cache (`local`), those that need to be
// downloaded (`remote`), and those that should be removed (`extraneous`).
let plan = Planner::with_requirements(&requirements)
.with_editable_requirements(&editables)
.with_editable_requirements(editables)
.build(
site_packages,
reinstall,
+1
View File
@@ -1,5 +1,6 @@
pub(crate) mod check;
pub(crate) mod compile;
mod editables;
pub(crate) mod freeze;
pub(crate) mod install;
pub(crate) mod list;
+9 -166
View File
@@ -2,23 +2,18 @@ use std::borrow::Cow;
use std::fmt::Write;
use anstream::eprint;
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result};
use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::{
IndexLocations, InstalledMetadata, LocalDist, LocalEditable, LocalEditables, Name, ResolvedDist,
};
use distribution_types::{IndexLocations, InstalledMetadata, LocalDist, Name, ResolvedDist};
use install_wheel_rs::linker::LinkMode;
use platform_tags::Tags;
use pypi_types::Yanked;
use requirements_txt::EditableRequirement;
use uv_auth::store_credentials_from_url;
use uv_cache::{ArchiveTarget, ArchiveTimestamp, Cache};
use uv_client::{
BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClient, RegistryClientBuilder,
};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Concurrency, ConfigSettings, IndexStrategy, NoBinary, NoBuild, PreviewMode, Reinstall,
SetupPyStrategy,
@@ -27,8 +22,8 @@ use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
use uv_fs::Simplified;
use uv_installer::{is_dynamic, Downloader, Plan, Planner, ResolvedEditable, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment, PythonVersion, Target};
use uv_installer::{Downloader, Plan, Planner, SitePackages};
use uv_interpreter::{PythonEnvironment, PythonVersion, Target};
use uv_requirements::{
ExtrasSpecification, NamedRequirementsResolver, RequirementsSource, RequirementsSpecification,
SourceTreeResolver,
@@ -39,6 +34,7 @@ use uv_resolver::{
use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;
use crate::commands::pip::editables::ResolvedEditables;
use crate::commands::reporters::{DownloadReporter, InstallReporter, ResolverReporter};
use crate::commands::{compile_bytecode, elapsed, ChangeEvent, ChangeEventKind, ExitStatus};
use crate::printer::Printer;
@@ -300,7 +296,7 @@ pub(crate) async fn pip_sync(
};
// Resolve any editables.
let resolved_editables = resolve_editables(
let editables = ResolvedEditables::resolve(
editables,
&site_packages,
reinstall,
@@ -323,7 +319,7 @@ pub(crate) async fn pip_sync(
reinstalls,
extraneous,
} = Planner::with_requirements(&requirements)
.with_editable_requirements(&resolved_editables.editables)
.with_editable_requirements(&editables)
.build(
site_packages,
reinstall,
@@ -611,156 +607,3 @@ pub(crate) async fn pip_sync(
Ok(ExitStatus::Success)
}
#[derive(Debug)]
struct ResolvedEditables {
/// The set of resolved editables, including both those that were already installed and those
/// that were built.
editables: Vec<ResolvedEditable>,
/// The temporary directory in which the built editables were stored.
#[allow(dead_code)]
temp_dir: Option<tempfile::TempDir>,
}
/// Resolve the set of editables that need to be installed.
#[allow(clippy::too_many_arguments)]
async fn resolve_editables(
editables: Vec<EditableRequirement>,
site_packages: &SitePackages<'_>,
reinstall: &Reinstall,
hasher: &HashStrategy,
interpreter: &Interpreter,
tags: &Tags,
cache: &Cache,
client: &RegistryClient,
build_dispatch: &BuildDispatch<'_>,
concurrency: Concurrency,
printer: Printer,
) -> Result<ResolvedEditables> {
// Partition the editables into those that are already installed, and those that must be built.
let mut installed = Vec::with_capacity(editables.len());
let mut uninstalled = Vec::with_capacity(editables.len());
for editable in editables {
match reinstall {
Reinstall::None => {
let existing = site_packages.get_editables(editable.raw());
match existing.as_slice() {
[] => uninstalled.push(editable),
[dist] => {
if ArchiveTimestamp::up_to_date_with(
&editable.path,
ArchiveTarget::Install(dist),
)? && !is_dynamic(&editable)
{
installed.push((*dist).clone());
} else {
uninstalled.push(editable);
}
}
_ => {
uninstalled.push(editable);
}
}
}
Reinstall::All => {
uninstalled.push(editable);
}
Reinstall::Packages(packages) => {
let existing = site_packages.get_editables(editable.raw());
match existing.as_slice() {
[] => uninstalled.push(editable),
[dist] => {
if packages.contains(dist.name()) {
uninstalled.push(editable);
} else if ArchiveTimestamp::up_to_date_with(
&editable.path,
ArchiveTarget::Install(dist),
)? && !is_dynamic(&editable)
{
installed.push((*dist).clone());
} else {
uninstalled.push(editable);
}
}
_ => {
uninstalled.push(editable);
}
}
}
}
}
// Build any editable installs.
let (built_editables, temp_dir) = if uninstalled.is_empty() {
(Vec::new(), None)
} else {
let start = std::time::Instant::now();
let downloader = Downloader::new(
cache,
tags,
hasher,
DistributionDatabase::new(client, build_dispatch, concurrency.downloads),
)
.with_reporter(DownloadReporter::from(printer).with_length(uninstalled.len() as u64));
let editables = LocalEditables::from_editables(uninstalled.iter().map(|editable| {
let EditableRequirement {
url,
path,
extras,
origin: _,
} = editable;
LocalEditable {
url: url.clone(),
path: path.clone(),
extras: extras.clone(),
}
}));
let editable_wheel_dir = tempfile::tempdir_in(cache.root())?;
let editables: Vec<_> = downloader
.build_editables(editables, editable_wheel_dir.path())
.await
.context("Failed to build editables")?
.into_iter()
.collect();
// Validate that the editables are compatible with the target Python version.
for editable in &editables {
if let Some(python_requires) = editable.metadata.requires_python.as_ref() {
if !python_requires.contains(interpreter.python_version()) {
return Err(anyhow!(
"Editable `{}` requires Python {}, but {} is installed",
editable.metadata.name,
python_requires,
interpreter.python_version()
));
}
}
}
let s = if editables.len() == 1 { "" } else { "s" };
writeln!(
printer.stderr(),
"{}",
format!(
"Built {} in {}",
format!("{} editable{}", editables.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
(editables, Some(editable_wheel_dir))
};
Ok(ResolvedEditables {
editables: installed
.into_iter()
.map(ResolvedEditable::Installed)
.chain(built_editables.into_iter().map(ResolvedEditable::Built))
.collect::<Vec<_>>(),
temp_dir,
})
}
+1 -4
View File
@@ -837,18 +837,15 @@ fn install_editable() {
----- stdout -----
----- stderr -----
Built 1 editable in [TIME]
Resolved 10 packages in [TIME]
Downloaded 6 packages in [TIME]
Installed 7 packages in [TIME]
Installed 6 packages in [TIME]
+ black==24.3.0
+ click==8.1.7
+ mypy-extensions==1.0.0
+ packaging==24.0
+ pathspec==0.12.1
+ platformdirs==4.2.0
- poetry-editable==0.1.0 (from file://[WORKSPACE]/scripts/packages/poetry_editable)
+ poetry-editable==0.1.0 (from file://[WORKSPACE]/scripts/packages/poetry_editable)
"###
);
}