pioduino/uv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Rebase: Uninstall existing non-editable versions when installing editable requirements bug (#682)

Browse Source 
Separate branch for rebasing #677 onto main because i don't trust the
rebase enough to force push.

Closes #677.

---

If you install `black` from PyPI, then `-e ../black`, we need to
uninstall the existing `black`. This sounds simple, but that in turn
requires that we _know_ `-e ../black` maps to the package `black`, so
that we can mark it for uninstallation in the install plan. This, in
turn, means that we need to build editable dependencies prior to the
install plan.

This is just a bunch of reorganization to fix that specific bug
(installing multiple versions of `black` if you run through the above
workflow): we now run through the list of editables upfront, mark those
that are already installed, build those that aren't, and then ensure
that `InstallPlan` correctly removes those that need to be removed, etc.

Closes #676.

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>
This commit is contained in:
konsti
2023-12-18 10:28:14 +01:00
committed by GitHub
parent 0bb2c92246
commit f4f67ebde0
13 changed files with 626 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/puffin-cli/src/commands/pip_compile.rs
+4 -9
View File
@@ -114,9 +114,6 @@ pub(crate) async fn pip_compile(
.map(|spec| spec.requirements)
.unwrap_or_default();
// Create a manifest of the requirements.
let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
// Detect the current Python interpreter.
let platform = Platform::current()?;
let venv = Virtualenv::from_env(platform, &cache)?;
@@ -127,13 +124,9 @@ pub(crate) async fn pip_compile(
venv.python_executable().display()
);
// Determine the compatible platform tags.
let tags = Tags::from_interpreter(venv.interpreter())?;
// Determine the interpreter to use for resolution.
// Determine the tags, markers, and interpreter to use for resolution.
let interpreter = venv.interpreter().clone();
// Determine the markers to use for resolution.
let tags = Tags::from_interpreter(venv.interpreter())?;
let markers = python_version.map_or_else(
|| Cow::Borrowed(venv.interpreter().markers()),
|python_version| Cow::Owned(python_version.markers(venv.interpreter().markers())),
@@ -144,6 +137,7 @@ pub(crate) async fn pip_compile(
.index_urls(index_urls.clone())
.build();
let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
let build_dispatch = BuildDispatch::new(
client.clone(),
cache.clone(),
@@ -203,6 +197,7 @@ pub(crate) async fn pip_compile(
editable_metadata
};
// Create a manifest of the requirements.
let manifest = Manifest::new(
requirements,
constraints,
crates/puffin-cli/src/commands/pip_install.rs
+66 -64
View File
@@ -18,9 +18,10 @@ use puffin_cache::Cache;
use puffin_client::{RegistryClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_installer::{
BuiltEditable, Downloader, EditableMode, InstallPlan, Reinstall, SitePackages,
BuiltEditable, Downloader, InstallPlan, Reinstall, ResolvedEditable, SitePackages,
};
use puffin_interpreter::Virtualenv;
use puffin_normalize::PackageName;
use puffin_resolver::{
Manifest, PreReleaseMode, ResolutionGraph, ResolutionMode, ResolutionOptions, Resolver,
};
@@ -62,8 +63,32 @@ pub(crate) async fn pip_install(
let start = std::time::Instant::now();
// Determine the requirements.
let spec = specification(requirements, constraints, overrides, extras)?;
// Read all requirements from the provided sources.
let RequirementsSpecification {
project,
requirements,
constraints,
overrides,
editables,
extras: used_extras,
} = specification(requirements, constraints, overrides, extras)?;
// Check that all provided extras are used
if let ExtrasSpecification::Some(extras) = extras {
let mut unused_extras = extras
.iter()
.filter(|extra| !used_extras.contains(extra))
.collect::<Vec<_>>();
if !unused_extras.is_empty() {
unused_extras.sort_unstable();
unused_extras.dedup();
let s = if unused_extras.len() == 1 { "" } else { "s" };
return Err(anyhow!(
"Requested extra{s} not found: {}",
unused_extras.iter().join(", ")
));
}
}
// Detect the current Python interpreter.
let platform = Platform::current()?;
@@ -73,11 +98,15 @@ pub(crate) async fn pip_install(
venv.python_executable().display()
);
// Determine the set of installed packages.
let site_packages =
SitePackages::from_executable(&venv).context("Failed to list installed packages")?;
// If the requirements are already satisfied, we're done. Ideally, the resolver would be fast
// enough to let us remove this check. But right now, for large environments, it's an order of
// magnitude faster to validate the environment than to resolve the requirements.
if reinstall.is_none() && satisfied(&spec, &venv)? {
let num_requirements = spec.requirements.len() + spec.editables.len();
if reinstall.is_none() && site_packages.satisfies(&requirements, &editables, &constraints)? {
let num_requirements = requirements.len() + editables.len();
let s = if num_requirements == 1 { "" } else { "s" };
writeln!(
printer,
@@ -92,13 +121,9 @@ pub(crate) async fn pip_install(
return Ok(ExitStatus::Success);
}
// Determine the compatible platform tags.
let tags = Tags::from_interpreter(venv.interpreter())?;
// Determine the interpreter to use for resolution.
// Determine the tags, markers, and interpreter to use for resolution.
let interpreter = venv.interpreter().clone();
// Determine the markers to use for resolution.
let tags = Tags::from_interpreter(venv.interpreter())?;
let markers = venv.interpreter().markers();
// Instantiate a client.
@@ -107,6 +132,7 @@ pub(crate) async fn pip_install(
.build();
let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer);
let build_dispatch = BuildDispatch::new(
client.clone(),
cache.clone(),
@@ -121,12 +147,12 @@ pub(crate) async fn pip_install(
// installation, and should live for the duration of the command. If an editable is already
// installed in the environment, we'll still re-build it here.
let editable_wheel_dir;
let editables = if spec.editables.is_empty() {
let editables = if editables.is_empty() {
vec![]
} else {
editable_wheel_dir = tempdir_in(venv.root())?;
build_editables(
&spec.editables,
&editables,
editable_wheel_dir.path(),
&cache,
&tags,
@@ -139,15 +165,18 @@ pub(crate) async fn pip_install(
// Resolve the requirements.
let resolution = match resolve(
spec,
requirements,
constraints,
overrides,
project,
&editables,
&site_packages,
reinstall,
&tags,
markers,
&client,
&build_dispatch,
options,
&venv,
printer,
)
.await
@@ -168,7 +197,8 @@ pub(crate) async fn pip_install(
// Sync the environment.
install(
&resolution,
&editables,
editables,
site_packages,
reinstall,
link_mode,
index_urls,
@@ -228,15 +258,6 @@ fn specification(
Ok(spec)
}
/// Returns `true` if the requirements are already satisfied.
fn satisfied(spec: &RequirementsSpecification, venv: &Virtualenv) -> Result<bool, Error> {
Ok(SitePackages::from_executable(venv)?.satisfies(
&spec.requirements,
&spec.editables,
&spec.constraints,
)?)
}
/// Build a set of editable distributions.
async fn build_editables(
editables: &[EditableRequirement],
@@ -290,36 +311,27 @@ async fn build_editables(
/// Resolve a set of requirements, similar to running `pip-compile`.
#[allow(clippy::too_many_arguments)]
async fn resolve(
spec: RequirementsSpecification,
requirements: Vec<Requirement>,
constraints: Vec<Requirement>,
overrides: Vec<Requirement>,
project: Option<PackageName>,
editables: &[BuiltEditable],
site_packages: &SitePackages<'_>,
reinstall: &Reinstall,
tags: &Tags,
markers: &MarkerEnvironment,
client: &RegistryClient,
build_dispatch: &BuildDispatch,
options: ResolutionOptions,
venv: &Virtualenv,
mut printer: Printer,
) -> Result<ResolutionGraph, Error> {
let start = std::time::Instant::now();
// Create a manifest of the requirements.
let RequirementsSpecification {
project,
requirements,
constraints,
overrides,
editables: _,
extras: _,
} = spec;
// Respect preferences from the existing environments.
let preferences: Vec<Requirement> = match reinstall {
Reinstall::All => vec![],
Reinstall::None => SitePackages::from_executable(venv)?
.requirements()
.collect(),
Reinstall::Packages(packages) => SitePackages::from_executable(venv)?
Reinstall::None => site_packages.requirements().collect(),
Reinstall::Packages(packages) => site_packages
.requirements()
.filter(|requirement| !packages.contains(&requirement.name))
.collect(),
@@ -336,6 +348,7 @@ async fn resolve(
})
.collect();
// Create a manifest of the requirements.
let manifest = Manifest::new(
requirements,
constraints,
@@ -369,7 +382,8 @@ async fn resolve(
#[allow(clippy::too_many_arguments)]
async fn install(
resolution: &Resolution,
built_editables: &[BuiltEditable],
built_editables: Vec<BuiltEditable>,
site_packages: SitePackages<'_>,
reinstall: &Reinstall,
link_mode: LinkMode,
index_urls: IndexUrls,
@@ -384,26 +398,30 @@ async fn install(
// Partition into those that should be linked from the cache (`local`), those that need to be
// downloaded (`remote`), and those that should be removed (`extraneous`).
let requirements = resolution.requirements();
let editables = built_editables
.into_iter()
.map(ResolvedEditable::Built)
.collect::<Vec<_>>();
let InstallPlan {
local,
remote,
reinstalls,
editables,
extraneous: _,
} = InstallPlan::from_requirements(
&resolution.requirements(),
&resolution.editable_requirements(),
&requirements,
editables,
site_packages,
reinstall,
&index_urls,
cache,
venv,
tags,
EditableMode::Mutable,
)
.context("Failed to determine installation plan")?;
// Nothing to do.
if remote.is_empty() && local.is_empty() && reinstalls.is_empty() && editables.is_empty() {
if remote.is_empty() && local.is_empty() && reinstalls.is_empty() {
let s = if resolution.len() == 1 { "" } else { "s" };
writeln!(
printer,
@@ -430,18 +448,6 @@ async fn install(
})
.collect::<Vec<_>>();
// Map any local editable requirements back to those that were built ahead of time.
let built_editables = editables
.iter()
.map(|editable| {
let built_editable = built_editables
.iter()
.find(|built_editable| built_editable.editable.requirement == *editable)
.expect("Editable should be built");
built_editable.wheel.clone()
})
.collect::<Vec<_>>();
// Download, build, and unzip any missing distributions.
let wheels = if remote.is_empty() {
vec![]
@@ -487,11 +493,7 @@ async fn install(
}
// Install the resolved distributions.
let wheels = wheels
.into_iter()
.chain(local)
.chain(built_editables)
.collect::<Vec<_>>();
let wheels = wheels.into_iter().chain(local).collect::<Vec<_>>();
if !wheels.is_empty() {
let start = std::time::Instant::now();
puffin_installer::Installer::new(venv)
crates/puffin-cli/src/commands/pip_sync.rs
+159 -106
View File
@@ -4,18 +4,16 @@ use anyhow::{bail, Context, Result};
use colored::Colorize;
use fs_err as fs;
use itertools::Itertools;
use tempfile::tempdir_in;
use tracing::debug;
use distribution_types::{AnyDist, CachedDist, LocalEditable, Metadata};
use distribution_types::{AnyDist, LocalEditable, Metadata};
use install_wheel_rs::linker::LinkMode;
use pep508_rs::Requirement;
use platform_host::Platform;
use platform_tags::Tags;
use puffin_cache::Cache;
use puffin_client::RegistryClientBuilder;
use puffin_client::{RegistryClient, RegistryClientBuilder};
use puffin_dispatch::BuildDispatch;
use puffin_installer::{Downloader, EditableMode, InstallPlan, Reinstall, SitePackages};
use puffin_installer::{Downloader, InstallPlan, Reinstall, ResolvedEditable, SitePackages};
use puffin_interpreter::Virtualenv;
use puffin_traits::OnceMap;
use pypi_types::{IndexUrls, Yanked};
@@ -27,6 +25,7 @@ use crate::printer::Printer;
use crate::requirements::{RequirementsSource, RequirementsSpecification};
/// Install a set of locked requirements into the current Python environment.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn pip_sync(
sources: &[RequirementsSource],
reinstall: &Reinstall,
@@ -36,44 +35,19 @@ pub(crate) async fn pip_sync(
cache: Cache,
mut printer: Printer,
) -> Result<ExitStatus> {
let start = std::time::Instant::now();
// Read all requirements from the provided sources.
let (requirements, editables) = RequirementsSpecification::requirements_and_editables(sources)?;
if requirements.is_empty() && editables.is_empty() {
let num_requirements = requirements.len() + editables.len();
if num_requirements == 0 {
writeln!(printer, "No requirements found")?;
return Ok(ExitStatus::Success);
}
sync_requirements(
&requirements,
reinstall,
&editables,
link_mode,
index_urls,
no_build,
&cache,
printer,
)
.await
}
/// Install a set of locked requirements into the current Python environment.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn sync_requirements(
requirements: &[Requirement],
reinstall: &Reinstall,
editable_requirements: &[EditableRequirement],
link_mode: LinkMode,
index_urls: IndexUrls,
no_build: bool,
cache: &Cache,
mut printer: Printer,
) -> Result<ExitStatus> {
let start = std::time::Instant::now();
// Detect the current Python interpreter.
let platform = Platform::current()?;
let venv = Virtualenv::from_env(platform, cache)?;
let venv = Virtualenv::from_env(platform, &cache)?;
debug!(
"Using Python interpreter: {}",
venv.python_executable().display()
@@ -82,34 +56,60 @@ pub(crate) async fn sync_requirements(
// Determine the current environment markers.
let tags = Tags::from_interpreter(venv.interpreter())?;
// Prep the registry client.
let client = RegistryClientBuilder::new(cache.clone())
.index_urls(index_urls.clone())
.build();
// Prep the build context.
let build_dispatch = BuildDispatch::new(
client.clone(),
cache.clone(),
venv.interpreter().clone(),
fs::canonicalize(venv.python_executable())?,
no_build,
index_urls.clone(),
);
// Determine the set of installed packages.
let site_packages =
SitePackages::from_executable(&venv).context("Failed to list installed packages")?;
// Resolve any editables.
let resolved_editables = resolve_editables(
editables,
&site_packages,
reinstall,
&venv,
&tags,
&cache,
&client,
&build_dispatch,
printer,
)
.await?;
// Partition into those that should be linked from the cache (`local`), those that need to be
// downloaded (`remote`), and those that should be removed (`extraneous`).
let InstallPlan {
local,
editables,
remote,
reinstalls,
extraneous,
} = InstallPlan::from_requirements(
requirements,
editable_requirements,
&requirements,
resolved_editables.editables,
site_packages,
reinstall,
&index_urls,
cache,
&cache,
&venv,
&tags,
EditableMode::Immutable,
)
.context("Failed to determine installation plan")?;
// Nothing to do.
if remote.is_empty()
&& local.is_empty()
&& reinstalls.is_empty()
&& extraneous.is_empty()
&& editables.is_empty()
{
let num_requirements = requirements.len() + editable_requirements.len();
if remote.is_empty() && local.is_empty() && reinstalls.is_empty() && extraneous.is_empty() {
let s = if num_requirements == 1 { "" } else { "s" };
writeln!(
printer,
@@ -182,66 +182,15 @@ pub(crate) async fn sync_requirements(
}
}
let build_dispatch = BuildDispatch::new(
client.clone(),
cache.clone(),
venv.interpreter().clone(),
fs::canonicalize(venv.python_executable())?,
no_build,
index_urls.clone(),
);
let downloader = Downloader::new(cache, &tags, &client, &build_dispatch).with_reporter(
DownloadReporter::from(printer).with_length((editables.len() + remote.len()) as u64),
);
// Build any editable requirements.
let editable_wheel_dir = tempdir_in(venv.root())?;
let built_editables = if editables.is_empty() {
Vec::new()
} else {
let start = std::time::Instant::now();
let editables: Vec<LocalEditable> = editables
.into_iter()
.map(|editable| match editable.clone() {
EditableRequirement::Path { path, .. } => Ok(LocalEditable {
requirement: editable,
path,
}),
EditableRequirement::Url(_) => {
bail!("url editables are not supported yet");
}
})
.collect::<Result<_>>()?;
let built_editables: Vec<CachedDist> = downloader
.build_editables(editables, editable_wheel_dir.path())
.await
.context("Failed to build editables")?
.into_iter()
.map(|built_editable| built_editable.wheel)
.collect();
let s = if built_editables.len() == 1 { "" } else { "s" };
writeln!(
printer,
"{}",
format!(
"Built {} in {}",
format!("{} editable{}", built_editables.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
built_editables
};
// Download, build, and unzip any missing distributions.
let wheels = if remote.is_empty() {
Vec::new()
} else {
let start = std::time::Instant::now();
let downloader = Downloader::new(&cache, &tags, &client, &build_dispatch)
.with_reporter(DownloadReporter::from(printer).with_length(remote.len() as u64));
let wheels = downloader
.download(remote, &OnceMap::default())
.await
@@ -296,11 +245,7 @@ pub(crate) async fn sync_requirements(
}
// Install the resolved distributions.
let wheels = wheels
.into_iter()
.chain(local)
.chain(built_editables)
.collect::<Vec<_>>();
let wheels = wheels.into_iter().chain(local).collect::<Vec<_>>();
if !wheels.is_empty() {
let start = std::time::Instant::now();
puffin_installer::Installer::new(&venv)
@@ -376,3 +321,111 @@ pub(crate) async fn sync_requirements(
Ok(ExitStatus::Success)
}
#[derive(Debug)]
struct ResolvedEditables {
/// The set of resolved editables, including both those that were already installed and those
/// that were built.
editables: Vec<ResolvedEditable>,
/// The temporary directory in which the built editables were stored.
#[allow(dead_code)]
temp_dir: Option<tempfile::TempDir>,
}
/// Resolve the set of editables that need to be installed.
#[allow(clippy::too_many_arguments)]
async fn resolve_editables(
editables: Vec<EditableRequirement>,
site_packages: &SitePackages<'_>,
reinstall: &Reinstall,
venv: &Virtualenv,
tags: &Tags,
cache: &Cache,
client: &RegistryClient,
build_dispatch: &BuildDispatch,
mut printer: Printer,
) -> Result<ResolvedEditables> {
// Partition the editables into those that are already installed, and those that must be built.
let mut installed = Vec::with_capacity(editables.len());
let mut uninstalled = Vec::with_capacity(editables.len());
for editable in editables {
match reinstall {
Reinstall::None => {
if let Some(dist) = site_packages.get_editable(editable.raw()) {
installed.push(dist.clone());
} else {
uninstalled.push(editable);
}
}
Reinstall::All => {
uninstalled.push(editable);
}
Reinstall::Packages(packages) => {
if let Some(dist) = site_packages.get_editable(editable.raw()) {
if packages.contains(dist.name()) {
uninstalled.push(editable);
} else {
installed.push(dist.clone());
}
} else {
uninstalled.push(editable);
}
}
}
}
// Build any editable installs.
let (built_editables, temp_dir) = if uninstalled.is_empty() {
(Vec::new(), None)
} else {
let start = std::time::Instant::now();
let temp_dir = tempfile::tempdir_in(venv.root())?;
let downloader = Downloader::new(cache, tags, client, build_dispatch)
.with_reporter(DownloadReporter::from(printer).with_length(uninstalled.len() as u64));
let local_editables: Vec<LocalEditable> = uninstalled
.iter()
.map(|editable| match editable {
EditableRequirement::Path { path, .. } => Ok(LocalEditable {
requirement: editable.clone(),
path: path.clone(),
}),
EditableRequirement::Url(_) => {
bail!("Editable installs for URLs are not yet supported");
}
})
.collect::<Result<_>>()?;
let built_editables: Vec<_> = downloader
.build_editables(local_editables, temp_dir.path())
.await
.context("Failed to build editables")?
.into_iter()
.collect();
let s = if built_editables.len() == 1 { "" } else { "s" };
writeln!(
printer,
"{}",
format!(
"Built {} in {}",
format!("{} editable{}", built_editables.len(), s).bold(),
elapsed(start.elapsed())
)
.dimmed()
)?;
(built_editables, Some(temp_dir))
};
Ok(ResolvedEditables {
editables: installed
.into_iter()
.map(ResolvedEditable::Installed)
.chain(built_editables.into_iter().map(ResolvedEditable::Built))
.collect::<Vec<_>>(),
temp_dir,
})
}