From d8408b1783789bdc4aa9f3422951009891f32506 Mon Sep 17 00:00:00 2001 From: konsti Date: Fri, 10 Nov 2023 19:33:49 +0100 Subject: [PATCH] Add source to failing metadata parsing (#387) Before: ``` cargo run --bin puffin-dev -q -- resolve-cli "transformers[accelerate, agents, all, audio, codecarbon, deepspeed, deepspeed-testing, dev, dev-tensorflow, dev-torch, docs, docs_specific, flax, flax-speech, ftfy, integrations, ja, modelcreation, onnx, onnxruntime, optuna, quality, ray, retrieval, sagemaker, sentencepiece, serving, sigopt, sklearn, speech, testing, tf, tf-cpu, tf-speech, timm, tokenizers, torch, torch-speech, torch-vision, torchhub, video, vision]" puffin-dev failed Caused by: No solution found when resolving: transformers[accelerate,agents,all,audio,codecarbon,deepspeed,deepspeed-testing,dev,dev-tensorflow,dev-torch,docs,docs-specific,flax,flax-speech,ftfy,integrations,ja,modelcreation,onnx,onnxruntime,optuna,quality,ray,retrieval,sagemaker,sentencepiece,serving,sigopt,sklearn,speech,testing,tf,tf-cpu,tf-speech,timm,tokenizers,torch,torch-speech,torch-vision,torchhub,video,vision] Caused by: Not a valid package or extra name: ".none". Names must start and end with a letter or digit and may only contain -, _, ., and alphanumeric characters ``` After: ``` cargo run --bin puffin-dev -q -- resolve-cli "transformers[accelerate, agents, all, audio, codecarbon, deepspeed, deepspeed-testing, dev, dev-tensorflow, dev-torch, docs, docs_specific, flax, flax-speech, ftfy, integrations, ja, modelcreation, onnx, onnxruntime, optuna, quality, ray, retrieval, sagemaker, sentencepiece, serving, sigopt, sklearn, speech, testing, tf, tf-cpu, tf-speech, timm, tokenizers, torch, torch-speech, torch-vision, torchhub, video, vision]" puffin-dev failed Caused by: No solution found when resolving: transformers[accelerate,agents,all,audio,codecarbon,deepspeed,deepspeed-testing,dev,dev-tensorflow,dev-torch,docs,docs-specific,flax,flax-speech,ftfy,integrations,ja,modelcreation,onnx,onnxruntime,optuna,quality,ray,retrieval,sagemaker,sentencepiece,serving,sigopt,sklearn,speech,testing,tf,tf-cpu,tf-speech,timm,tokenizers,torch,torch-speech,torch-vision,torchhub,video,vision] Caused by: Couldn't parse metadata in fastapi-0.10.1-py3-none-any.whl (https://files.pythonhosted.org/packages/6d/b8/97ac91cb7c8f661a810334bde310ed2b2885cce6d2baab1a50b0c7a17d83/fastapi-0.10.1-py3-none-any.whl) Caused by: Not a valid package or extra name: ".none". Names must start and end with a letter or digit and may only contain -, _, ., and alphanumeric characters ``` --- builder.dockerfile | 1 + crates/puffin-client/src/client.rs | 12 ++++++++---- crates/puffin-client/src/error.rs | 5 +++-- crates/puffin-dispatch/src/lib.rs | 2 +- .../puffin-resolver/src/distribution/built_dist.rs | 4 ++-- .../puffin-resolver/src/distribution/source_dist.rs | 4 ++-- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/builder.dockerfile b/builder.dockerfile index 1a6ced1d0..a1ab94768 100644 --- a/builder.dockerfile +++ b/builder.dockerfile @@ -7,6 +7,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ autoconf \ build-essential \ + cmake \ curl \ make \ pkg-config \ diff --git a/crates/puffin-client/src/client.rs b/crates/puffin-client/src/client.rs index dc5f0fae5..a239397e1 100644 --- a/crates/puffin-client/src/client.rs +++ b/crates/puffin-client/src/client.rs @@ -212,8 +212,9 @@ impl RegistryClient { let cache_dir = self.cache.join(WHEEL_METADATA_FROM_INDEX).join("pypi"); let cache_file = format!("{}.json", filename.stem()); - let response_callback = |response: Response| async move { - Ok(Metadata21::parse(response.bytes().await?.as_ref())?) + let response_callback = |response: Response| async { + Metadata21::parse(response.bytes().await?.as_ref()) + .map_err(|err| Error::MetadataParseError(filename, url.clone(), err)) }; let req = self.client_raw.get(url.clone()).build()?; self.cached_client @@ -243,7 +244,8 @@ impl RegistryClient { let mut reader = AsyncHttpRangeReader::from_head_response(client, response).await?; trace!("Getting metadata for {filename} by range request"); let text = wheel_metadata_from_remote_zip(filename, &mut reader).await?; - let metadata = Metadata21::parse(text.as_bytes())?; + let metadata = Metadata21::parse(text.as_bytes()) + .map_err(|err| Error::MetadataParseError(filename.clone(), url.clone(), err))?; Ok(metadata) }; @@ -308,7 +310,9 @@ impl RegistryClient { .await .map_err(|err| Error::Zip(filename.clone(), err))?; - Ok(Metadata21::parse(&contents)?) + let metadata = Metadata21::parse(&contents) + .map_err(|err| Error::MetadataParseError(filename.clone(), url.clone(), err))?; + Ok(metadata) } /// Stream a file from an external URL. diff --git a/crates/puffin-client/src/error.rs b/crates/puffin-client/src/error.rs index e24d794d3..614a904ca 100644 --- a/crates/puffin-client/src/error.rs +++ b/crates/puffin-client/src/error.rs @@ -3,6 +3,7 @@ use std::io; use async_http_range_reader::AsyncHttpRangeReaderError; use async_zip::error::ZipError; use thiserror::Error; +use url::Url; use distribution_filename::{WheelFilename, WheelFilenameError}; @@ -23,8 +24,8 @@ pub enum Error { PackageNotFound(String), /// The metadata file could not be parsed. - #[error(transparent)] - MetadataParseError(#[from] pypi_types::Error), + #[error("Couldn't parse metadata in {0} ({1})")] + MetadataParseError(WheelFilename, Url, #[source] pypi_types::Error), /// The metadata file was not found in the registry. #[error("File `{0}` was not found in the registry at {1}.")] diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index e3687a9f9..ad6577a26 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -94,7 +94,7 @@ impl BuildContext for BuildDispatch { ); let resolution_graph = resolver.resolve().await.with_context(|| { format!( - "No solution found when resolving build dependencies for source distribution: {}", + "No solution found when resolving: {}", requirements.iter().map(ToString::to_string).join(", "), ) })?; diff --git a/crates/puffin-resolver/src/distribution/built_dist.rs b/crates/puffin-resolver/src/distribution/built_dist.rs index a8864669b..d0635c842 100644 --- a/crates/puffin-resolver/src/distribution/built_dist.rs +++ b/crates/puffin-resolver/src/distribution/built_dist.rs @@ -61,9 +61,9 @@ impl<'a> BuiltDistFetcher<'a> { // Read the metadata from the wheel. let wheel = CachedWheel::new(wheel_file, WheelFilename::from_str(wheel_filename)?); - let metadata21 = wheel.read_dist_info()?; + let metadata = wheel.read_dist_info()?; debug!("Finished downloading: {dist}"); - Ok(metadata21) + Ok(metadata) } } diff --git a/crates/puffin-resolver/src/distribution/source_dist.rs b/crates/puffin-resolver/src/distribution/source_dist.rs index 0cdf1833a..d5a8c9f7e 100644 --- a/crates/puffin-resolver/src/distribution/source_dist.rs +++ b/crates/puffin-resolver/src/distribution/source_dist.rs @@ -159,10 +159,10 @@ impl<'a, T: BuildContext> SourceDistFetcher<'a, T> { wheel_dir.join(&disk_filename), WheelFilename::from_str(&disk_filename)?, ); - let metadata21 = wheel.read_dist_info()?; + let metadata = wheel.read_dist_info()?; debug!("Finished building: {dist}"); - Ok(metadata21) + Ok(metadata) } /// Given a remote source distribution, return a precise variant, if possible.