From c57fb4dcefdb9387fced1d2e8a7fcb7749cdf071 Mon Sep 17 00:00:00 2001 From: Jo <10510431+j178@users.noreply.github.com> Date: Wed, 19 Feb 2025 22:57:56 +0800 Subject: [PATCH] Make fetching github releases faster (#11614) I noticed that the latest two `sync-python-releases` jobs failed due to `httpx.RemoteProtocolError: peer closed connection without sending complete message body (incomplete chunked read)`. For the current python-build-standalone release, each request page (defaulting to 30 items per page) takes about 20 seconds and loads around 32MB of data. This extensive data load might be causing the request to frequently fail. In this PR, I reduced number of items per page to 10 and added `Accept-Encoding: gzip, deflate` to the request header. Now, it takes about 6 seconds to load, and the compressed response size has been reduced to 534KB. I hope this would addresses the request failure. --- crates/uv-python/fetch-download-metadata.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/uv-python/fetch-download-metadata.py b/crates/uv-python/fetch-download-metadata.py index 4a858c120..ef2651c74 100755 --- a/crates/uv-python/fetch-download-metadata.py +++ b/crates/uv-python/fetch-download-metadata.py @@ -238,7 +238,9 @@ class CPythonFinder(Finder): # Collect all available Python downloads for page in range(1, pages + 1): logging.info("Fetching CPython release page %d", page) - resp = await self.client.get(self.RELEASE_URL, params={"page": page}) + resp = await self.client.get( + self.RELEASE_URL, params={"page": page, "per_page": 10} + ) resp.raise_for_status() rows = resp.json() if not rows: @@ -595,7 +597,10 @@ async def find() -> None: "`GITHUB_TOKEN` env var not found, you may hit rate limits for GitHub API requests." ) - headers = {"X-GitHub-Api-Version": "2022-11-28"} + headers = { + "X-GitHub-Api-Version": "2022-11-28", + "Accept-Encoding": "gzip, deflate", + } if token: headers["Authorization"] = "Bearer " + token client = httpx.AsyncClient(follow_redirects=True, headers=headers, timeout=15)