From 94e08a153bd907d686e8ca84053cdd08dcf9b528 Mon Sep 17 00:00:00 2001 From: shamilbi Date: Mon, 2 Mar 2026 18:24:50 +0200 Subject: [PATCH 1/5] Accept-Encoding: gzip --- morgan/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/morgan/__init__.py b/morgan/__init__.py index bb72c79..2cab956 100644 --- a/morgan/__init__.py +++ b/morgan/__init__.py @@ -2,6 +2,7 @@ import argparse import configparser +import gzip import hashlib import json import os @@ -166,12 +167,18 @@ def _mirror( # noqa: C901, PLR0912 f"{self.index_url}{requirement.name}/", headers={ "Accept": "application/vnd.pypi.simple.v1+json", + "Accept-Encoding": "gzip", }, ) response_url = "" - with urllib.request.urlopen(request) as response: # noqa: S310 - data = json.load(response) + with urllib.request.urlopen(request) as response: + bytes1 = response.read() + try: + bytes2 = gzip.decompress(bytes1) + data = json.loads(bytes2) + except gzip.BadGzipFile: + data = json.loads(bytes1) response_url = str(response.url) if not data: msg = f"Failed loading metadata: {response}" From c854e906e07b04e161f563f1648001e22b1aecfd Mon Sep 17 00:00:00 2001 From: shamilbi Date: Mon, 2 Mar 2026 18:32:27 +0200 Subject: [PATCH 2/5] fix: ruff warnings --- morgan/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/morgan/__init__.py b/morgan/__init__.py index 2cab956..93b11de 100644 --- a/morgan/__init__.py +++ b/morgan/__init__.py @@ -140,6 +140,7 @@ def copy_server(self): with open(outpath, "w") as out: out.write(inspect.getsource(server)) + # ruff: noqa: PLR0915 def _mirror( # noqa: C901, PLR0912 self, requirement: packaging.requirements.Requirement, @@ -163,7 +164,7 @@ def _mirror( # noqa: C901, PLR0912 # get information about this package from the Simple API in JSON # format as per PEP 691 - request = urllib.request.Request( # noqa: S310 + request = urllib.request.Request( f"{self.index_url}{requirement.name}/", headers={ "Accept": "application/vnd.pypi.simple.v1+json", @@ -172,6 +173,7 @@ def _mirror( # noqa: C901, PLR0912 ) response_url = "" + # ruff: noqa: S310 with urllib.request.urlopen(request) as response: bytes1 = response.read() try: @@ -603,7 +605,7 @@ def _download_file( return True print("\t{}...".format(fileinfo["url"]), end=" ") - with urllib.request.urlopen(fileinfo["url"]) as inp, open(target, "wb") as out: # noqa: S310 + with urllib.request.urlopen(fileinfo["url"]) as inp, open(target, "wb") as out: out.write(inp.read()) print("done") From 3d2f9501e4c1a53d8a75b01e138a78c5120f648b Mon Sep 17 00:00:00 2001 From: shamilbi Date: Tue, 3 Mar 2026 19:20:20 +0200 Subject: [PATCH 3/5] +download_req() --- morgan/utils.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/morgan/utils.py b/morgan/utils.py index a2e48cb..909d26e 100644 --- a/morgan/utils.py +++ b/morgan/utils.py @@ -1,7 +1,10 @@ from __future__ import annotations +import gzip +import json import os import re +import urllib.request from collections import OrderedDict from typing import TYPE_CHECKING, Iterable @@ -48,6 +51,7 @@ def is_simple_case(self, req: Requirement) -> bool: if not specifier: return True # ruff: noqa: SLF001 + # pylint: disable=protected-access if all(spec.operator in (">", ">=") for spec in specifier._specs): return True return False @@ -151,3 +155,31 @@ def __setitem__(self, key, value): self[key].extend(value) else: super().__setitem__(key, value) + + +def download_req(index_url: str, req_name: str) -> tuple[dict, str]: + # get information about this package from the Simple API in JSON + # format as per PEP 691 + url = index_url.rstrip("/") + request = urllib.request.Request( + f"{url}/{req_name}/", + headers={ + "Accept": "application/vnd.pypi.simple.v1+json", + "Accept-Encoding": "gzip", + }, + ) + + response_url = "" + # ruff: noqa: S310 + with urllib.request.urlopen(request) as response: + bytes1 = response.read() + try: + bytes2 = gzip.decompress(bytes1) + data = json.loads(bytes2) + except gzip.BadGzipFile: + data = json.loads(bytes1) + response_url = str(response.url) + if data: + return data, response_url + msg = f"Failed loading metadata: {response}" + raise RuntimeError(msg) From 1030920500e8c138afed5586095a3d351f9d99d0 Mon Sep 17 00:00:00 2001 From: shamilbi Date: Tue, 3 Mar 2026 19:22:09 +0200 Subject: [PATCH 4/5] refactor: use download_req() --- morgan/__init__.py | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/morgan/__init__.py b/morgan/__init__.py index 93b11de..c09d747 100644 --- a/morgan/__init__.py +++ b/morgan/__init__.py @@ -2,9 +2,7 @@ import argparse import configparser -import gzip import hashlib -import json import os import os.path import re @@ -27,6 +25,7 @@ from morgan.utils import ( Cache, ListExtendingOrderedDict, + download_req, is_requirement_relevant, to_single_dash, touch_file, @@ -140,7 +139,6 @@ def copy_server(self): with open(outpath, "w") as out: out.write(inspect.getsource(server)) - # ruff: noqa: PLR0915 def _mirror( # noqa: C901, PLR0912 self, requirement: packaging.requirements.Requirement, @@ -160,31 +158,7 @@ def _mirror( # noqa: C901, PLR0912 else: print(f"{requirement}") - data: dict | None = None - - # get information about this package from the Simple API in JSON - # format as per PEP 691 - request = urllib.request.Request( - f"{self.index_url}{requirement.name}/", - headers={ - "Accept": "application/vnd.pypi.simple.v1+json", - "Accept-Encoding": "gzip", - }, - ) - - response_url = "" - # ruff: noqa: S310 - with urllib.request.urlopen(request) as response: - bytes1 = response.read() - try: - bytes2 = gzip.decompress(bytes1) - data = json.loads(bytes2) - except gzip.BadGzipFile: - data = json.loads(bytes1) - response_url = str(response.url) - if not data: - msg = f"Failed loading metadata: {response}" - raise RuntimeError(msg) + data, response_url = download_req(self.index_url, requirement.name) # check metadata version ~1.0 v_str = data["meta"]["api-version"] @@ -458,7 +432,7 @@ def _matches_environments( # noqa: C901, PLR0912 if fileinfo.get("tags"): # At least one of the tags must match ALL of our environments for tag in fileinfo["tags"]: - (intrp_name, intrp_ver) = parse_interpreter(tag.interpreter) + intrp_name, intrp_ver = parse_interpreter(tag.interpreter) if intrp_name not in ("py", "cp"): continue @@ -605,6 +579,7 @@ def _download_file( return True print("\t{}...".format(fileinfo["url"]), end=" ") + # ruff: noqa: S310 with urllib.request.urlopen(fileinfo["url"]) as inp, open(target, "wb") as out: out.write(inp.read()) print("done") From f4f138f272a0b231816ac6f7f41dd8f5bc73602b Mon Sep 17 00:00:00 2001 From: shamilbi Date: Tue, 3 Mar 2026 20:06:08 +0200 Subject: [PATCH 5/5] fix: Content-Encoding: https://github.com/ido50/morgan/pull/86#discussion_r2878060290 --- morgan/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/morgan/utils.py b/morgan/utils.py index 909d26e..1fa74b1 100644 --- a/morgan/utils.py +++ b/morgan/utils.py @@ -172,12 +172,12 @@ def download_req(index_url: str, req_name: str) -> tuple[dict, str]: response_url = "" # ruff: noqa: S310 with urllib.request.urlopen(request) as response: - bytes1 = response.read() - try: - bytes2 = gzip.decompress(bytes1) - data = json.loads(bytes2) - except gzip.BadGzipFile: - data = json.loads(bytes1) + # Check if response is gzip-encoded + if response.headers.get("Content-Encoding") == "gzip": + with gzip.GzipFile(fileobj=response) as gzip_response: + data = json.load(gzip_response) + else: + data = json.load(response) response_url = str(response.url) if data: return data, response_url