From dfc79a1fdc38d03c0e2df63687ec4d3a1ea9e895 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Tue, 28 Jul 2026 21:50:24 +0100 Subject: [PATCH] feat(parses): accept gzip-compressed upload bodies (Content-Encoding: gzip) Pure-ASGI GzipRequestMiddleware inflates gzip request bodies before FastAPI parsing and the ingest HMAC check read them - the signing contract stays "HMAC over the uncompressed JSON" for both formats, and requests without the header pass through untouched, so pre-gzip plugin versions keep working unchanged. Guards: 16 MiB decompressed cap (413, zip-bomb), invalid gzip -> 400. Why: ACT JSON compresses 10-20x. A raider on a ~120 kbit/s effective route (2026-07-28 incident) could not push a 282 KB payload inside the plugin's HttpClient timeout; gzipped it's ~25 KB / ~2 s. Plugin v0.1.16 ships the compression side. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 2 + backend/server/app.py | 8 ++ backend/server/core/gzip_request.py | 106 +++++++++++++++++++++++++++ tests/server/test_gzip_request.py | 110 ++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+) create mode 100644 backend/server/core/gzip_request.py create mode 100644 tests/server/test_gzip_request.py diff --git a/CLAUDE.md b/CLAUDE.md index 4284fb30..47a2ae4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,6 +58,8 @@ Plugin auto-detects the EQ2 server from its log file path (`/logs/ None: app.add_middleware(RequestContextMiddleware) + # Innermost body-touching layer: transparently gunzip request bodies + # (Content-Encoding: gzip) before FastAPI parsing / the ingest HMAC check + # read them. Plain requests pass through untouched — pre-gzip ACT plugin + # versions keep working. See backend/server/core/gzip_request.py. + from backend.server.core.gzip_request import GzipRequestMiddleware + + app.add_middleware(GzipRequestMiddleware) + # API routers — one entry per router, registered at /api prefix _ROUTERS = [ health_router, diff --git a/backend/server/core/gzip_request.py b/backend/server/core/gzip_request.py new file mode 100644 index 00000000..1083367c --- /dev/null +++ b/backend/server/core/gzip_request.py @@ -0,0 +1,106 @@ +"""Transparent gzip request-body decompression (pure ASGI middleware). + +The ACT plugin (v0.1.16+) gzips its ingest payloads — ACT JSON compresses +10-20×, which is the difference between a 2 s and a 20+ s upload on a slow +route (2026-07-28 incident: a raider on a ~120 kbit/s effective route could +not push a 282 KB payload inside the plugin's HttpClient timeout). + +When a request carries ``Content-Encoding: gzip``, the middleware drains the +body, decompresses it (bounded — zip-bomb guard), strips the header, fixes +``Content-Length``, and hands the plain body downstream. Everything after it +— FastAPI's model parsing AND the HMAC check's ``request.body()`` — sees the +uncompressed bytes, so the plugin's signature contract (HMAC over the +uncompressed JSON) holds for both compressed and plain uploads. Requests +without the header pass through completely untouched, so pre-gzip plugin +versions keep working unchanged. +""" + +from __future__ import annotations + +import gzip +import json +import logging +import zlib + +_log = logging.getLogger(__name__) + +# Decompressed-size cap. The plugin's own payload cap is 10 MiB of JSON; +# anything bigger than this after decompression is hostile or broken. +MAX_DECOMPRESSED_BYTES = 16 * 1024 * 1024 + + +def _plain_response(send, status: int, detail: str): + body = json.dumps({"detail": detail}).encode() + + async def _send() -> None: + await send( + { + "type": "http.response.start", + "status": status, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode()), + ], + } + ) + await send({"type": "http.response.body", "body": body}) + + return _send() + + +class GzipRequestMiddleware: + """Decompress gzip-encoded request bodies before anything reads them.""" + + def __init__(self, app) -> None: + self.app = app + + async def __call__(self, scope, receive, send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + headers = dict(scope.get("headers") or []) + if headers.get(b"content-encoding", b"").strip().lower() != b"gzip": + await self.app(scope, receive, send) + return + + # Drain the (compressed) body. + chunks: list[bytes] = [] + while True: + message = await receive() + if message["type"] == "http.disconnect": + return # client went away mid-upload; nothing to serve + chunks.append(message.get("body", b"")) + if not message.get("more_body", False): + break + + try: + # gzip.decompress inflates fully in one go — enforce the cap by + # streaming through a zlib decompressor with a max_length bound. + d = zlib.decompressobj(wbits=16 + zlib.MAX_WBITS) # gzip container + body = d.decompress(b"".join(chunks), MAX_DECOMPRESSED_BYTES) + if d.unconsumed_tail: + await _plain_response(send, 413, "Decompressed payload too large.") + return + d.flush() + except (zlib.error, gzip.BadGzipFile, EOFError) as exc: + _log.warning("[gzip-request] bad gzip body on %s: %s", scope.get("path"), exc) + await _plain_response(send, 400, "Request body is not valid gzip.") + return + + # Downstream must see a plain request: drop content-encoding, fix + # content-length, and replay the decompressed body as one message. + scope = dict(scope) + scope["headers"] = [ + (k, v) for k, v in scope["headers"] if k != b"content-encoding" and k != b"content-length" + ] + [(b"content-length", str(len(body)).encode())] + + sent = False + + async def replay_receive(): + nonlocal sent + if sent: + return await receive() # any follow-up (http.disconnect) + sent = True + return {"type": "http.request", "body": body, "more_body": False} + + await self.app(scope, replay_receive, send) diff --git a/tests/server/test_gzip_request.py b/tests/server/test_gzip_request.py new file mode 100644 index 00000000..77c18e07 --- /dev/null +++ b/tests/server/test_gzip_request.py @@ -0,0 +1,110 @@ +"""Tests for GzipRequestMiddleware — transparent request-body decompression. + +The ACT plugin (v0.1.16+) gzips ingest payloads and signs the UNCOMPRESSED +JSON; the middleware inflates the body before FastAPI parsing and the HMAC +check read it, so both compressed and plain uploads validate identically. +""" + +from __future__ import annotations + +import gzip +import json +import zlib +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from tests.server._parses_ingest_fixtures import ( + _fake_require_user, + _minimal_payload, + _sign, +) + + +def _gzipped_signed_kwargs(payload: dict, token: str = "eq2c_test_token") -> dict: + """What a v0.1.16+ plugin sends: gzip body, HMAC over the UNCOMPRESSED + JSON (the signing contract is unchanged from plain uploads).""" + body_bytes = json.dumps(payload).encode("utf-8") + return { + "content": gzip.compress(body_bytes), + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Content-Encoding": "gzip", + "X-Lexicon-Signature": _sign(body_bytes, token), + }, + } + + +@pytest.mark.asyncio +async def test_gzipped_ingest_validates_hmac_and_inserts(app): + sync_result = ("inserted", 42, 2, 1, 2) + with ( + patch("backend.server.api.parses.ingest.require_user_session_or_token", _fake_require_user), + patch("backend.server.api.parses.ingest._resolve_uploader_guild_async", new=AsyncMock(return_value="Exordium")), + patch("backend.server.api.parses.ingest._resolve_and_update_snapshots", new=AsyncMock()), + patch("backend.server.api.parses.ingest._ingest_payload_sync", new=MagicMock(return_value=sync_result)), + ): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post("/api/parses/ingest", **_gzipped_signed_kwargs(_minimal_payload())) + + assert r.status_code == 201 + assert r.json()["status"] == "inserted" + + +@pytest.mark.asyncio +async def test_gzipped_ingest_rejects_tampered_body(app): + """The HMAC contract survives compression: a signature over different + JSON than what was gzipped must 401 exactly like a plain tamper.""" + kwargs = _gzipped_signed_kwargs(_minimal_payload()) + kwargs["headers"]["X-Lexicon-Signature"] = _sign(b'{"tampered": true}', "eq2c_test_token") + with patch("backend.server.api.parses.ingest.require_user_session_or_token", _fake_require_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post("/api/parses/ingest", **kwargs) + assert r.status_code == 401 + + +@pytest.mark.asyncio +async def test_garbage_gzip_body_is_400(app): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post( + "/api/parses/ingest", + content=b"this is not gzip at all", + headers={"Content-Encoding": "gzip", "Content-Type": "application/json"}, + ) + assert r.status_code == 400 + assert "gzip" in r.json()["detail"].lower() + + +@pytest.mark.asyncio +async def test_zip_bomb_is_413(app): + # 20 MB of zeros compresses to ~20 KB but inflates past the 16 MB cap. + bomb = gzip.compress(b"\x00" * (20 * 1024 * 1024)) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post( + "/api/parses/ingest", + content=bomb, + headers={"Content-Encoding": "gzip", "Content-Type": "application/json"}, + ) + assert r.status_code == 413 + + +@pytest.mark.asyncio +async def test_requests_without_content_encoding_pass_through_untouched(app): + """Pre-gzip plugins and every browser request must be completely + unaffected — a plain (unsigned, unauthenticated) ingest POST still gets + the normal 401, not a middleware error.""" + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post("/api/parses/ingest", json=_minimal_payload()) + assert r.status_code == 401 + + +def test_decompress_cap_matches_module_constant(): + from backend.server.core import gzip_request + + # The cap must comfortably exceed the plugin's own 10 MiB payload cap. + assert gzip_request.MAX_DECOMPRESSED_BYTES >= 10 * 1024 * 1024 + # zlib gzip-container wbits sanity — decompressing a gzip stream works. + d = zlib.decompressobj(wbits=16 + zlib.MAX_WBITS) + assert d.decompress(gzip.compress(b"hello")) == b"hello"