From 5279f591c091e77f26f980fdb2b0379cd6d7c2c6 Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sat, 4 Jul 2026 18:11:30 -0700 Subject: [PATCH 1/7] fix: use keyless Rijksmuseum data.rijksmuseum.nl Search API The old www.rijksmuseum.nl/api endpoint required an API key, and the app refused Rijksmuseum searches with "RIJKSMUSEUM_API_KEY not configured". Rijksmuseum's new Search API (https://data.rijksmuseum.nl/docs/search) is public and needs no key. - Rewrite backend/services/sources/rijksmuseum.py against the new API: search on creator/title/description (no free-text param exists), resolve each Linked Open Data id (object -> VisualItem -> DigitalObject) concurrently to get metadata and the IIIF image URL; thumbnails via IIIF size parameters. - Drop the RIJKSMUSEUM_API_KEY gate from the router and the setting from config, .env.example, Settings page, and readme. Co-Authored-By: Claude Fable 5 --- .env.example | 1 - backend/config.py | 1 - backend/routers/sources.py | 2 - backend/services/sources/rijksmuseum.py | 184 ++++++++++++++++++------ frontend/src/pages/Settings.tsx | 3 +- readme.md | 3 +- 6 files changed, 140 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index ce87034..6eea75d 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,6 @@ THUMBNAIL_DIR=./data/thumbnails TV_RESOLUTION=4K PORTRAIT_HANDLING=blur UNSPLASH_API_KEY= -RIJKSMUSEUM_API_KEY= NASA_API_KEY= REDDIT_USER_AGENT=sawsube/1.0 (local self-hosted) HOST=0.0.0.0 diff --git a/backend/config.py b/backend/config.py index 565f878..ef7c759 100644 --- a/backend/config.py +++ b/backend/config.py @@ -16,7 +16,6 @@ class Settings(BaseSettings): TV_RESOLUTION: str = "4K" # 4K | 1080p PORTRAIT_HANDLING: str = "blur" # blur | crop | skip UNSPLASH_API_KEY: str = "" - RIJKSMUSEUM_API_KEY: str = "" NASA_API_KEY: str = "" PEXELS_API_KEY: str = "" PIXABAY_API_KEY: str = "" diff --git a/backend/routers/sources.py b/backend/routers/sources.py index ee8f985..8eb0480 100644 --- a/backend/routers/sources.py +++ b/backend/routers/sources.py @@ -105,8 +105,6 @@ async def nasa_import(): # ── Rijksmuseum ──────────────────────────────────────────────────────────── @router.get("/rijksmuseum/search") async def rijks_search(q: str = Query(...), per_page: int = 20): - if not settings.RIJKSMUSEUM_API_KEY: - raise HTTPException(503, "RIJKSMUSEUM_API_KEY not configured — add it to .env") return await rijksmuseum.search(q, per_page) diff --git a/backend/services/sources/rijksmuseum.py b/backend/services/sources/rijksmuseum.py index 459e2f2..bb13bab 100644 --- a/backend/services/sources/rijksmuseum.py +++ b/backend/services/sources/rijksmuseum.py @@ -1,55 +1,147 @@ from __future__ import annotations +import asyncio import httpx -from ...config import settings +# Rijksmuseum's new Search API (https://data.rijksmuseum.nl/docs/search) is +# public and needs no API key. Search returns Linked Open Data identifiers; +# each is resolved (object -> VisualItem -> DigitalObject) to get metadata +# and the IIIF image URL. +SEARCH_URL = "https://data.rijksmuseum.nl/search/collection" +RESOLVE_URL = "https://id.rijksmuseum.nl/{oid}" +HEADERS = {"Accept": "application/ld+json"} -async def search(query: str, per_page: int = 20) -> list[dict]: - if not settings.RIJKSMUSEUM_API_KEY: +AAT_OBJECT_NUMBER = "http://vocab.getty.edu/aat/300312355" +AAT_ENGLISH = "http://vocab.getty.edu/aat/300388277" + +_CONCURRENCY = 8 + + +def _as_list(v) -> list: + if v is None: return [] - url = "https://www.rijksmuseum.nl/api/en/collection" - params = { - "key": settings.RIJKSMUSEUM_API_KEY, - "q": query, "ps": per_page, - "imgonly": "True", "toppieces": "True", - } - async with httpx.AsyncClient(timeout=15.0) as c: - r = await c.get(url, params=params) - r.raise_for_status() - j = r.json() - out = [] - for art in j.get("artObjects", []): - webimg = art.get("webImage") or {} - if not webimg.get("url"): - continue - out.append({ - "id": art["objectNumber"], - "url": webimg["url"], - "thumb": (art.get("headerImage") or webimg).get("url"), - "title": art.get("title"), - "credit": art.get("principalOrFirstMaker"), - "html": art.get("links", {}).get("web"), - }) - return out - - -async def get(object_number: str) -> dict | None: - if not settings.RIJKSMUSEUM_API_KEY: + return v if isinstance(v, list) else [v] + + +def _ids(items) -> set[str]: + return {i.get("id") for i in _as_list(items) if isinstance(i, dict)} + + +def _title(art: dict) -> str | None: + names = [n for n in _as_list(art.get("identified_by")) if n.get("type") == "Name"] + for n in names: + if AAT_ENGLISH in _ids(n.get("language")): + return n.get("content") + return names[0].get("content") if names else None + + +def _object_number(art: dict) -> str | None: + for n in _as_list(art.get("identified_by")): + if n.get("type") == "Identifier" and AAT_OBJECT_NUMBER in _ids(n.get("classified_as")): + return n.get("content") + return None + + +def _creator(art: dict) -> str | None: + for ref in _as_list((art.get("produced_by") or {}).get("referred_to_by")): + if ref.get("content"): + return ref["content"] + return None + + +def _web_link(art: dict) -> str | None: + for sub in _as_list(art.get("subject_of")): + for dig in _as_list(sub.get("digitally_carried_by")): + if dig.get("format") == "text/html": + for ap in _as_list(dig.get("access_point")): + if ap.get("id"): + return ap["id"] + return None + + +async def _resolve(client: httpx.AsyncClient, oid: str) -> dict | None: + r = await client.get(RESOLVE_URL.format(oid=oid), headers=HEADERS) + if r.status_code != 200: return None - url = f"https://www.rijksmuseum.nl/api/en/collection/{object_number}" - params = {"key": settings.RIJKSMUSEUM_API_KEY} - async with httpx.AsyncClient(timeout=15.0) as c: - r = await c.get(url, params=params) - if r.status_code != 200: - return None - j = r.json() - art = j.get("artObject") or {} - webimg = art.get("webImage") or {} - if not webimg.get("url"): + return r.json() + + +async def _image_url(client: httpx.AsyncClient, art: dict) -> str | None: + shows = _as_list(art.get("shows")) + if not shows or not shows[0].get("id"): return None + visual = await _resolve(client, shows[0]["id"].rsplit("/", 1)[-1]) + if not visual: + return None + shown_by = _as_list(visual.get("digitally_shown_by")) + if not shown_by or not shown_by[0].get("id"): + return None + digital = await _resolve(client, shown_by[0]["id"].rsplit("/", 1)[-1]) + if not digital: + return None + for ap in _as_list(digital.get("access_point")): + if ap.get("id"): + return ap["id"] + return None + + +async def _fetch_item(client: httpx.AsyncClient, sem: asyncio.Semaphore, oid: str) -> dict | None: + async with sem: + try: + art = await _resolve(client, oid) + if not art: + return None + url = await _image_url(client, art) + if not url: + return None + # IIIF Image API URL — swap "max" for a width to get a thumbnail + thumb = url.replace("/full/max/", "/full/400,/") + return { + "id": oid, + "url": url, + "thumb": thumb, + "title": _title(art), + "credit": _creator(art), + "html": _web_link(art), + } + except httpx.HTTPError: + return None + + +async def search(query: str, per_page: int = 20) -> list[dict]: + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as c: + # The new API has no free-text parameter; try the most useful fields + # in turn until one matches. + ids: list[str] = [] + for field in ("creator", "title", "description"): + r = await c.get(SEARCH_URL, params={field: query, "imageAvailable": "true"}) + r.raise_for_status() + items = _as_list(r.json().get("orderedItems")) + ids = [i["id"].rsplit("/", 1)[-1] for i in items if i.get("id")][:per_page] + if ids: + break + if not ids: + return [] + sem = asyncio.Semaphore(_CONCURRENCY) + results = await asyncio.gather(*(_fetch_item(c, sem, oid) for oid in ids)) + return [r for r in results if r] + + +async def get(object_id: str) -> dict | None: + oid = object_id.rstrip("/").rsplit("/", 1)[-1] + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as c: + try: + art = await _resolve(c, oid) + if not art: + return None + url = await _image_url(c, art) + if not url: + return None + except httpx.HTTPError: + return None return { - "id": art.get("objectNumber"), - "url": webimg["url"], - "title": art.get("title"), - "credit": art.get("principalOrFirstMaker"), - "html": (art.get("links") or {}).get("web"), + "id": _object_number(art) or oid, + "url": url, + "title": _title(art), + "credit": _creator(art), + "html": _web_link(art), } diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 94f7f61..3a1cda2 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -54,13 +54,12 @@ export default function Settings() {

- API keys (Unsplash, Rijksmuseum, NASA) and other defaults are configured via the + API keys (Unsplash, NASA) and other defaults are configured via the .env file in the backend directory. Edit and restart the backend to apply.

  • UNSPLASH_API_KEY
    • -
  • RIJKSMUSEUM_API_KEY
  • TV_RESOLUTION (4K | 1080p)
  • PORTRAIT_HANDLING (blur | crop | skip)
  • IMAGE_FOLDER (downloaded source images)
    • diff --git a/readme.md b/readme.md index 86c6816..b33a638 100644 --- a/readme.md +++ b/readme.md @@ -176,10 +176,9 @@ cp .env.example .env | `PEXELS_API_KEY` | [pexels.com/api](https://www.pexels.com/api/) — free | | `PIXABAY_API_KEY` | [pixabay.com/api/docs](https://pixabay.com/api/docs/) — free | | `OPENVERSE_CLIENT_ID` / `OPENVERSE_CLIENT_SECRET` | [api.openverse.org](https://api.openverse.org/) — free | -| `RIJKSMUSEUM_API_KEY` | [data.rijksmuseum.nl](https://data.rijksmuseum.nl/object-metadata/api/) — free | | `NASA_API_KEY` | [api.nasa.gov](https://api.nasa.gov) — free (optional; public key works with rate limits) | -Reddit requires no API key. +Reddit and the Rijksmuseum require no API key (the Rijksmuseum's [new Search API](https://data.rijksmuseum.nl/docs/search) is public). #### Radarrzen (movie manager on TV) From 02dd7b581c1b8e1a7b158b959b749877dcb86942 Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sat, 4 Jul 2026 18:23:20 -0700 Subject: [PATCH 2/7] feat: add pagination to Rijksmuseum source Search results now return {items, next} where next is an opaque continuation token (search field + API pageToken + offset within the 100-id API page). The Sources page shows a "Load more" button that appends the next page to the grid. Co-Authored-By: Claude Fable 5 --- backend/routers/sources.py | 4 +- backend/services/sources/rijksmuseum.py | 62 +++++++++++++++++++------ frontend/src/pages/Sources.tsx | 13 ++++-- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/backend/routers/sources.py b/backend/routers/sources.py index 8eb0480..64d294c 100644 --- a/backend/routers/sources.py +++ b/backend/routers/sources.py @@ -104,8 +104,8 @@ async def nasa_import(): # ── Rijksmuseum ──────────────────────────────────────────────────────────── @router.get("/rijksmuseum/search") -async def rijks_search(q: str = Query(...), per_page: int = 20): - return await rijksmuseum.search(q, per_page) +async def rijks_search(q: str = Query(...), per_page: int = 20, page_token: str = ""): + return await rijksmuseum.search(q, per_page, page_token or None) @router.post("/rijksmuseum/import", response_model=ImageOut) diff --git a/backend/services/sources/rijksmuseum.py b/backend/services/sources/rijksmuseum.py index bb13bab..0360702 100644 --- a/backend/services/sources/rijksmuseum.py +++ b/backend/services/sources/rijksmuseum.py @@ -1,5 +1,6 @@ from __future__ import annotations import asyncio +from urllib.parse import parse_qs, urlparse import httpx # Rijksmuseum's new Search API (https://data.rijksmuseum.nl/docs/search) is @@ -107,23 +108,56 @@ async def _fetch_item(client: httpx.AsyncClient, sem: asyncio.Semaphore, oid: st return None -async def search(query: str, per_page: int = 20) -> list[dict]: +async def _search_page(c: httpx.AsyncClient, field: str, query: str, api_token: str) -> tuple[list[str], str]: + params = {field: query, "imageAvailable": "true"} + if api_token: + params["pageToken"] = api_token + r = await c.get(SEARCH_URL, params=params) + r.raise_for_status() + j = r.json() + ids = [i["id"].rsplit("/", 1)[-1] for i in _as_list(j.get("orderedItems")) if i.get("id")] + next_url = (j.get("next") or {}).get("id", "") + next_token = (parse_qs(urlparse(next_url).query).get("pageToken") or [""])[0] + return ids, next_token + + +async def search(query: str, per_page: int = 20, page_token: str | None = None) -> dict: + """Returns {"items": [...], "next": token-or-None}. + + Continuation tokens are "||": API pages + hold up to 100 ids while only per_page are resolved per call, so the + offset tracks the position within the current API page. + """ async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as c: - # The new API has no free-text parameter; try the most useful fields - # in turn until one matches. + field, api_token, offset = "", "", 0 ids: list[str] = [] - for field in ("creator", "title", "description"): - r = await c.get(SEARCH_URL, params={field: query, "imageAvailable": "true"}) - r.raise_for_status() - items = _as_list(r.json().get("orderedItems")) - ids = [i["id"].rsplit("/", 1)[-1] for i in items if i.get("id")][:per_page] - if ids: - break - if not ids: - return [] + next_api_token = "" + if page_token: + try: + field, api_token, off = page_token.split("|", 2) + offset = int(off) + except ValueError: + return {"items": [], "next": None} + ids, next_api_token = await _search_page(c, field, query, api_token) + else: + # The new API has no free-text parameter; try the most useful + # fields in turn until one matches. + for field in ("creator", "title", "description"): + ids, next_api_token = await _search_page(c, field, query, "") + if ids: + break + page_ids = ids[offset:offset + per_page] + if not page_ids: + return {"items": [], "next": None} + if offset + per_page < len(ids): + next_token = f"{field}|{api_token}|{offset + per_page}" + elif next_api_token: + next_token = f"{field}|{next_api_token}|0" + else: + next_token = None sem = asyncio.Semaphore(_CONCURRENCY) - results = await asyncio.gather(*(_fetch_item(c, sem, oid) for oid in ids)) - return [r for r in results if r] + results = await asyncio.gather(*(_fetch_item(c, sem, oid) for oid in page_ids)) + return {"items": [r for r in results if r], "next": next_token} async def get(object_id: str) -> dict | None: diff --git a/frontend/src/pages/Sources.tsx b/frontend/src/pages/Sources.tsx index 231a394..14b25a3 100644 --- a/frontend/src/pages/Sources.tsx +++ b/frontend/src/pages/Sources.tsx @@ -89,21 +89,28 @@ function Nasa() { function Rijks() { const [q, setQ] = useState('vermeer') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const search = async () => { - try { setItems(await api.get(`/api/sources/rijksmuseum/search?q=${encodeURIComponent(q)}`)) } + const search = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/rijksmuseum/search?q=${encodeURIComponent(q)}${token ? `&page_token=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return (
    setQ(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && search()} /> - +
    { try { await api.post('/api/sources/rijksmuseum/import', { id: it.id }); t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && }
    ) } From 875766b4f313d2c243533650055b55ebb58767ec Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sat, 4 Jul 2026 18:35:33 -0700 Subject: [PATCH 3/7] feat: add pagination to all search sources Unsplash, Pexels, Pixabay, and Openverse searches take a page number; Reddit and Reddit Gallery use Reddit's native after cursor. All search endpoints now return {items, next} like the Rijksmuseum source, and every source tab gets a "Load more" button that appends the next page. Also updates the source tests for the new shape and repairs the Reddit tests, which still patched httpx.AsyncClient after the service moved to urllib (they now patch _fetch_reddit_json), and pins the Unsplash router test's API key so it doesn't depend on the local .env. Co-Authored-By: Claude Fable 5 --- backend/routers/sources.py | 23 ++-- backend/services/sources/openverse.py | 8 +- backend/services/sources/pexels.py | 10 +- backend/services/sources/pixabay.py | 14 ++- backend/services/sources/reddit.py | 12 +- backend/services/sources/reddit_gallery.py | 12 +- backend/services/sources/unsplash.py | 10 +- backend/tests/test_router_sources.py | 9 +- backend/tests/test_sources_pexels.py | 16 +-- backend/tests/test_sources_pixabay.py | 10 +- backend/tests/test_sources_reddit.py | 129 +++++++++------------ frontend/src/pages/Sources.tsx | 76 +++++++++--- 12 files changed, 189 insertions(+), 140 deletions(-) diff --git a/backend/routers/sources.py b/backend/routers/sources.py index 64d294c..23aad2e 100644 --- a/backend/routers/sources.py +++ b/backend/routers/sources.py @@ -57,10 +57,10 @@ async def scan(fid: int, s: AsyncSession = Depends(get_session)): # ── Unsplash ─────────────────────────────────────────────────────────────── @router.get("/unsplash/search") -async def unsplash_search(q: str = Query(...), per_page: int = 20): +async def unsplash_search(q: str = Query(...), per_page: int = 20, page: int = 1): if not settings.UNSPLASH_API_KEY: raise HTTPException(503, "UNSPLASH_API_KEY not configured — add it to .env") - return await unsplash.search(q, per_page) + return await unsplash.search(q, per_page, page) @router.post("/unsplash/import", response_model=ImageOut) @@ -126,10 +126,10 @@ async def rijks_import(payload: ImportPayload): # ── Pexels ──────────────────────────────────────────────────────────────── @router.get("/pexels/search") -async def pexels_search(q: str = Query(...), per_page: int = 20): +async def pexels_search(q: str = Query(...), per_page: int = 20, page: int = 1): if not settings.PEXELS_API_KEY: raise HTTPException(503, "PEXELS_API_KEY not configured — add it to .env") - return await pexels.search(q, per_page) + return await pexels.search(q, per_page, page) @router.post("/pexels/import", response_model=ImageOut) @@ -151,10 +151,10 @@ async def pexels_import(payload: ImportPayload): # ── Pixabay ────────────────────────────────────────────────────────────── @router.get("/pixabay/search") -async def pixabay_search(q: str = Query(...), per_page: int = 20): +async def pixabay_search(q: str = Query(...), per_page: int = 20, page: int = 1): if not settings.PIXABAY_API_KEY: raise HTTPException(503, "PIXABAY_API_KEY not configured — add it to .env") - return await pixabay.search(q, per_page) + return await pixabay.search(q, per_page, page) @router.post("/pixabay/import", response_model=ImageOut) @@ -176,8 +176,8 @@ async def pixabay_import(payload: ImportPayload): # ── Reddit ───────────────────────────────────────────────────────────────── @router.get("/reddit/fetch") -async def reddit_fetch(sub: str = Query(...), sort: str = "top", t: str = "week", limit: int = 20): - return await reddit.fetch(sub, sort, t, limit) +async def reddit_fetch(sub: str = Query(...), sort: str = "top", t: str = "week", limit: int = 20, after: str = ""): + return await reddit.fetch(sub, sort, t, limit, after) @router.post("/reddit/import", response_model=ImageOut) @@ -195,8 +195,8 @@ async def reddit_import(payload: ImportPayload): # ── Reddit Galleries ──────────────────────────────────────────────────────── @router.get("/reddit-gallery/fetch") -async def reddit_gallery_fetch(sub: str = Query(...), sort: str = "top", t: str = "week", limit: int = 25): - return await reddit_gallery.fetch(sub, sort, t, limit) +async def reddit_gallery_fetch(sub: str = Query(...), sort: str = "top", t: str = "week", limit: int = 25, after: str = ""): + return await reddit_gallery.fetch(sub, sort, t, limit, after) @router.post("/reddit-gallery/import", response_model=ImageOut) @@ -222,8 +222,9 @@ async def openverse_search( license_type: str = "", aspect_ratio: str = "wide", size: str = "large", + page: int = 1, ): - return await openverse.search(q, page_size, category, license_type, aspect_ratio, size) + return await openverse.search(q, page_size, category, license_type, aspect_ratio, size, page) @router.post("/openverse/import", response_model=ImageOut) diff --git a/backend/services/sources/openverse.py b/backend/services/sources/openverse.py index da49ceb..3ee5652 100644 --- a/backend/services/sources/openverse.py +++ b/backend/services/sources/openverse.py @@ -67,10 +67,13 @@ async def search( license_type: str = "", aspect_ratio: str = "wide", size: str = "large", -) -> list[dict]: + page: int = 1, +) -> dict: + page = max(1, int(page)) params: dict = { "q": q, "page_size": min(max(int(page_size), 1), 100), + "page": page, "mature": "false", "filter_dead": "true", } @@ -108,7 +111,8 @@ async def search( "width": item.get("width"), "height": item.get("height"), }) - return out + has_more = out and page < int(j.get("page_count") or 0) + return {"items": out, "next": str(page + 1) if has_more else None} async def get(image_id: str) -> dict | None: diff --git a/backend/services/sources/pexels.py b/backend/services/sources/pexels.py index 3a407cc..0d1827b 100644 --- a/backend/services/sources/pexels.py +++ b/backend/services/sources/pexels.py @@ -25,15 +25,17 @@ def _normalise(p: dict) -> dict: } -async def search(query: str, per_page: int = 20) -> list[dict]: +async def search(query: str, per_page: int = 20, page: int = 1) -> dict: if not settings.PEXELS_API_KEY: - return [] - params = {"query": query, "per_page": min(int(per_page), 80), "orientation": "landscape"} + return {"items": [], "next": None} + page = max(1, int(page)) + params = {"query": query, "per_page": min(int(per_page), 80), "page": page, "orientation": "landscape"} async with httpx.AsyncClient(timeout=15.0) as c: r = await c.get(f"{_BASE}/search", params=params, headers=_headers()) r.raise_for_status() j = r.json() - return [_normalise(p) for p in j.get("photos", [])] + items = [_normalise(p) for p in j.get("photos", [])] + return {"items": items, "next": str(page + 1) if items and j.get("next_page") else None} async def get(photo_id: str) -> dict | None: diff --git a/backend/services/sources/pixabay.py b/backend/services/sources/pixabay.py index dfb2293..56cb2a5 100644 --- a/backend/services/sources/pixabay.py +++ b/backend/services/sources/pixabay.py @@ -33,23 +33,29 @@ def _normalise(p: dict) -> dict: } -async def search(query: str, per_page: int = 20) -> list[dict]: +async def search(query: str, per_page: int = 20, page: int = 1) -> dict: if not settings.PIXABAY_API_KEY: - return [] + return {"items": [], "next": None} + page = max(1, int(page)) + per_page = max(3, min(int(per_page), 200)) params = { "key": settings.PIXABAY_API_KEY, "q": query, "image_type": "photo", "orientation": "horizontal", "safesearch": "true", - "per_page": max(3, min(int(per_page), 200)), + "per_page": per_page, + "page": page, "order": "popular", } async with httpx.AsyncClient(timeout=15.0) as c: r = await c.get(_BASE, params=params) r.raise_for_status() j = r.json() - return [_normalise(p) for p in j.get("hits", [])] + items = [_normalise(p) for p in j.get("hits", [])] + # totalHits is how many results the API will actually serve (max 500) + has_more = items and page * per_page < int(j.get("totalHits") or 0) + return {"items": items, "next": str(page + 1) if has_more else None} async def get(photo_id: str) -> dict | None: diff --git a/backend/services/sources/reddit.py b/backend/services/sources/reddit.py index 032b46f..a63446e 100644 --- a/backend/services/sources/reddit.py +++ b/backend/services/sources/reddit.py @@ -26,10 +26,10 @@ def _fetch_reddit_json(url: str, params: dict, user_agent: str) -> dict: return _json.loads(resp.read().decode()) -async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) -> list[dict]: +async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20, after: str = "") -> dict: global _last_call if not _SUB_RE.match(sub or ""): - return [] + return {"items": [], "next": None} if sort not in {"top", "hot", "new", "rising", "controversial"}: sort = "top" if t not in {"hour", "day", "week", "month", "year", "all"}: @@ -42,6 +42,8 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - await asyncio.sleep(delay) url = f"https://www.reddit.com/r/{sub}/{sort}.json" params = {"limit": limit, "t": t} + if after: + params["after"] = after user_agent = settings.REDDIT_USER_AGENT try: j = await loop.run_in_executor( @@ -49,10 +51,10 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - ) except urllib.error.HTTPError as exc: log.warning("Reddit fetch blocked (HTTP %s): %s", exc.code, url) - return [] + return {"items": [], "next": None} except Exception as exc: log.warning("Reddit fetch error: %s", exc) - return [] + return {"items": [], "next": None} _last_call = loop.time() out = [] for child in (j.get("data") or {}).get("children", []): @@ -71,4 +73,4 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - "html": "https://www.reddit.com" + d.get("permalink", ""), "subreddit": d.get("subreddit"), }) - return out + return {"items": out, "next": (j.get("data") or {}).get("after")} diff --git a/backend/services/sources/reddit_gallery.py b/backend/services/sources/reddit_gallery.py index 6a831be..513acda 100644 --- a/backend/services/sources/reddit_gallery.py +++ b/backend/services/sources/reddit_gallery.py @@ -85,11 +85,11 @@ def _extract_gallery_images(post: dict) -> list[dict]: return out -async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) -> list[dict]: +async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20, after: str = "") -> dict: """Fetch gallery posts from a subreddit. Returns flat list of individual images.""" global _last_call if not _SUB_RE.match(sub or ""): - return [] + return {"items": [], "next": None} if sort not in {"top", "hot", "new", "rising", "controversial"}: sort = "top" if t not in {"hour", "day", "week", "month", "year", "all"}: @@ -104,6 +104,8 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - await asyncio.sleep(delay) url = f"https://www.reddit.com/r/{sub}/{sort}.json" params = {"limit": post_limit, "t": t} + if after: + params["after"] = after user_agent = settings.REDDIT_USER_AGENT try: j = await loop.run_in_executor( @@ -111,10 +113,10 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - ) except urllib.error.HTTPError as exc: log.warning("Reddit gallery fetch blocked (HTTP %s): %s", exc.code, url) - return [] + return {"items": [], "next": None} except Exception as exc: log.warning("Reddit gallery fetch error: %s", exc) - return [] + return {"items": [], "next": None} _last_call = loop.time() out: list[dict] = [] @@ -125,4 +127,4 @@ async def fetch(sub: str, sort: str = "top", t: str = "week", limit: int = 20) - continue out.extend(_extract_gallery_images(d)) - return out + return {"items": out, "next": (j.get("data") or {}).get("after")} diff --git a/backend/services/sources/unsplash.py b/backend/services/sources/unsplash.py index 43719fc..4ddb1fc 100644 --- a/backend/services/sources/unsplash.py +++ b/backend/services/sources/unsplash.py @@ -3,17 +3,17 @@ from ...config import settings -async def search(query: str, per_page: int = 20) -> list[dict]: +async def search(query: str, per_page: int = 20, page: int = 1) -> dict: if not settings.UNSPLASH_API_KEY: - return [] + return {"items": [], "next": None} url = "https://api.unsplash.com/search/photos" - params = {"query": query, "per_page": per_page, "orientation": "landscape"} + params = {"query": query, "per_page": per_page, "page": max(1, int(page)), "orientation": "landscape"} headers = {"Authorization": f"Client-ID {settings.UNSPLASH_API_KEY}"} async with httpx.AsyncClient(timeout=15.0) as c: r = await c.get(url, params=params, headers=headers) r.raise_for_status() j = r.json() - return [ + items = [ { "id": p["id"], "url": p["urls"]["full"], @@ -27,6 +27,8 @@ async def search(query: str, per_page: int = 20) -> list[dict]: } for p in j.get("results", []) ] + has_more = items and params["page"] < int(j.get("total_pages") or 0) + return {"items": items, "next": str(params["page"] + 1) if has_more else None} async def get(photo_id: str) -> dict | None: diff --git a/backend/tests/test_router_sources.py b/backend/tests/test_router_sources.py index 7979468..b278cd9 100644 --- a/backend/tests/test_router_sources.py +++ b/backend/tests/test_router_sources.py @@ -33,8 +33,9 @@ async def test_delete_folder_404(app_client): async def test_unsplash_search_mocked(app_client): - fake = [{"id": "x", "url": "https://example.com/x.jpg"}] - with patch("backend.routers.sources.unsplash.search", + fake = {"items": [{"id": "x", "url": "https://example.com/x.jpg"}], "next": None} + with patch("backend.routers.sources.settings.UNSPLASH_API_KEY", "testkey"), \ + patch("backend.routers.sources.unsplash.search", new=AsyncMock(return_value=fake)): r = await app_client.get("/api/sources/unsplash/search?q=mountain") assert r.json() == fake @@ -54,10 +55,10 @@ async def test_nasa_apod_no_image(app_client): async def test_reddit_fetch_mocked(app_client): with patch("backend.routers.sources.reddit.fetch", - new=AsyncMock(return_value=[{"id": "abc", "url": "https://i.redd.it/x.jpg"}])): + new=AsyncMock(return_value={"items": [{"id": "abc", "url": "https://i.redd.it/x.jpg"}], "next": None})): r = await app_client.get("/api/sources/reddit/fetch?sub=earthporn&sort=top&t=week") assert r.status_code == 200 - assert r.json()[0]["id"] == "abc" + assert r.json()["items"][0]["id"] == "abc" async def test_reddit_import_requires_url(app_client): diff --git a/backend/tests/test_sources_pexels.py b/backend/tests/test_sources_pexels.py index efd91d3..e76f581 100644 --- a/backend/tests/test_sources_pexels.py +++ b/backend/tests/test_sources_pexels.py @@ -64,7 +64,7 @@ async def test_search_empty_when_no_api_key(tmp_workdir, monkeypatch): importlib.reload(cfg_mod) importlib.reload(mod) result = await mod.search("landscape") - assert result == [] + assert result == {"items": [], "next": None} async def test_get_none_when_no_api_key(tmp_workdir, monkeypatch): @@ -86,9 +86,11 @@ async def test_search_returns_normalised_list(tmp_workdir, monkeypatch): payload = {"photos": [_FAKE_PHOTO], "total_results": 1, "page": 1, "per_page": 1} with patch("backend.services.sources.pexels.httpx.AsyncClient", _fake_client(payload)): - results = await mod.search("landscape", per_page=1) + out = await mod.search("landscape", per_page=1) + results = out["items"] assert len(results) == 1 + assert out["next"] is None # no next_page in payload r = results[0] assert r["id"] == "123456" assert r["url"] == _FAKE_PHOTO["src"]["original"] @@ -109,7 +111,7 @@ async def test_search_empty_photos_key(tmp_workdir, monkeypatch): payload = {"photos": [], "total_results": 0, "page": 1, "per_page": 1} with patch("backend.services.sources.pexels.httpx.AsyncClient", _fake_client(payload)): results = await mod.search("nothing") - assert results == [] + assert results == {"items": [], "next": None} async def test_get_returns_normalised_dict(tmp_workdir, monkeypatch): @@ -185,9 +187,9 @@ async def test_router_pexels_search_mocked(tmp_workdir, monkeypatch): import importlib, backend.config as cfg_mod importlib.reload(cfg_mod) - mock_results = [{"id": "1", "url": "http://example.com/img.jpg", "thumb": "http://example.com/t.jpg", + mock_results = {"items": [{"id": "1", "url": "http://example.com/img.jpg", "thumb": "http://example.com/t.jpg", "title": "Test", "credit": "Tester", "credit_url": None, "html": "http://pexels.com/photo/1", - "width": 1920, "height": 1080}] + "width": 1920, "height": 1080}], "next": None} with patch("backend.services.sources.pexels.search", return_value=mock_results): import backend.main as main_mod importlib.reload(main_mod) @@ -196,7 +198,7 @@ async def test_router_pexels_search_mocked(tmp_workdir, monkeypatch): r = await c.get("/api/sources/pexels/search?q=landscape") assert r.status_code == 200 data = r.json() - assert isinstance(data, list) + assert isinstance(data["items"], list) # ── live integration test (only runs when real key in env) ───────────────── @@ -212,7 +214,7 @@ async def test_live_pexels_search(tmp_workdir, monkeypatch): import backend.services.sources.pexels as mod importlib.reload(mod) - results = await mod.search("mountain landscape", per_page=5) + results = (await mod.search("mountain landscape", per_page=5))["items"] assert len(results) > 0 r = results[0] for key in ("id", "url", "thumb", "title", "credit"): diff --git a/backend/tests/test_sources_pixabay.py b/backend/tests/test_sources_pixabay.py index e344f01..be208dc 100644 --- a/backend/tests/test_sources_pixabay.py +++ b/backend/tests/test_sources_pixabay.py @@ -66,7 +66,7 @@ def _reload(monkeypatch, key: str = "testkey"): async def test_search_empty_when_no_key(tmp_workdir, monkeypatch): mod = _reload(monkeypatch, key="") result = await mod.search("landscape") - assert result == [] + assert result == {"items": [], "next": None} async def test_get_none_when_no_key(tmp_workdir, monkeypatch): @@ -79,9 +79,11 @@ async def test_search_returns_normalised_list(tmp_workdir, monkeypatch): mod = _reload(monkeypatch) payload = {"total": 1, "totalHits": 1, "hits": [_FAKE_HIT]} with patch("backend.services.sources.pixabay.httpx.AsyncClient", _fake_client(payload)): - results = await mod.search("flower", per_page=1) + out = await mod.search("flower", per_page=1) + results = out["items"] assert len(results) == 1 + assert out["next"] is None # totalHits=1 already covered by page 1 r = results[0] assert r["id"] == "195893" assert r["url"] == _FAKE_HIT["largeImageURL"] @@ -97,7 +99,7 @@ async def test_search_empty_hits(tmp_workdir, monkeypatch): payload = {"total": 0, "totalHits": 0, "hits": []} with patch("backend.services.sources.pixabay.httpx.AsyncClient", _fake_client(payload)): results = await mod.search("nothing here xyz") - assert results == [] + assert results == {"items": [], "next": None} async def test_get_returns_normalised_dict(tmp_workdir, monkeypatch): @@ -165,7 +167,7 @@ async def test_live_pixabay_search(tmp_workdir, monkeypatch): import backend.services.sources.pixabay as mod importlib.reload(mod) - results = await mod.search("mountain landscape", per_page=5) + results = (await mod.search("mountain landscape", per_page=5))["items"] assert len(results) > 0 r = results[0] for key in ("id", "url", "thumb", "title", "credit"): diff --git a/backend/tests/test_sources_reddit.py b/backend/tests/test_sources_reddit.py index 0491542..d55def8 100644 --- a/backend/tests/test_sources_reddit.py +++ b/backend/tests/test_sources_reddit.py @@ -1,37 +1,37 @@ """Reddit source validation tests (SSRF guards, whitelist enforcement).""" from __future__ import annotations import pytest -from unittest.mock import AsyncMock, patch +from unittest.mock import patch + +_EMPTY = {"items": [], "next": None} + + +def _fake_fetcher(response_json: dict, captured: dict | None = None): + """Stand-in for reddit._fetch_reddit_json (sync, run via executor).""" + def fetcher(url: str, params: dict, user_agent: str) -> dict: + if captured is not None: + captured["url"] = url + captured["params"] = params + return response_json + return fetcher async def test_invalid_subreddit_returns_empty(tmp_workdir): from backend.services.sources import reddit - assert await reddit.fetch("../../etc/passwd") == [] - assert await reddit.fetch("has spaces") == [] - assert await reddit.fetch("") == [] - assert await reddit.fetch("a" * 100) == [] + assert await reddit.fetch("../../etc/passwd") == _EMPTY + assert await reddit.fetch("has spaces") == _EMPTY + assert await reddit.fetch("") == _EMPTY + assert await reddit.fetch("a" * 100) == _EMPTY async def test_valid_subreddit_normalises_sort_and_t(tmp_workdir): from backend.services.sources import reddit - captured = {} - - class FakeResponse: - def raise_for_status(self): pass - def json(self): return {"data": {"children": []}} - - class FakeClient: - def __init__(self, *a, **kw): pass - async def __aenter__(self): return self - async def __aexit__(self, *a): pass - async def get(self, url, params=None): - captured["url"] = url - captured["params"] = params - return FakeResponse() + captured: dict = {} - with patch("backend.services.sources.reddit.httpx.AsyncClient", FakeClient): + with patch("backend.services.sources.reddit._fetch_reddit_json", + _fake_fetcher({"data": {"children": []}}, captured)): out = await reddit.fetch("earthporn", sort="evilsort", t="evilt", limit=999) - assert out == [] + assert out == _EMPTY assert "earthporn" in captured["url"] # invalid sort defaulted in URL "/top.json" assert "/top.json" in captured["url"] @@ -42,67 +42,52 @@ async def get(self, url, params=None): async def test_limit_clamped_low(tmp_workdir): """limit < 1 clamps to 1.""" from backend.services.sources import reddit - captured = {} + captured: dict = {} - class FakeResponse: - def raise_for_status(self): pass - def json(self): return {"data": {"children": []}} - - class FakeClient: - def __init__(self, *a, **kw): pass - async def __aenter__(self): return self - async def __aexit__(self, *a): pass - async def get(self, url, params=None): - captured["params"] = params - return FakeResponse() - - with patch("backend.services.sources.reddit.httpx.AsyncClient", FakeClient): + with patch("backend.services.sources.reddit._fetch_reddit_json", + _fake_fetcher({"data": {"children": []}}, captured)): await reddit.fetch("aww", limit=-5) assert captured["params"]["limit"] == 1 -async def test_preview_urls_are_html_unescaped(tmp_workdir): +async def test_after_param_passed_and_next_returned(tmp_workdir): from backend.services.sources import reddit + captured: dict = {} - class FakeResponse: - def raise_for_status(self): - pass - - def json(self): - return { - "data": { - "children": [ - { - "data": { - "id": "abc", - "post_hint": "image", - "url": "https://i.redd.it/test.jpeg", - "thumbnail": "https://preview.redd.it/test.jpeg?width=140&height=93&auto=webp", - "title": "Example", - "author": "user", - "permalink": "/r/pics/comments/abc/example/", - "subreddit": "pics", - } - } - ] - } - } + with patch("backend.services.sources.reddit._fetch_reddit_json", + _fake_fetcher({"data": {"children": [], "after": "t3_xyz"}}, captured)): + out = await reddit.fetch("pics", after="t3_abc") + assert captured["params"]["after"] == "t3_abc" + assert out["next"] == "t3_xyz" - class FakeClient: - def __init__(self, *a, **kw): - pass - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - pass +async def test_preview_urls_are_html_unescaped(tmp_workdir): + from backend.services.sources import reddit - async def get(self, url, params=None): - return FakeResponse() + payload = { + "data": { + "children": [ + { + "data": { + "id": "abc", + "post_hint": "image", + "url": "https://i.redd.it/test.jpeg", + "thumbnail": "https://preview.redd.it/test.jpeg?width=140&height=93&auto=webp", + "title": "Example", + "author": "user", + "permalink": "/r/pics/comments/abc/example/", + "subreddit": "pics", + } + } + ], + "after": None, + } + } - with patch("backend.services.sources.reddit.httpx.AsyncClient", FakeClient): + with patch("backend.services.sources.reddit._fetch_reddit_json", _fake_fetcher(payload)): out = await reddit.fetch("pics") - assert out[0]["url"] == "https://i.redd.it/test.jpeg" - assert out[0]["thumb"] == "https://preview.redd.it/test.jpeg?width=140&height=93&auto=webp" + items = out["items"] + assert items[0]["url"] == "https://i.redd.it/test.jpeg" + assert items[0]["thumb"] == "https://preview.redd.it/test.jpeg?width=140&height=93&auto=webp" + assert out["next"] is None diff --git a/frontend/src/pages/Sources.tsx b/frontend/src/pages/Sources.tsx index 14b25a3..3decf23 100644 --- a/frontend/src/pages/Sources.tsx +++ b/frontend/src/pages/Sources.tsx @@ -48,21 +48,28 @@ function Grid({ items, onImport }: { items: any[]; onImport: (it: any) => void } function Unsplash() { const [q, setQ] = useState('landscape') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const search = async () => { - try { setItems(await api.get(`/api/sources/unsplash/search?q=${encodeURIComponent(q)}`)) } + const search = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/unsplash/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return (
    setQ(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && search()} /> - +
    { try { await api.post('/api/sources/unsplash/import', { id: it.id }); t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && }
    ) } @@ -118,21 +125,28 @@ function Rijks() { function Pixabay() { const [q, setQ] = useState('landscape') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const search = async () => { - try { setItems(await api.get(`/api/sources/pixabay/search?q=${encodeURIComponent(q)}`)) } + const search = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/pixabay/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return (
    setQ(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && search()} placeholder="Search Pixabay photos…" /> - +
    { try { await api.post('/api/sources/pixabay/import', { id: it.id }); t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && }
    ) } @@ -140,21 +154,28 @@ function Pixabay() { function Pexels() { const [q, setQ] = useState('landscape') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const search = async () => { - try { setItems(await api.get(`/api/sources/pexels/search?q=${encodeURIComponent(q)}`)) } + const search = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/pexels/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return (
    setQ(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && search()} placeholder="Search Pexels photos…" /> - +
    { try { await api.post('/api/sources/pexels/import', { id: it.id }); t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && }
    ) } @@ -164,9 +185,15 @@ function Reddit() { const [sort, setSort] = useState('top') const [tt, setTt] = useState('week') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const fetchIt = async () => { - try { setItems(await api.get(`/api/sources/reddit/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=24`)) } + const fetchIt = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/reddit/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=24${token ? `&after=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return ( @@ -179,7 +206,7 @@ function Reddit() { - + { try { @@ -190,6 +217,7 @@ function Reddit() { t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && } ) } @@ -201,15 +229,19 @@ function Openverse() { const [aspectRatio, setAspectRatio] = useState('wide') const [size, setSize] = useState('large') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const search = async () => { + const search = async (token?: string | null) => { try { const params = new URLSearchParams({ q, page_size: '24' }) if (category) params.set('category', category) if (licenseType) params.set('license_type', licenseType) if (aspectRatio) params.set('aspect_ratio', aspectRatio) if (size) params.set('size', size) - setItems(await api.get(`/api/sources/openverse/search?${params}`)) + if (token) params.set('page', token) + const r = await api.get<{ items: any[]; next: string | null }>(`/api/sources/openverse/search?${params}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return ( @@ -241,7 +273,7 @@ function Openverse() { - + { try { @@ -249,6 +281,7 @@ function Openverse() { t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && } ) } @@ -258,9 +291,15 @@ function RedditGallery() { const [sort, setSort] = useState('top') const [tt, setTt] = useState('week') const [items, setItems] = useState([]) + const [next, setNext] = useState(null) const t = useToast() - const fetchIt = async () => { - try { setItems(await api.get(`/api/sources/reddit-gallery/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=25`)) } + const fetchIt = async (token?: string | null) => { + try { + const r = await api.get<{ items: any[]; next: string | null }>( + `/api/sources/reddit-gallery/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=25${token ? `&after=${encodeURIComponent(token)}` : ''}`) + setItems(token ? [...items, ...r.items] : r.items) + setNext(r.next) + } catch (e: any) { t.push({ type: 'error', text: e.message }) } } return ( @@ -274,7 +313,7 @@ function RedditGallery() { - + { try { @@ -285,6 +324,7 @@ function RedditGallery() { t.push({ type: 'success', text: 'Imported' }) } catch (e: any) { t.push({ type: 'error', text: e.message }) } }} /> + {next && } ) } From ae63776eb99c634550573ab50df52ca8716afaff Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sat, 4 Jul 2026 18:39:44 -0700 Subject: [PATCH 4/7] fix: dedupe items when appending search pages Pexels (and potentially others) repeat items across pages, which would produce duplicate React keys in the grid; skip items already shown. Co-Authored-By: Claude Fable 5 --- frontend/src/pages/Sources.tsx | 20 +++++++++++++------- frontend/tsconfig.tsbuildinfo | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/frontend/src/pages/Sources.tsx b/frontend/src/pages/Sources.tsx index 3decf23..f16de25 100644 --- a/frontend/src/pages/Sources.tsx +++ b/frontend/src/pages/Sources.tsx @@ -28,6 +28,12 @@ export default function Sources() { ) } +// Some APIs (e.g. Pexels) repeat items across pages; skip ones already shown +function appendUnique(prev: any[], more: any[]) { + const seen = new Set(prev.map((it) => it.id || it.url)) + return [...prev, ...more.filter((it) => !seen.has(it.id || it.url))] +} + function Grid({ items, onImport }: { items: any[]; onImport: (it: any) => void }) { return (
    @@ -54,7 +60,7 @@ function Unsplash() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/unsplash/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } @@ -102,7 +108,7 @@ function Rijks() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/rijksmuseum/search?q=${encodeURIComponent(q)}${token ? `&page_token=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } @@ -131,7 +137,7 @@ function Pixabay() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/pixabay/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } @@ -160,7 +166,7 @@ function Pexels() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/pexels/search?q=${encodeURIComponent(q)}${token ? `&page=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } @@ -191,7 +197,7 @@ function Reddit() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/reddit/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=24${token ? `&after=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } @@ -240,7 +246,7 @@ function Openverse() { if (size) params.set('size', size) if (token) params.set('page', token) const r = await api.get<{ items: any[]; next: string | null }>(`/api/sources/openverse/search?${params}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } } @@ -297,7 +303,7 @@ function RedditGallery() { try { const r = await api.get<{ items: any[]; next: string | null }>( `/api/sources/reddit-gallery/fetch?sub=${sub}&sort=${sort}&t=${tt}&limit=25${token ? `&after=${encodeURIComponent(token)}` : ''}`) - setItems(token ? [...items, ...r.items] : r.items) + setItems(token ? appendUnique(items, r.items) : r.items) setNext(r.next) } catch (e: any) { t.push({ type: 'error', text: e.message }) } diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index 961081a..b777716 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/App.tsx","./src/main.tsx","./src/test-setup.ts","./src/components/Sidebar.test.tsx","./src/components/Sidebar.tsx","./src/components/Toast.test.tsx","./src/components/Toast.tsx","./src/lib/api.test.ts","./src/lib/api.ts","./src/lib/hooks.test.tsx","./src/lib/hooks.ts","./src/lib/ws.test.ts","./src/lib/ws.ts","./src/pages/Dashboard.tsx","./src/pages/Discover.tsx","./src/pages/Library.tsx","./src/pages/Schedules.tsx","./src/pages/Settings.tsx","./src/pages/Sources.tsx","./src/pages/TVControl.tsx","./src/pages/TizenBrew.tsx"],"version":"5.9.3"} \ No newline at end of file +{"root":["./src/app.tsx","./src/main.tsx","./src/test-setup.ts","./src/components/sidebar.test.tsx","./src/components/sidebar.tsx","./src/components/toast.test.tsx","./src/components/toast.tsx","./src/lib/api.test.ts","./src/lib/api.ts","./src/lib/hooks.test.tsx","./src/lib/hooks.ts","./src/lib/ws.test.ts","./src/lib/ws.ts","./src/pages/dashboard.tsx","./src/pages/debloat.tsx","./src/pages/discover.tsx","./src/pages/library.tsx","./src/pages/schedules.tsx","./src/pages/settings.tsx","./src/pages/sources.tsx","./src/pages/tvcontrol.tsx","./src/pages/tizenbrew.tsx"],"version":"5.9.3"} \ No newline at end of file From 406a61875a66ab96a692069a62d7619dca6dfecb Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sat, 4 Jul 2026 22:25:49 -0700 Subject: [PATCH 5/7] fix: graceful TV/upstream failures on the TV page - list_mattes: the samsungtvws matte list can contain dicts like {"matte_type": ...}; returning them raw crashed the TV page (React error #31). Normalise to strings. - remove_from_tv: re-sending an image after removal leaves multiple TVImage rows per (tv, image); scalar_one_or_none() then raised MultipleResultsFound (500). Handle all rows, and return 503 instead of marking the image removed when the TV is unreachable. - tv thumbnails: cache each successfully fetched thumbnail on disk and serve the cached copy when the TV is asleep/unreachable, instead of 404ing the whole grid. - NASA APOD: catch httpx errors (timeouts to api.nasa.gov returned 500). Co-Authored-By: Claude Fable 5 --- backend/routers/images.py | 38 ++++++++++++++++++++------- backend/services/sources/nasa_apod.py | 13 +++++---- backend/services/tv_manager.py | 11 +++++--- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/backend/routers/images.py b/backend/routers/images.py index ecd9c3d..a07f47d 100644 --- a/backend/routers/images.py +++ b/backend/routers/images.py @@ -214,7 +214,7 @@ async def send_to_tv(image_id: int, tv_id: int, display: bool = True, await s.commit() existing = (await s.execute( select(TVImage).where(TVImage.tv_id == tv_id, TVImage.image_id == image_id, TVImage.is_on_tv.is_(True)) - )).scalar_one_or_none() + )).scalars().first() if existing and existing.remote_id: ti = existing else: @@ -236,14 +236,17 @@ async def send_to_tv(image_id: int, tv_id: int, display: bool = True, @router.delete("/{image_id}/tv/{tv_id}") async def remove_from_tv(image_id: int, tv_id: int, s: AsyncSession = Depends(get_session)): tv = await s.get(TV, tv_id) - ti = (await s.execute( + # re-sending after a remove can leave multiple rows per (tv, image) + rows = (await s.execute( select(TVImage).where(TVImage.tv_id == tv_id, TVImage.image_id == image_id) - )).scalar_one_or_none() - if not tv or not ti: + )).scalars().all() + if not tv or not rows: raise HTTPException(404) - if ti.remote_id: - await tv_manager.delete_image(tv, ti.remote_id) - ti.is_on_tv = False + for ti in rows: + if ti.is_on_tv and ti.remote_id: + if not await tv_manager.delete_image(tv, ti.remote_id): + raise HTTPException(503, "TV unreachable — try again when the TV is on") + ti.is_on_tv = False await s.commit() return {"ok": True} @@ -256,15 +259,30 @@ async def list_on_tv(tv_id: int, s: AsyncSession = Depends(get_session)): return rows +def _tv_thumb_cache_path(tv_id: int, remote_id: str) -> str: + safe = _SAFE_NAME_RE.sub("_", remote_id) + return os.path.join(settings.IMAGE_CACHE_DIR, "tv_thumbs", f"{tv_id}_{safe}.jpg") + + @router.get("/tv/{tv_id}/thumbnail/{remote_id}") async def tv_thumb(tv_id: int, remote_id: str, s: AsyncSession = Depends(get_session)): tv = await s.get(TV, tv_id) if not tv: raise HTTPException(404) + cache_path = _tv_thumb_cache_path(tv_id, remote_id) data = await tv_manager.get_thumbnail(tv, remote_id) - if not data: - raise HTTPException(404) - return Response(content=data, media_type="image/jpeg") + if data: + try: + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + with open(cache_path, "wb") as f: + f.write(data) + except OSError as e: + log.warning("tv thumb cache write failed %s: %s", cache_path, e) + return Response(content=data, media_type="image/jpeg") + # TV unreachable (asleep/off) — serve the last cached copy if we have one + if os.path.exists(cache_path): + return FileResponse(cache_path, media_type="image/jpeg") + raise HTTPException(404) # ── Sync all library images → TV ──────────────────────────────────────────── diff --git a/backend/services/sources/nasa_apod.py b/backend/services/sources/nasa_apod.py index a241f70..3d08df9 100644 --- a/backend/services/sources/nasa_apod.py +++ b/backend/services/sources/nasa_apod.py @@ -6,11 +6,14 @@ async def today() -> dict | None: url = "https://api.nasa.gov/planetary/apod" params = {"api_key": settings.NASA_API_KEY or "DEMO_KEY"} - async with httpx.AsyncClient(timeout=15.0) as c: - r = await c.get(url, params=params) - if r.status_code != 200: - return None - j = r.json() + try: + async with httpx.AsyncClient(timeout=15.0) as c: + r = await c.get(url, params=params) + if r.status_code != 200: + return None + j = r.json() + except httpx.HTTPError: + return None if j.get("media_type") != "image": return {"unsupported": True, "title": j.get("title"), "media_type": j.get("media_type")} return { diff --git a/backend/services/tv_manager.py b/backend/services/tv_manager.py index 4fa71ee..4a59a86 100644 --- a/backend/services/tv_manager.py +++ b/backend/services/tv_manager.py @@ -403,9 +403,14 @@ async def list_mattes(self, tv: TV) -> list[str]: try: art = await conn._ensure_art() res = await art.get_matte_list() - if isinstance(res, list): - return res - return list(res) if res else [] + out: list[str] = [] + for m in (list(res) if res else []): + # entries may be plain strings or dicts like {"matte_type": "shadowbox_polar"} + if isinstance(m, dict): + m = m.get("matte_type") or "" + if m: + out.append(str(m)) + return out except Exception as e: log.warning("list_mattes failed TV %s: %s", tv.id, e, exc_info=True) conn.art = None From a88b51a80b21fe240397e82202c55a92a960cbc1 Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sun, 5 Jul 2026 10:37:39 -0700 Subject: [PATCH 6/7] =?UTF-8?q?fix:=20TV=20thumbnails=20never=20loaded=20?= =?UTF-8?q?=E2=80=94=20wrong=20get=5Fthumbnail=20signature?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NickWaterton samsungtvws fork pinned in requirements.txt has get_thumbnail(content_id_list, as_dict=False), not as_bytes=True, so every thumbnail fetch raised TypeError and the /tv grid 404'd even with the TV awake. A single content id with as_dict=False returns raw bytes. Co-Authored-By: Claude Fable 5 --- backend/services/tv_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/services/tv_manager.py b/backend/services/tv_manager.py index 4a59a86..834a09b 100644 --- a/backend/services/tv_manager.py +++ b/backend/services/tv_manager.py @@ -324,7 +324,8 @@ async def get_thumbnail(self, tv: TV, content_id: str) -> bytes | None: conn = await self.get(tv) try: art = await conn._ensure_art() - data = await art.get_thumbnail(content_id, as_bytes=True) + # NickWaterton fork: single content id + as_dict=False returns raw bytes + data = await art.get_thumbnail(content_id) return data if isinstance(data, (bytes, bytearray)) else None except Exception as e: log.warning("get_thumbnail failed TV %s content_id=%s: %s", tv.id, content_id, e, exc_info=True) From 1cc14792bb3239fc9b83c650eb47c7f1a63d048f Mon Sep 17 00:00:00 2001 From: Eliav Kadosh Date: Sun, 5 Jul 2026 10:50:54 -0700 Subject: [PATCH 7/7] fix: TizenBrew install failed on relative WGT paths TIZENBREW_DOWNLOAD_DIR defaults to the relative ./data/tizenbrew, but the tizen CLI resolves relative paths against its own bin directory, so "tizen package -o data/tizenbrew/signed" wrote the re-signed WGT into tizen-studio/tools/ide/bin/data/... and the subsequent "tizen install -n data/tizenbrew/signed/x.wgt" failed with "There is no package with named ...". Resolve the download dir (and resign_wgt's paths) to absolute paths. Co-Authored-By: Claude Fable 5 --- backend/services/tizenbrew_service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/services/tizenbrew_service.py b/backend/services/tizenbrew_service.py index 51a8b2d..875d21f 100644 --- a/backend/services/tizenbrew_service.py +++ b/backend/services/tizenbrew_service.py @@ -163,7 +163,9 @@ def requires_certificate(year: int | None) -> bool: class TizenBrewService: def __init__(self) -> None: - self.download_dir = Path(getattr(settings, "TIZENBREW_DOWNLOAD_DIR", "./data/tizenbrew")) + # Absolute path: relative paths handed to the tizen CLI get resolved + # against the CLI's own bin directory, not our working directory + self.download_dir = Path(getattr(settings, "TIZENBREW_DOWNLOAD_DIR", "./data/tizenbrew")).resolve() self.download_dir.mkdir(parents=True, exist_ok=True) # Track running jobs by tv_id to prevent overlap self._jobs: dict[int, asyncio.Task] = {} @@ -705,6 +707,9 @@ async def resign_wgt( self, tizen_path: str, wgt_path: str, profile_name: str, output_dir: str, tv_id: int | None = None, ) -> dict[str, Any]: + # The tizen CLI resolves relative paths against its own bin dir — force absolute + output_dir = str(Path(output_dir).resolve()) + wgt_path = str(Path(wgt_path).resolve()) Path(output_dir).mkdir(parents=True, exist_ok=True) if tv_id is not None: await ws_manager.broadcast({