lovdataAPI.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
lovdataAPI.py
-----------------
Et produksjonsklart FastAPI-lag som wrapper Lovdata sitt offentlige API og
legger på søk, indeksering og uthenting/parsing av XML fra de publiserte ZIP-filene.

Funksjonelt innhold:
  - Helse:                  GET  /health
  - List public data:       GET  /v1/public/list
  - Last ned ZIP (stream):  GET  /v1/public/get/{filename}
  - Indexér ZIP:            GET  /v1/public/index/{filename}
  - Hent + parse XML:       GET  /v1/public/extract/{filename}/{member}
  - Søk i public data:      GET  /v1/public/search?q=...&latest=3&max_hits=20&regex=false&case_sensitive=false
  - Lesende proxy (allowlist): GET /proxy?path=v1/publicData/list

Kjør lokalt:
    uvicorn lovdataAPI:app --host 0.0.0.0 --port 8000

Produksjon:
    - Container: FROM python:3.11-slim, installer avhengigheter (fastapi, uvicorn[standard], httpx).
    - Konfigurer miljøvariabler under.

Miljøvariabler:
    LOVDATA_BASE_URL (default: https://api.lovdata.no)
    LOVDATA_TIMEOUT_SECONDS (default: 30)
    LOVDATA_API_KEY (default: tom; hvis satt sendes som Bearer-token i Authorization-header)
    LOG_LEVEL (default: INFO)
    LOVDATA_CACHE_DIR (default: /tmp/lovdata-cache)
    LOVDATA_CACHE_TTL_MINUTES (default: 1440)

Merk:
    - Koden bruker httpx (async) med eksponentiell backoff og robust feilhåndtering.
    - ZIP-filer caches lokalt for ytelse (TTL styrt av miljøvariabel).
"""
from __future__ import annotations

import asyncio
import json
import logging
import os
from typing import Any, AsyncIterator, Dict, List, Optional

import httpx
from fastapi import Depends, FastAPI, HTTPException, Path, Query, status
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, JSONResponse, PlainTextResponse
from pydantic import BaseModel, Field

# --------------------------
# Konfigurasjon og logging
# --------------------------

LOVDATA_BASE_URL = os.getenv("LOVDATA_BASE_URL", "https://api.lovdata.no").rstrip("/")
TIMEOUT_S = float(os.getenv("LOVDATA_TIMEOUT_SECONDS", "30"))
API_KEY = os.getenv("LOVDATA_API_KEY", "").strip()
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()

logging.basicConfig(
    level=getattr(logging, LOG_LEVEL, logging.INFO),
    format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
logger = logging.getLogger("lovdataAPI")

# --------------------------
# HTTP-klient med backoff
# --------------------------

class LovdataClient:
    def __init__(self, base_url: str = LOVDATA_BASE_URL, timeout: float = TIMEOUT_S, api_key: str = API_KEY) -> None:
        headers = {}
        if api_key:
            headers["Authorization"] = f"Bearer {api_key}"
        self._client = httpx.AsyncClient(
            base_url=base_url,
            headers=headers,
            timeout=httpx.Timeout(timeout),
            follow_redirects=True,
        )

    async def _request_with_retry(
        self,
        method: str,
        url: str,
        *,
        params: Optional[Dict[str, Any]] = None,
        headers: Optional[Dict[str, str]] = None,
        max_retries: int = 3,
        initial_backoff: float = 0.5,
        stream: bool = False,
    ) -> httpx.Response:
        last_exc: Optional[BaseException] = None
        backoff = initial_backoff
        for attempt in range(1, max_retries + 2):
            try:
                # httpx.AsyncClient.request() doesn't support stream parameter directly
                # Use stream() method for streaming requests
                if stream:
                    # For streaming, we need to use the stream context manager
                    # But we can't return from inside a context manager, so we'll handle it differently
                    resp = await self._client.request(method, url, params=params, headers=headers)
                    # For streaming, we need to keep the connection open
                    # The actual streaming is handled in the stream() method
                else:
                    resp = await self._client.request(method, url, params=params, headers=headers)
                if resp.status_code >= 500:
                    raise httpx.HTTPStatusError("Server error", request=resp.request, response=resp)
                return resp
            except (httpx.HTTPError, httpx.ReadTimeout) as exc:
                last_exc = exc
                if attempt > max_retries:
                    break
                logger.warning("HTTP error on %s %s (attempt %d/%d): %s", method, url, attempt, max_retries, exc)
                await asyncio.sleep(backoff)
                backoff *= 2
        assert last_exc is not None
        raise last_exc

    async def get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
        resp = await self._request_with_retry("GET", path, params=params)
        if resp.status_code == 404:
            raise HTTPException(status_code=404, detail="Ressursen ble ikke funnet hos Lovdata")
        if resp.status_code >= 400:
            raise HTTPException(status_code=resp.status_code, detail=f"Feil fra Lovdata: {resp.text}")
        try:
            return resp.json()
        except json.JSONDecodeError:
            raise HTTPException(status_code=502, detail="Ugyldig JSON fra Lovdata")

    async def stream(self, path: str, params: Optional[Dict[str, Any]] = None) -> AsyncIterator[bytes]:
        # Use httpx streaming directly instead of request_with_retry
        last_exc: Optional[BaseException] = None
        max_retries = 3
        backoff = 0.5
        for attempt in range(1, max_retries + 2):
            try:
                async with self._client.stream("GET", path, params=params) as resp:
                    if resp.status_code == 404:
                        raise HTTPException(status_code=404, detail="Ressursen ble ikke funnet hos Lovdata")
                    if resp.status_code >= 400:
                        body = await resp.aread()
                        raise HTTPException(status_code=resp.status_code, detail=f"Feil fra Lovdata: {body.decode('utf-8', 'ignore')}")
                    if resp.status_code >= 500:
                        raise httpx.HTTPStatusError("Server error", request=resp.request, response=resp)
                    async for chunk in resp.aiter_bytes():
                        yield chunk
                    return
            except (httpx.HTTPError, httpx.ReadTimeout) as exc:
                last_exc = exc
                if attempt > max_retries:
                    break
                logger.warning("HTTP error on GET %s (attempt %d/%d): %s", path, attempt, max_retries, exc)
                await asyncio.sleep(backoff)
                backoff *= 2
            except HTTPException:
                raise
        if last_exc:
            raise last_exc
        raise Exception("Failed to stream after retries")

    async def aclose(self) -> None:
        await self._client.aclose()


async def get_client() -> LovdataClient:
    client = LovdataClient()
    try:
        yield client
    finally:
        await client.aclose()

# --------------------------
# Pydantic-modeller (grunn)
# --------------------------

class PublicDataListResponse(BaseModel):
    files: List[str] = Field(default_factory=list, description="Tilgjengelige filnavn (ZIP) for publikumsdata")

# --------------------------
# FastAPI-app
# --------------------------

app = FastAPI(
    title="Lovdata Public API Wrapper",
    version="1.1.0",
    description=(
        "Et slankt API som tilbyr sikre oppslag mot Lovdata sitt offentlige API. "
        "Støtter /v1/publicData (list/get), ZIP-indeksering, XML-parsing og fulltekstsøk."
    ),
)

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Stram inn i prod ved behov
    allow_credentials=False,
    allow_methods=["GET", "OPTIONS"],
    allow_headers=["*"],
)

@app.get("/health", response_class=PlainTextResponse, tags=["internal"])
async def health() -> str:
    return "OK"

# --------------------------
# Public Data endpoints
# --------------------------

@app.get("/v1/public/list", response_model=PublicDataListResponse, tags=["public-data"])
async def list_public_data(client: LovdataClient = Depends(get_client)) -> PublicDataListResponse:
    raw = await client.get_json("/v1/publicData/list")
    files = []
    if isinstance(raw, dict) and "files" in raw:
        # Handle dict with "files" key
        for item in raw.get("files", []):
            if isinstance(item, dict) and "filename" in item:
                files.append(str(item["filename"]))
            elif isinstance(item, str):
                files.append(item)
            else:
                files.append(str(item))
    elif isinstance(raw, list):
        # Handle list of dicts or strings
        for item in raw:
            if isinstance(item, dict) and "filename" in item:
                files.append(str(item["filename"]))
            elif isinstance(item, str):
                files.append(item)
            else:
                files.append(str(item))
    else:
        logger.warning("Uventet struktur for /v1/publicData/list: %r", raw)
        files = []
    return PublicDataListResponse(files=files)

@app.get("/v1/public/get/{filename}", tags=["public-data"])
async def get_public_zip(
    filename: str = Path(..., description="Filnavn fra /v1/publicData/list, f.eks. lti-2001-2023.zip"),
    client: LovdataClient = Depends(get_client),
):
    async def iterator():
        async for chunk in client.stream(f"/v1/publicData/get/{filename}"):
            yield chunk
    headers = {
        "Content-Disposition": f'attachment; filename="{filename}"',
        "Content-Type": "application/zip",
    }
    return StreamingResponse(iterator(), headers=headers, media_type="application/zip")

# --------------------------
# Read-only proxy (GET) med allowlist
# --------------------------

ALLOWED_PROXY_PREFIXES = (
    "/v1/publicData/",
    "/swagger",
    "/swagger/index.html",
)

@app.get("/proxy", tags=["proxy"])
async def read_only_proxy(
    path: str = Query(..., description="Path under Lovdata, uten ledende '/' - f.eks. 'v1/publicData/list'"),
    client: LovdataClient = Depends(get_client),
    raw: bool = Query(False, description="Hvis true, returneres råtekst fra Lovdata; ellers forsøkes JSON først."),
):
    normalized = "/" + path.lstrip("/")
    if not any(normalized.startswith(prefix) for prefix in ALLOWED_PROXY_PREFIXES):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=f"Proxy er kun tillatt for path som starter med: {', '.join(ALLOWED_PROXY_PREFIXES)}",
        )
    try:
        data = await client.get_json(normalized)
        return JSONResponse(content=data)
    except HTTPException as he:
        if he.status_code in (404, 502):
            raise

    async def text_stream() -> AsyncIterator[bytes]:
        async for chunk in client.stream(normalized):
            yield chunk
    return StreamingResponse(text_stream(), media_type="text/plain; charset=utf-8")

# ==========================
# Nedlasting, caching og XML
# ==========================
import io
import re
import tarfile
import zipfile
from datetime import datetime, timedelta
from xml.etree import ElementTree as ET

CACHE_DIR = os.getenv("LOVDATA_CACHE_DIR", "/tmp/lovdata-cache")
CACHE_TTL_MINUTES = int(os.getenv("LOVDATA_CACHE_TTL_MINUTES", "1440"))  # 24h

os.makedirs(CACHE_DIR, exist_ok=True)

def _cache_path(filename: str) -> str:
    safe = filename.replace("/", "_")
    return os.path.join(CACHE_DIR, safe)

async def _download_zip_to_cache(filename: str, client: LovdataClient) -> str:
    dst = _cache_path(filename)
    if os.path.exists(dst):
        age = datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(dst))
        if age <= timedelta(minutes=CACHE_TTL_MINUTES):
            return dst
    tmp = dst + ".part"
    with open(tmp, "wb") as f:
        async for chunk in client.stream(f"/v1/publicData/get/{filename}"):
            f.write(chunk)
    os.replace(tmp, dst)
    return dst

def _zip_members_info(zf: zipfile.ZipFile):
    infos = []
    for zi in zf.infolist():
        infos.append({
            "name": zi.filename,
            "size": zi.file_size,
            "compress_size": zi.compress_size,
            "date_time": "%04d-%02d-%02d %02d:%02d:%02d" % zi.date_time
        })
    return infos

def _tar_members_info(tf: tarfile.TarFile):
    infos = []
    for member in tf.getmembers():
        if member.isfile():
            infos.append({
                "name": member.name,
                "size": member.size,
                "compress_size": None,  # tarfile doesn't provide compressed size easily
                "date_time": member.mtime and datetime.fromtimestamp(member.mtime).strftime("%Y-%m-%d %H:%M:%S") or ""
            })
    return infos

def _xml_text(elem: ET.Element) -> str:
    parts = []
    def _walk(e):
        if e.text:
            parts.append(e.text)
        for c in list(e):
            _walk(c)
        if e.tail:
            parts.append(e.tail)
    _walk(elem)
    return " ".join(p.strip() for p in parts if p and p.strip())

def _best_effort_title(root: ET.Element) -> str:
    candidates = [
        ".//tittel", ".//tittel1", ".//title", ".//overskrift", ".//heading",
        ".//Tittel", ".//Title"
    ]
    for xp in candidates:
        n = root.find(xp)
        if n is not None:
            t = _xml_text(n).strip()
            if t:
                return t[:300]
    return root.tag

def _best_effort_date(root: ET.Element) -> str:
    for xp in [".//dato", ".//ikrafttredelse", ".//kunngjort", ".//published", ".//datoKunngjort"]:
        n = root.find(xp)
        if n is not None:
            txt = _xml_text(n).strip()
            if txt:
                return txt[:50]
    return ""

# --------------------------
# Nye Pydantic-modeller
# --------------------------

class ZipIndexResponse(BaseModel):
    filename: str
    entries: List[Dict[str, Any]]

class XmlExtractResponse(BaseModel):
    filename: str
    member: str
    root_tag: str
    title: Optional[str] = None
    date: Optional[str] = None
    text: str

class SearchHit(BaseModel):
    filename: str
    member: str
    title: Optional[str] = None
    date: Optional[str] = None
    snippet: str

class SearchResponse(BaseModel):
    query: str
    hits: List[SearchHit]
    searched_files: List[str]

# --------------------------
# Endepunkt: index av ZIP
# --------------------------

@app.get("/v1/public/index/{filename}", response_model=ZipIndexResponse, tags=["public-data"])
async def index_zip(
    filename: str = Path(..., description="Filnavn fra /v1/publicData/list"),
    client: LovdataClient = Depends(get_client),
) -> ZipIndexResponse:
    try:
        local = await _download_zip_to_cache(filename, client)
        if not os.path.exists(local):
            raise HTTPException(status_code=500, detail=f"Kunne ikke laste ned eller finne fil: {filename}")
        entries = []
        try:
            # Handle tar.bz2 files
            if filename.endswith('.tar.bz2') or filename.endswith('.tbz2'):
                with tarfile.open(local, 'r:bz2') as tf:
                    entries = _tar_members_info(tf)
            # Handle ZIP files (for backward compatibility)
            else:
                with zipfile.ZipFile(local, "r") as zf:
                    entries = _zip_members_info(zf)
        except (zipfile.BadZipFile, tarfile.TarError) as e:
            logger.error("Feil ved åpning av arkiv %s: %s", filename, str(e))
            raise HTTPException(status_code=502, detail=f"Ugyldig arkiv fra kilde: {type(e).__name__} - {str(e)}")
        except Exception as e:
            logger.error("Uventet feil ved indeksering av %s: %s", filename, str(e), exc_info=True)
            raise HTTPException(status_code=500, detail=f"Feil ved indeksering: {type(e).__name__} - {str(e)}")
        return ZipIndexResponse(filename=filename, entries=entries)
    except HTTPException:
        raise
    except Exception as e:
        logger.error("Uventet feil i index_zip for %s: %s", filename, str(e), exc_info=True)
        raise HTTPException(status_code=500, detail=f"Intern feil: {type(e).__name__} - {str(e)}")

# --------------------------
# Endepunkt: hent/parsing av XML
# --------------------------

@app.get("/v1/public/extract/{filename}/{member:path}", response_model=XmlExtractResponse, tags=["public-data"])
async def extract_xml(
    filename: str,
    member: str,
    client: LovdataClient = Depends(get_client),
) -> XmlExtractResponse:
    local = await _download_zip_to_cache(filename, client)
    data = None
    try:
        # Handle tar.bz2 files
        if filename.endswith('.tar.bz2') or filename.endswith('.tbz2'):
            with tarfile.open(local, 'r:bz2') as tf:
                try:
                    tar_member = tf.getmember(member)
                    if tar_member.isfile():
                        f = tf.extractfile(tar_member)
                        if f is None:
                            raise HTTPException(status_code=404, detail=f"Medlem '{member}' kunne ikke ekstraheres fra tar.bz2")
                        data = f.read()
                        f.close()
                    else:
                        raise HTTPException(status_code=404, detail=f"Medlem '{member}' er ikke en fil i tar.bz2")
                except KeyError:
                    raise HTTPException(status_code=404, detail=f"Medlem '{member}' ikke funnet i tar.bz2")
        # Handle ZIP files (for backward compatibility)
        else:
            with zipfile.ZipFile(local, "r") as zf:
                if member not in zf.namelist():
                    raise HTTPException(status_code=404, detail="Filen finnes ikke i ZIP")
                with zf.open(member) as fh:
                    data = fh.read()
    except (zipfile.BadZipFile, tarfile.TarError) as e:
        raise HTTPException(status_code=502, detail=f"Ugyldig arkiv fra kilde: {type(e).__name__}")
    
    if data is None:
        raise HTTPException(status_code=404, detail=f"Kunne ikke ekstrahere '{member}' fra arkiv")
    
    try:
        root = ET.fromstring(data)
    except ET.ParseError:
        text = data.decode("utf-8", "ignore")
        return XmlExtractResponse(
            filename=filename, member=member, root_tag="(raw)",
            title=None, date=None, text=text
        )

    title = _best_effort_title(root)
    date = _best_effort_date(root)
    text = _xml_text(root)
    return XmlExtractResponse(
        filename=filename, member=member, root_tag=root.tag, title=title, date=date, text=text
    )

# --------------------------
# Endepunkt: søk i ZIP-er
# --------------------------

@app.get("/v1/public/search", response_model=SearchResponse, tags=["search"])
async def search_public_data(
    q: str = Query(..., min_length=2, description="Søkestreng. Bruk regex=true for regulære uttrykk."),
    filenames: Optional[List[str]] = Query(None, description="Valgfritt: spesifikke ZIP-filer å søke i"),
    latest: int = Query(3, ge=1, le=50, description="Hvis filenames ikke er satt, søk i N siste ZIP-er"),
    max_hits: int = Query(20, ge=1, le=200),
    case_sensitive: bool = Query(False),
    regex: bool = Query(False, description="Tolk q som et regulært uttrykk"),
    client: LovdataClient = Depends(get_client),
) -> SearchResponse:
    files_to_search: List[str] = []
    if filenames:
        files_to_search = list(filenames)
    else:
        raw = await client.get_json("/v1/publicData/list")
        listing = []
        if isinstance(raw, dict) and "files" in raw:
            # Handle dict with "files" key
            for item in raw.get("files", []):
                if isinstance(item, dict) and "filename" in item:
                    listing.append(str(item["filename"]))
                elif isinstance(item, str):
                    listing.append(item)
                else:
                    listing.append(str(item))
        elif isinstance(raw, list):
            # Handle list of dicts or strings
            for item in raw:
                if isinstance(item, dict) and "filename" in item:
                    listing.append(str(item["filename"]))
                elif isinstance(item, str):
                    listing.append(item)
                else:
                    listing.append(str(item))
        listing.sort()
        files_to_search = listing[-latest:] if listing else []

    flags = 0 if case_sensitive else re.IGNORECASE
    pattern = re.compile(q, flags) if regex else None
    hits: List[SearchHit] = []

    for fn in files_to_search:
        local = await _download_zip_to_cache(fn, client)
        try:
            # Handle tar.bz2 files
            if fn.endswith('.tar.bz2') or fn.endswith('.tbz2'):
                with tarfile.open(local, 'r:bz2') as tf:
                    for member in tf.getmembers():
                        if not member.isfile():
                            continue
                        name = member.name
                        if not name.lower().endswith((".xml", ".html", ".htm")):
                            continue
                        try:
                            f = tf.extractfile(member)
                            if f is None:
                                continue
                            data = f.read()
                            f.close()
                            is_xml = True
                            try:
                                root = ET.fromstring(data)
                            except ET.ParseError:
                                is_xml = False
                            if is_xml:
                                title = _best_effort_title(root)
                                date = _best_effort_date(root)
                                text = _xml_text(root)
                            else:
                                title = None
                                date = None
                                text = data.decode("utf-8", "ignore")
                            if regex:
                                m = pattern.search(text)
                                matched = m is not None
                                span = m.span() if m else None
                            else:
                                idx = text.find(q) if case_sensitive else text.lower().find(q.lower())
                                matched = idx >= 0
                                span = (idx, idx + len(q)) if matched else None
                            if matched:
                                start = max(0, (span[0] if span else 0) - 80)
                                end = min(len(text), (span[1] if span else 0) + 80)
                                snippet = text[start:end].replace("\n", " ").replace("\r", " ").replace("\t", " ")
                                snippet = snippet.replace("\u0000", " ").replace("\x00", " ")
                                hits.append(SearchHit(filename=fn, member=name, title=title, date=date, snippet=snippet))
                                if len(hits) >= max_hits:
                                    return SearchResponse(query=q, hits=hits, searched_files=files_to_search)
                        except Exception as e:
                            logger.warning("Feil under lesing av %s:%s: %s", fn, name, type(e).__name__)
                            continue
            # Handle ZIP files (for backward compatibility)
            else:
                with zipfile.ZipFile(local, "r") as zf:
                    for name in zf.namelist():
                        if not name.lower().endswith((".xml", ".html", ".htm")):
                            continue
                        try:
                            with zf.open(name) as fh:
                                data = fh.read()
                            is_xml = True
                            try:
                                root = ET.fromstring(data)
                            except ET.ParseError:
                                is_xml = False
                            if is_xml:
                                title = _best_effort_title(root)
                                date = _best_effort_date(root)
                                text = _xml_text(root)
                            else:
                                title = None
                                date = None
                                text = data.decode("utf-8", "ignore")
                            if regex:
                                m = pattern.search(text)
                                matched = m is not None
                                span = m.span() if m else None
                            else:
                                idx = text.find(q) if case_sensitive else text.lower().find(q.lower())
                                matched = idx >= 0
                                span = (idx, idx + len(q)) if matched else None
                            if matched:
                                start = max(0, (span[0] if span else 0) - 80)
                                end = min(len(text), (span[1] if span else 0) + 80)
                                snippet = text[start:end].replace("\n", " ").replace("\r", " ").replace("\t", " ")
                                snippet = snippet.replace("\u0000", " ").replace("\x00", " ")
                                hits.append(SearchHit(filename=fn, member=name, title=title, date=date, snippet=snippet))
                                if len(hits) >= max_hits:
                                    return SearchResponse(query=q, hits=hits, searched_files=files_to_search)
                        except Exception as e:
                            logger.warning("Feil under lesing av %s:%s: %s", fn, name, type(e).__name__)
                            continue
        except (zipfile.BadZipFile, tarfile.TarError) as e:
            logger.error("Ugyldig arkiv: %s (%s)", fn, type(e).__name__)
            continue

    return SearchResponse(query=q, hits=hits, searched_files=files_to_search)

# --------------------------
# Runtime entrypoint
# --------------------------

if __name__ == "__main__":
    import uvicorn
    uvicorn.run("lovdataAPI:app", host="0.0.0.0", port=int(os.getenv("PORT", "8000")), reload=False)