diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b6ba4e..d5c5d05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,5 +16,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - run: pip install -e ".[dev]" + - run: sudo apt-get update && sudo apt-get install -y ffmpeg + - run: pip install -e ".[dev]" -e "./sonilo-video-kit[dev]" - run: pytest + - run: pytest sonilo-video-kit diff --git a/.github/workflows/publish-video-kit.yml b/.github/workflows/publish-video-kit.yml new file mode 100644 index 0000000..260ff46 --- /dev/null +++ b/.github/workflows/publish-video-kit.yml @@ -0,0 +1,44 @@ +name: Publish sonilo-video-kit to PyPI + +on: + workflow_dispatch: + push: + tags: ["sonilo-video-kit-v*"] + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + # PyPI Trusted Publishing (OIDC). No PYPI_TOKEN secret needed. + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: sudo apt-get update && sudo apt-get install -y ffmpeg + - run: pip install -e ".[dev]" -e "./sonilo-video-kit[dev]" build + - run: pytest sonilo-video-kit + - name: Verify tag matches pyproject version + if: github.ref_type == 'tag' + run: | + TAG_VERSION="${GITHUB_REF_NAME#sonilo-video-kit-v}" + PKG_VERSION=$(python -c "import tomllib; print(tomllib.load(open('sonilo-video-kit/pyproject.toml','rb'))['project']['version'])") + if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then + echo "Tag version ($TAG_VERSION) does not match pyproject version ($PKG_VERSION)" + exit 1 + fi + - run: python -m build sonilo-video-kit + - name: Publish to PyPI + # Authenticates via OIDC Trusted Publishing — configured as a pending + # publisher for project `sonilo-video-kit` (repo sonilo-python, + # workflow publish-video-kit.yml). Tag refs only: a workflow_dispatch + # on a branch runs tests/build but never publishes. To retry a failed + # publish, dispatch on the tag: + # gh workflow run publish-video-kit.yml --ref sonilo-video-kit-vX.Y.Z + if: github.ref_type == 'tag' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: sonilo-video-kit/dist + skip-existing: true diff --git a/.gitignore b/.gitignore index af95f7e..8389dca 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ build/ .pytest_cache/ .DS_Store +/docs/ diff --git a/sonilo-video-kit/LICENSE b/sonilo-video-kit/LICENSE new file mode 100644 index 0000000..e62a1a5 --- /dev/null +++ b/sonilo-video-kit/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sonilo AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sonilo-video-kit/README.md b/sonilo-video-kit/README.md new file mode 100644 index 0000000..10f2e78 --- /dev/null +++ b/sonilo-video-kit/README.md @@ -0,0 +1,128 @@ +# sonilo-video-kit + +Video helpers for the [Sonilo](https://sonilo.com) API: generate a soundtrack +for a video and mix it in locally with ffmpeg. Python ≥ 3.9. + +Requires `ffmpeg` + `ffprobe` on your PATH (macOS: `brew install ffmpeg`, +Debian/Ubuntu: `apt-get install ffmpeg`) — or pass `ffmpeg_path`/`ffprobe_path` +to any function. + +## Installation + +```bash +pip install sonilo sonilo-video-kit +``` + +`sonilo` (the core client) is a required dependency — it is installed +automatically alongside the kit, but is shown here for clarity. + +## Quickstart + +```python +from sonilo_video_kit import generate_music_for_video, mix_with_video + +track = generate_music_for_video("./clip.mp4", prompt="upbeat, energetic") # uses SONILO_API_KEY + +mix_with_video( + video="./clip.mp4", + audio=track.audio, + output="./clip.scored.mp4", +) +``` + +## Loudness-matched mixing + +By default the kit measures the loudness (LUFS) of your video's own audio and +of the generated music, then sits the music 4 LU below the original — so +dialogue stays intelligible without hand-tuning. The final file is normalized +to −14 LUFS (streaming-platform delivery level) with a −1 dBFS peak limiter. +The delivery-normalize boost is capped at +12 dB; attenuation (bringing an +overly loud render down to target) is uncapped. + +- `music_volume` (0–1, default 0.5): 0.5 is the matched level; each step of + 0.25 shifts ±6 dB (full range ±12 dB). 0 mutes the music. +- `original_volume` (0–1, default 1): absolute — 1 keeps the original exactly + as recorded, 0 removes it entirely. +- `loudness_match=False` switches both knobs to plain absolute gains. +- `normalize=False` skips the delivery-loudness pass. + +If loudness measurement fails (exotic codecs, unreadable audio), the kit +silently falls back to absolute-gain behavior rather than failing your render. + +## Ducking + +`mix_with_video` sits the music at a fixed level under the original audio. +`duck_music_under_speech` goes further: it rides the music down whenever +someone speaks and back up in the gaps. + +```python +from sonilo_video_kit import duck_music_under_speech + +duck_music_under_speech( + video="./interview.mp4", + audio=track.audio, + output="./interview.ducked.mp4", +) +``` + +Unlike `mix_with_video`, which is entirely local and free, this calls the +Sonilo ducking API and is **billed on your video's duration**. The kit +uploads only the video's extracted audio track — your picture never leaves +the machine and is copied into the result untouched. The API sets the +ducking curve itself (speech gate, duck depth, −14 LUFS delivery, −1 dBTP +ceiling), so there are no volume knobs to pass. + +Requirements are enforced locally, before anything is uploaded or charged: +the video must have an audio track and a real picture, it must run no longer +than **360 s**, `output` must carry a file extension and live in a directory +that already exists and is writable, and your picture must be +stream-copyable into `output`'s container. The ducked audio is always written +as **AAC**, so `output` must be a container that can carry AAC — `.webm` +(Vorbis/Opus only) is rejected before the API call whatever your picture +codec. Both the **360 s** limit and the amount billed are measured on the +**picture**, not the container (a video whose audio outlives its picture is +billed for the picture's length). Any failure raises before the API is called; +the kit never quietly falls back to an un-ducked mix. Use `mix_with_video` for +silent or longer videos. + +### Nothing you have paid for is thrown away + +The API charges when the job is **submitted**, and the task then runs to +completion server-side whatever happens to your process — so every failure +after that point keeps the mix you paid for reachable. Transient failures are +retried with backoff (a 5xx while polling, a dropped connection, a 503 from the +storage host mid-download). + +If a failure after submit is **not** locally recoverable — the poll fails +terminally, the download can't complete, or the wait times out — the raised +`VideoKitError` names the **task id** and tells you the charge was already made +and the task is still finishing server-side. **No local rescue file is written +in this case.** Recover by polling `GET /v1/tasks/` yourself until it +reports `succeeded`, then download the `output_url` it returns: that re-fetches +the mix you already paid for, instead of calling `duck_music_under_speech` +again and being billed twice. + +If instead a final, purely-local step fails *after* the API call — remuxing +the ducked audio onto your picture, or placing the file at `output` (e.g. the +disk fills mid-mux) — the kit saves the downloaded ducked audio to +`.ducked.wav` and raises a `VideoKitError` naming that path, so you can +fix the local problem and finish locally. A rescue never overwrites an earlier +one (`.ducked.1.wav`, …), and `output` is always placed atomically. + +## Errors + +`VideoKitError` (invalid arguments, unreadable video), `FfmpegNotFoundError` +(ffmpeg/ffprobe missing — message includes install hints), `FfmpegError` +(ffmpeg failed — carries `exit_code` and `stderr_tail`), `DuckingFailedError` +(the ducking API accepted the job but could not finish it — carries `code` +and `refunded`). Errors from the Sonilo API pass through as the `sonilo` +package's typed errors. + +`refunded` reports what the server said **at the moment the task was polled**, +not a final verdict: the backend marks a task failed before it reverses the +charge (and retries a failed reversal), so `refunded: False` means the reversal +had not landed yet, not that you were definitely billed. + +## License + +MIT diff --git a/sonilo-video-kit/examples/score_video.py b/sonilo-video-kit/examples/score_video.py new file mode 100644 index 0000000..9df406e --- /dev/null +++ b/sonilo-video-kit/examples/score_video.py @@ -0,0 +1,19 @@ +"""Usage: SONILO_API_KEY=sk_... python examples/score_video.py clip.mp4 "upbeat, energetic" + +Generates a soundtrack for a video and mixes it in locally with ffmpeg. +Requires ffmpeg + ffprobe on PATH. +""" +import sys + +from sonilo_video_kit import generate_music_for_video, mix_with_video + +video = sys.argv[1] if len(sys.argv) > 1 else "clip.mp4" +prompt = sys.argv[2] if len(sys.argv) > 2 else "cinematic orchestral score" +output = "clip.scored.mp4" + +track = generate_music_for_video(video, prompt=prompt) + +out = mix_with_video(video=video, audio=track.audio, output=output) + +title = f' — "{track.title}"' if track.title else "" +print(f"Saved {out}{title}") diff --git a/sonilo-video-kit/pyproject.toml b/sonilo-video-kit/pyproject.toml new file mode 100644 index 0000000..1eb4101 --- /dev/null +++ b/sonilo-video-kit/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "sonilo-video-kit" +version = "0.1.0" +description = "Video helpers for the Sonilo API: generate a soundtrack and mix it into your video with ffmpeg" +readme = "README.md" +license = "MIT" +requires-python = ">=3.9" +authors = [{ name = "Sonilo AI" }] +dependencies = ["sonilo>=0.3,<0.4"] +keywords = ["sonilo", "video", "music", "ffmpeg", "soundtrack", "ducking", "ai"] + +[project.urls] +Repository = "https://github.com/sonilo-ai/sonilo-python" + +[project.optional-dependencies] +dev = ["pytest>=8", "respx>=0.21"] + +[tool.hatch.build.targets.wheel] +packages = ["src/sonilo_video_kit"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/sonilo-video-kit/src/sonilo_video_kit/__init__.py b/sonilo-video-kit/src/sonilo_video_kit/__init__.py new file mode 100644 index 0000000..d6409df --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/__init__.py @@ -0,0 +1,20 @@ +"""Video helpers for the Sonilo API (ffmpeg-based).""" +from .duck import ( + MAX_DUCKED_MIX_BYTES, MAX_DUCKING_DURATION_SECONDS, duck_music_under_speech, +) +from .errors import ( + DuckingFailedError, FfmpegError, FfmpegNotFoundError, VideoKitError, +) +from .generate import VideoMusicClient, generate_music_for_video +from .loudness import ( + DELIVERY_TARGET_LUFS, FALLBACK_MUSIC_LUFS, GAP_BELOW_VOICE_LU, OUTPUT_CEILING_DBFS, +) +from .mix import mix_with_video + +__all__ = sorted([ + "DELIVERY_TARGET_LUFS", "DuckingFailedError", "FALLBACK_MUSIC_LUFS", "FfmpegError", + "FfmpegNotFoundError", "GAP_BELOW_VOICE_LU", "MAX_DUCKED_MIX_BYTES", + "MAX_DUCKING_DURATION_SECONDS", "OUTPUT_CEILING_DBFS", "VideoKitError", + "VideoMusicClient", "duck_music_under_speech", "generate_music_for_video", + "mix_with_video", +]) diff --git a/sonilo-video-kit/src/sonilo_video_kit/_ducking_api.py b/sonilo-video-kit/src/sonilo_video_kit/_ducking_api.py new file mode 100644 index 0000000..08f9e33 --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/_ducking_api.py @@ -0,0 +1,496 @@ +"""Ducking API transport: submit / poll / download (ported from ducking-api.ts). + +Self-contained HTTP layer for the async `/v1/audio-ducking` endpoint. Three +concerns live here, matching the JS reference: + + - submit_ducking_job: POSTs the voice+music tracks. This is what CHARGES + the account, so it is deliberately NEVER retried (see its docstring). + - await_ducking_result: polls GET /v1/tasks/{id} until the task leaves + "processing", retrying transient (5xx / network) failures — the task + keeps running server-side no matter what happens to this client. + - download_ducked_mix: fetches the finished mix from its presigned URL + (a DIFFERENT, unauthenticated host — never routed through client._http, + so the customer's API key never reaches it), guarded against SSRF and + bounded by a byte cap / exact-size check. +""" +from __future__ import annotations + +import ipaddress +import os +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional, Protocol +from urllib.parse import quote, urlsplit + +import httpx +from sonilo.errors import error_from_response + +from ._ffmpeg import StrPath +from .errors import DuckingFailedError, VideoKitError + +Sleep = Callable[[float], None] + + +class DuckingClient(Protocol): + """The minimal client surface the ducking calls need — satisfied by + `sonilo.Sonilo`, whose `_http` is a configured httpx.Client (base_url + https://api.sonilo.com, Authorization header already set).""" + + _http: httpx.Client + + +@dataclass +class DuckingResult: + output_url: str + output_type: str + output_bytes: Optional[int] = None + + +# One initial try plus three retries. +_MAX_ATTEMPTS = 4 +_RETRY_BASE_SECONDS = 0.5 +_RETRY_MAX_SECONDS = 4.0 + +# Task/submit JSON envelopes are a few hundred bytes; 1 MB is orders of +# magnitude of headroom while still capping a compromised API trying to OOM +# the client. +MAX_JSON_BYTES = 1024 * 1024 + +# Per-attempt wall-clock cap for the mix download, mirroring ducking-api.ts's +# DEFAULT_DOWNLOAD_TIMEOUT_MS. +DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 120.0 + + +class _HttpStatusError(VideoKitError): + """A non-2xx from the (unauthed) download host. Carries `status_code` so + `_is_transient` classifies it exactly like the sonilo SDK's own + APIError (which also carries `status_code`).""" + + def __init__(self, status_code: int, what: str) -> None: + super().__init__(f"{what} (HTTP {status_code})") + self.status_code = status_code + + +class _DownloadTimeoutError(VideoKitError): + """A download attempt that blew its per-attempt wall-clock deadline. + Transient on purpose (see `_is_transient`): a stalled/slow attempt is + retried with a FRESH deadline, mirroring ducking-api.ts's + DownloadTimeoutError. httpx's own `timeout` only bounds the gap BETWEEN + chunks (it resets on every byte received), so a server dribbling bytes + just under that gap would otherwise stream forever; this bounds the + whole attempt's wall-clock lifetime instead.""" + + def __init__(self, timeout_seconds: float) -> None: + super().__init__( + f"The ducked-mix download did not complete within its {timeout_seconds}s " + "deadline and was aborted." + ) + self.timeout_seconds = timeout_seconds + + +def _retry_delay_seconds(attempt: int) -> float: + return min(_RETRY_BASE_SECONDS * (2 ** (attempt - 1)), _RETRY_MAX_SECONDS) + + +def _is_transient(err: BaseException) -> bool: + """Worth another go? Mirrors ducking-api.ts's isTransient: a blown + per-attempt download deadline is retried with a fresh one; a numeric + status >= 500 is transient (covers both the sonilo SDK's APIError and our + own _HttpStatusError); any other VideoKitError is terminal; anything else + (a network-level failure — connection reset, DNS, TLS) is transient.""" + if isinstance(err, _DownloadTimeoutError): + return True + status = getattr(err, "status_code", None) + if isinstance(status, int): + return status >= 500 + if isinstance(err, VideoKitError): + return False + return isinstance(err, Exception) + + +def _with_retry(op: "Callable[[], Any]", *, sleep: Sleep) -> Any: + attempt = 1 + while True: + try: + return op() + except Exception as err: # noqa: BLE001 - reclassified below + if attempt >= _MAX_ATTEMPTS or not _is_transient(err): + raise + sleep(_retry_delay_seconds(attempt)) + attempt += 1 + + +def _parse_json_capped(response: httpx.Response, what: str) -> Any: + """`response.json()`, but refusing to trust a body over MAX_JSON_BYTES.""" + if len(response.content) > MAX_JSON_BYTES: + raise VideoKitError( + f"The ducking API's {what} response exceeded {MAX_JSON_BYTES} bytes; " + "refusing to buffer it." + ) + return response.json() + + +# --------------------------------------------------------------------------- +# submit_ducking_job +# --------------------------------------------------------------------------- + + +def submit_ducking_job(client: DuckingClient, voice_path: StrPath, music_path: StrPath) -> str: + """POST the voice and music tracks to /v1/audio-ducking. Returns the task + id; the endpoint is async (submit + poll). + + Deliberately NOT retried, unlike the poll and the download below: the + POST is what CHARGES the account (the backend charges in the request + handler, before the background job is even spawned), and it carries no + idempotency key. A retry after a response we failed to read would risk + submitting — and paying for — the same job twice. Failing here is safe; + failing after here is what has to be recovered (see duck.py's rescue + path).""" + voice = Path(voice_path) + music = Path(music_path) + files = { + "voice_file": (voice.name, voice.read_bytes()), + "music_file": (music.name, music.read_bytes()), + } + response = client._http.post("/v1/audio-ducking", files=files) + if response.status_code >= 400: + raise error_from_response(response) + body = _parse_json_capped(response, "submit") + task_id = body.get("task_id") if isinstance(body, dict) else None + if not task_id: + raise VideoKitError("The ducking API accepted the request but returned no task_id") + return task_id + + +# --------------------------------------------------------------------------- +# await_ducking_result +# --------------------------------------------------------------------------- + + +def _extract_error_fields(body: "dict[str, Any]") -> "tuple[str, str]": + """The task's failure code/message. The backend's documented envelope + nests them under `error: {code, message}` (matching the sonilo SDK's + other task endpoints and ducking-api.ts's TaskBody); tolerate a flat + top-level `code`/`message` too, in case an older or alternate envelope + omits the wrapper.""" + error = body.get("error") + if isinstance(error, dict): + code = error.get("code") or "DUCKING_FAILED" + message = error.get("message") or "the ducking task failed" + else: + code = body.get("code") or "DUCKING_FAILED" + message = body.get("message") or "the ducking task failed" + return str(code), str(message) + + +def _dedup_failed_message(message: str, code: str, refunded: bool) -> str: + """Compose the final DuckingFailedError message, ported from errors.ts's + DuckingFailedError constructor (kept here rather than in errors.py: the + Python DuckingFailedError, ported earlier, stores its message as given — + this is the one call site that needs the composed text). + + The server derives `code` by splitting its own error_message on ":" + (error_message = "DUCKING_FAILED: audio processing failed"), so both + fields carry the code and a naive compose renders it twice. Strip the + "{code}:" prefix from `message` first.""" + prefix = f"{code}:" + detail = message[len(prefix):].strip() if message.startswith(prefix) else message + detail = detail or message + note = ( + " — the charge was refunded" + if refunded + else ( + " — the charge had not been reversed yet when the task was polled; the server " + "reverses it after marking the task failed, and retries a reversal that fails, " + "so it may still land. Check your usage before assuming you were billed for this." + ) + ) + return f"Ducking failed [{code}]: {detail}{note}" + + +def _poll_task(client: DuckingClient, task_id: str) -> "dict[str, Any]": + response = client._http.get(f"/v1/tasks/{quote(task_id, safe='')}") + if response.status_code >= 400: + raise error_from_response(response) + body = _parse_json_capped(response, "task") + if not isinstance(body, dict): + raise VideoKitError(f"Ducking task {task_id} returned a malformed response") + return body + + +def _sanitize_output_bytes(value: Any) -> Optional[int]: + """Only ever a POSITIVE integer or None. A real ducking artifact is never + 0 bytes, and a fractional/negative/wrong-typed value is not a byte + count — see the extensive comment on this exact check in ducking-api.ts.""" + if isinstance(value, bool): + return None + if isinstance(value, int) and value > 0: + return value + return None + + +def await_ducking_result( + client: DuckingClient, + task_id: str, + *, + poll_interval: float, + timeout: float, + deadline: Optional[float] = None, + sleep: Sleep = time.sleep, +) -> DuckingResult: + """Poll GET /v1/tasks/{id} until the task leaves `processing`. + + `deadline` is an absolute `time.monotonic()` ceiling; when omitted it is + computed from `timeout`. duck.py passes the same deadline into both this + call and download_ducked_mix, so one budget governs polling AND the + download together (see duck.ts's "ONE deadline governs the WHOLE + post-submit collection").""" + effective_deadline = deadline if deadline is not None else time.monotonic() + timeout + + while True: + body = _with_retry(lambda: _poll_task(client, task_id), sleep=sleep) + status = body.get("status") + + if status == "succeeded": + output_url = body.get("output_url") + if not output_url: + raise VideoKitError(f"Ducking task {task_id} succeeded but carried no output_url") + return DuckingResult( + output_url=output_url, + output_type=body.get("output_type") or "audio", + output_bytes=_sanitize_output_bytes(body.get("output_bytes")), + ) + + if status == "failed": + code, message = _extract_error_fields(body) + refunded = bool(body.get("refunded", False)) + raise DuckingFailedError( + _dedup_failed_message(message, code, refunded), code=code, refunded=refunded + ) + + if time.monotonic() + poll_interval >= effective_deadline: + raise VideoKitError(f"Ducking task {task_id} did not finish within {timeout}s") + sleep(poll_interval) + + +# --------------------------------------------------------------------------- +# assert_safe_download_url / download_ducked_mix +# --------------------------------------------------------------------------- + +_DECIMAL_DIGITS = set("0123456789") +_OCTAL_DIGITS = set("01234567") +_HEX_DIGITS = set("0123456789abcdefABCDEF") + + +def _parse_ipv4_number(label: str) -> Optional[int]: + """Parse one dot-separated label (or a whole no-dot host) as an IPv4 + "number", per the WHATWG URL standard's IPv4 number parser: a decimal, + `0x`/`0X`-prefixed hex, or legacy leading-zero octal integer. Returns + None if `label` is empty or is not such a number — i.e. an ordinary + domain label, which the WHATWG parser also leaves untouched. + + This is what a browser's URL parser runs BEFORE the IP-literal check + ever sees the host — see ducking-api.ts's isIpLiteralHost comment. + httpx/urlsplit + ipaddress.ip_address do none of this canonicalization, + so this Python port has to do it itself.""" + if not label: + return None + if len(label) >= 2 and label[0] == "0" and label[1] in ("x", "X"): + digits, base, allowed = label[2:], 16, _HEX_DIGITS + elif len(label) >= 2 and label[0] == "0": + digits, base, allowed = label[1:], 8, _OCTAL_DIGITS + else: + digits, base, allowed = label, 10, _DECIMAL_DIGITS + if digits == "": + return 0 + if not all(ch in allowed for ch in digits): + return None + return int(digits, base) + + +def _is_ip_literal_host(host: str) -> bool: + """True if `host` (already lowercased) denotes an IPv4/IPv6 literal in + ANY form a WHATWG-compliant URL parser (what JS's `new URL()` uses) + would canonicalize into one — not just the strict textual forms + `ipaddress.ip_address` accepts on its own. Mirrors ducking-api.ts's + `isIpLiteralHost`, whose intent relies on the browser URL parser having + already canonicalized the host before that check runs; this port has no + such parser in front of it, so it reimplements the canonicalization. + + Catches, in addition to the textbook dotted-quad / bracketed-v6 forms: + - a bare integer host in decimal, hex (`0x...`), or octal (leading + `0`) that fits in 32 bits, e.g. "2130706433" / "0x7f000001" / + "017700000001" — all 127.0.0.1; + - a dotted host whose every label is itself such a number, e.g. + "0x7f.0.0.1" or "0177.0.0.1"; + - a trailing dot on a dotted-quad, e.g. "127.0.0.1." (the DNS-root + dot; "127.0.0.1." resolves identically to "127.0.0.1", but + `ipaddress.ip_address` rejects the trailing-dot form outright). + + A host that is NOT an IP literal in any of these forms (an ordinary + domain) returns False and is left alone — a numeric-only label is not + a legal DNS TLD, so this cannot misclassify a real hostname.""" + stripped = host.rstrip(".") + if not stripped: + return False + + try: + ipaddress.ip_address(stripped) + return True + except ValueError: + pass + + if "." not in stripped: + # A bare integer host is the WHATWG IPv4 number parser applied to a + # single label: "2130706433", "0x7f000001", "017700000001" all take + # this path (and all denote 127.0.0.1). + number = _parse_ipv4_number(stripped) + return number is not None and 0 <= number <= 0xFFFFFFFF + + labels = stripped.split(".") + return all(_parse_ipv4_number(label) is not None for label in labels) + + +def assert_safe_download_url(url: str) -> None: + """Validate the presigned download URL before fetching it. + + `output_url` arrives in the task body, i.e. from the API — which a + compromise could turn hostile. Unchecked, it is an SSRF primitive. This + raises the bar against the obvious payloads: + - only `https` is allowed (a presigned R2 GET always is); + - IP-literal hosts (v4/v6) — including decimal/hex/octal and + trailing-dot encodings a browser's URL parser would canonicalize + into one, see `_is_ip_literal_host` — plus `localhost` and + `*.local` / `*.internal`, are refused: the cloud-metadata + (169.254.169.254), loopback, and internal-DNS targets an SSRF aims + at, never a real presigned host. + + What it does NOT stop: a public hostname that resolves to a private + address (DNS rebinding) — out of scope for a zero-dependency kit; the + download additionally disables redirects so a 200-looking URL cannot + 302 into internal infrastructure. + + On rejection: name only the scheme or host, never the full URL — its + query string carries the signing signature (a capability), and errors + get logged.""" + try: + parsed = urlsplit(url) + except ValueError as exc: + raise VideoKitError( + "The ducking API returned an output_url that is not a valid URL." + ) from exc + if parsed.scheme != "https": + raise VideoKitError( + f'The ducking API\'s output_url uses an unsupported scheme "{parsed.scheme}"; ' + "only https is allowed for the presigned download." + ) + host = (parsed.hostname or "").lower() + if not host: + raise VideoKitError("The ducking API returned an output_url with no host.") + + # Strip the DNS-root trailing dot(s) before the named-host blocklist, so + # "localhost." (which resolves identically to "localhost") cannot sneak + # through. Comparison-only: the fetch below still uses the original URL. + named_host = host.rstrip(".") + if ( + _is_ip_literal_host(host) + or named_host == "localhost" + or named_host.endswith(".local") + or named_host.endswith(".internal") + ): + raise VideoKitError( + f'The ducking API\'s output_url points at a non-public host "{host}", which is ' + "never a legitimate presigned download location; refusing to fetch it." + ) + + +def download_ducked_mix( + url: str, + dest_path: StrPath, + *, + max_bytes: int, + expected_bytes: Optional[int] = None, + timeout: Optional[float] = None, + deadline: Optional[float] = None, + sleep: Sleep = time.sleep, +) -> None: + """Download the finished mix, streamed to `dest_path` without ever + leaving partial bytes there. + + Deliberately uses a FRESH, unauthenticated httpx.Client, never + `client._http`: `url` is a presigned link on a different host, and + routing it through the authenticated client would put the customer's + API key on a request to that host. Redirects are disabled (a presigned + GET never legitimately 302s). Retried through transient failures — by + the time there is something to download, the task has succeeded and the + account has been charged, so losing the mix to one 503 would mean + paying twice for it.""" + assert_safe_download_url(url) + per_attempt = timeout if timeout is not None else DEFAULT_DOWNLOAD_TIMEOUT_SECONDS + dest = Path(dest_path) + + def _attempt() -> None: + attempt_timeout = per_attempt + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise VideoKitError( + "The ducked-mix download did not complete within the overall time " + "budget for this operation (shared with polling) and was stopped " + "before starting a fresh attempt past it." + ) + attempt_timeout = min(per_attempt, remaining) + + # Wall-clock ceiling for this WHOLE attempt (connect + stream), not + # just the inter-chunk gap httpx's own `timeout` bounds — see + # _DownloadTimeoutError. Read with time.monotonic() only, never + # wall-clock time.time(), so an NTP step can't shorten or extend it. + attempt_deadline = time.monotonic() + attempt_timeout + + tmp_path = dest.parent / f".{dest.name}.{uuid.uuid4().hex}.part" + with httpx.Client(follow_redirects=False, timeout=attempt_timeout) as http: + try: + with http.stream("GET", url) as response: + # >= 300, not just >= 400: `follow_redirects=False` means + # a 3xx comes back as an ordinary response object here + # (unlike JS's `redirect:"error"` fetch, which throws), + # so it must be rejected explicitly or a redirect body + # would be written out as the "mix". + if response.status_code >= 300: + raise _HttpStatusError( + response.status_code, "Could not download the ducked mix" + ) + total = 0 + with open(tmp_path, "wb") as handle: + for chunk in response.iter_bytes(): + # A server dribbling chunks just under httpx's + # inter-chunk read timeout never trips it (that + # timeout resets on every byte received); this + # wall-clock check bounds the attempt regardless + # of how the bytes are paced. + if time.monotonic() >= attempt_deadline: + raise _DownloadTimeoutError(attempt_timeout) + total += len(chunk) + if total > max_bytes: + raise VideoKitError( + "The ducked mix exceeded the maximum allowed size " + f"({max_bytes} bytes) and was refused." + ) + handle.write(chunk) + if expected_bytes is not None and total != expected_bytes: + raise VideoKitError( + f"The ducked mix download was {total} bytes but the ducking " + f"API declared exactly {expected_bytes} bytes; refusing this " + "truncated or altered download." + ) + os.replace(tmp_path, dest) + except Exception: + try: + os.remove(tmp_path) + except OSError: + pass + raise + + _with_retry(_attempt, sleep=sleep) diff --git a/sonilo-video-kit/src/sonilo_video_kit/_ffmpeg.py b/sonilo-video-kit/src/sonilo_video_kit/_ffmpeg.py new file mode 100644 index 0000000..56bb285 --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/_ffmpeg.py @@ -0,0 +1,465 @@ +"""ffmpeg/ffprobe subprocess layer (ported from ffmpeg.ts).""" +from __future__ import annotations + +import json +import math +import re +import subprocess +from dataclasses import dataclass +from os import PathLike +from typing import Any, Union + +from .errors import FfmpegError, FfmpegNotFoundError, VideoKitError + +_STDERR_TAIL_CHARS = 4096 +DEFAULT_TIMEOUT_SECONDS = 1200.0 + +StrPath = Union[str, "PathLike[str]"] + + +@dataclass +class ProcessResult: + stdout: str + stderr: str + + +def run_process( + binary: str, args: list[str], *, timeout: float = DEFAULT_TIMEOUT_SECONDS +) -> ProcessResult: + try: + proc = subprocess.run( + [binary, *args], + capture_output=True, + text=True, + timeout=timeout, + ) + except FileNotFoundError as exc: + raise FfmpegNotFoundError( + f"'{binary}' not found. Install ffmpeg or pass an explicit path.", + cause=exc, + ) from exc + except subprocess.TimeoutExpired as exc: + raw = exc.stderr + if isinstance(raw, bytes): + raw = raw.decode("utf-8", "replace") + tail = (raw or "")[-_STDERR_TAIL_CHARS:] + raise FfmpegError( + f"'{binary}' timed out after {timeout}s", + exit_code=None, + stderr_tail=tail, + ) from exc + except OSError as exc: + # ENOENT is FileNotFoundError (handled above); any other spawn error + # (e.g. EACCES on a non-executable binary) is wrapped, matching JS. + raise VideoKitError(f"Failed to run '{binary}': {exc}", cause=exc) from exc + + if proc.returncode != 0: + tail = (proc.stderr or "")[-_STDERR_TAIL_CHARS:] + raise FfmpegError( + f"'{binary}' exited with {proc.returncode}", + exit_code=proc.returncode, + stderr_tail=tail, + ) + return ProcessResult(stdout=proc.stdout or "", stderr=proc.stderr or "") + + +# ============================================================================ +# probe_video — ported from ffmpeg.ts::probeVideo and its helpers. +# +# See the JS source (packages/sonilo-video-kit/src/ffmpeg.ts) for the full +# rationale — "THE TIME MODEL" comment block there is the spec this section +# implements. Summary kept here only as an index into that reasoning: +# +# D = E - O +# D = the picture's end on the OUTPUT timeline (what we bill/mux to). +# E = the picture's end on the INPUT timeline: max(timestamp + duration) +# over the picture's own packets. +# O = the REBASE ORIGIN: the input timestamp ffmpeg maps to output zero. +# MPEG-TS/PS: the picture stream's own start_time (discontinuous clock). +# Every file-based container: format.start_time. +# +# Tiers (cheapest first, each applied ONLY where its conversion into E is the +# IDENTITY — i.e. no nonzero correction is trusted from a source's stated +# convention; a nonzero correction is always MEASURED instead): +# 1. Matroska/WebM per-track DURATION tag (only when origin == 0). +# 2. The streams[].duration FIELD (only when start_time - origin == 0, and +# the field isn't backfilled wholesale from the container's duration). +# 3. Measure E directly from the picture's own packets and rebase by O. +# There is deliberately no tier 4: an undeterminable picture end raises. +# ============================================================================ + +_START_AT_ORIGIN_EPSILON = 1e-3 + + +def _js_number(value: "str | None") -> float: + """Mimic JS `Number(value)`: NaN on unparsable, 0 on empty string, NaN on None.""" + if value is None: + return math.nan + text = value.strip() + if text == "": + return 0.0 + try: + return float(text) + except ValueError: + return math.nan + + +def _positive_seconds(value: "str | None") -> "float | None": + if value is None: + return None + seconds = _js_number(value) + return seconds if math.isfinite(seconds) and seconds > 0 else None + + +def _finite_seconds(value: "str | None") -> "float | None": + """Any finite number of seconds, including zero and negatives — unlike + `_positive_seconds`. `start_time` is legitimately 0 on almost every file.""" + if value is None: + return None + seconds = _js_number(value) + return seconds if math.isfinite(seconds) else None + + +_DURATION_TAG_RE = re.compile(r"^(\d+):(\d{1,2}):(\d{1,2}(?:\.\d+)?)$") + + +def _parse_duration_value(value: str) -> "float | None": + """`HH:MM:SS.nnnnnnnnn` (Matroska's per-stream DURATION tag) or plain seconds.""" + match = _DURATION_TAG_RE.match(value.strip()) + if match is None: + return _positive_seconds(value) + seconds = float(match.group(1)) * 3600 + float(match.group(2)) * 60 + float(match.group(3)) + return seconds if math.isfinite(seconds) and seconds > 0 else None + + +def _duration_tag_seconds(tags: "dict[str, Any] | None") -> "float | None": + """Matroska/WebM carry each track's own length as a TAG (`DURATION`, or a + language-suffixed `DURATION-eng`), authored per-track by the muxer — unlike + the `duration` field, never synthesized from the container.""" + if tags is None: + return None + for key, value in tags.items(): + name = key.upper() + if name != "DURATION" and not name.startswith("DURATION-"): + continue + seconds = _parse_duration_value(value) if isinstance(value, str) else None + if seconds is not None: + return seconds + return None + + +def _parse_rational(value: "str | None") -> "float | None": + if value is None: + return None + parts = value.split("/") + numerator = parts[0] + denominator = parts[1] if len(parts) > 1 else None + n = _js_number(numerator) + d = 1.0 if denominator is None else _js_number(denominator) + if not math.isfinite(n) or not math.isfinite(d) or n <= 0 or d <= 0: + return None + return n / d + + +def _frame_tolerance(stream: dict) -> float: + """How far a duration may legitimately sit from another measure of the same + picture: the last frame is displayed for 1/fps. 0.5s when fps is unknown + (itself a symptom — MPEG-TS reports avg_frame_rate=0/0).""" + fps = _parse_rational(stream.get("avg_frame_rate")) or _parse_rational(stream.get("r_frame_rate")) + if fps is None: + return 0.5 + return min(max(1 / fps + 0.05, 0.05), 2) + + +def _frames_accounted_per_track(stream: dict) -> bool: + """Does the demuxer account for this track's frames individually? A + presence check only — see the JS comment for why `nb_frames / + avg_frame_rate` is not a usable cross-check on the duration field.""" + return _positive_seconds(stream.get("nb_frames")) is not None + + +def _rebase_origin_seconds(format_: dict, stream: dict) -> float: + """O: the input timestamp ffmpeg maps to output zero.""" + names = (format_.get("format_name") or "").split(",") + discontinuous = any(n in ("mpegts", "mpegtsraw", "mpeg") for n in names) + declared = ( + _finite_seconds(stream.get("start_time")) + if discontinuous + else _finite_seconds(format_.get("start_time")) + ) + return declared if declared is not None else 0.0 + + +def _is_zero(seconds: float) -> bool: + return abs(seconds) <= _START_AT_ORIGIN_EPSILON + + +def _measure_picture_end(video: StrPath, stream_index: "int | None", ffprobe_path: str) -> "float | None": + """E, measured: the picture's end on the input timeline, from its own + packets. Demuxes, never decodes. Selects the picture by ABSOLUTE index, + not `v:0`, so attached cover art in `v:0`'s slot cannot be selected + instead of the genuine picture stream.""" + selector = str(stream_index) if stream_index is not None else "v:0" + result = run_process( + ffprobe_path, + [ + "-v", "error", + "-select_streams", selector, + "-show_entries", "packet=pts_time,dts_time,duration_time", + "-of", "csv=p=0", + str(video), + ], + ) + end: "float | None" = None + for line in result.stdout.split("\n"): + if line == "": + continue + parts = line.split(",") + pts_text = parts[0] if len(parts) > 0 else None + dts_text = parts[1] if len(parts) > 1 else None + dur_text = parts[2] if len(parts) > 2 else None + pts = _js_number(pts_text) + dts = _js_number(dts_text) + # "N/A" -> NaN. AVI has no pts; prefer pts, fall back to dts. + at = pts if math.isfinite(pts) else (dts if math.isfinite(dts) else None) + if at is None: + continue + dur = _js_number(dur_text) + until = at + (dur if math.isfinite(dur) and dur > 0 else 0) + if end is None or until > end: + end = until + return end + + +def _picture_duration_seconds( + video: StrPath, + stream: dict, + format_: dict, + container_seconds: float, + ffprobe_path: str, +) -> "float | None": + """D: the picture's end on the output timeline. See the module-level time + model comment. There is deliberately no tier 4 — callers raise instead of + guessing when this returns None.""" + origin = _rebase_origin_seconds(format_, stream) + tolerance = _frame_tolerance(stream) + + def plausible(seconds: "float | None") -> bool: + # The picture can never outlive the container: format.duration is the + # maximum over ALL streams, this one included. + return seconds is not None and seconds > 0 and seconds <= container_seconds + tolerance + + # TIER 1 — the Matroska/WebM DURATION tag. Applied only when the origin is + # zero, so the tag's "end position from container zero" convention needs + # no correction. + if _is_zero(origin): + tag_end = _duration_tag_seconds(stream.get("tags")) + if plausible(tag_end): + return tag_end + + # TIER 2 — the per-stream `duration` FIELD. Applied only when + # `start_time - origin == 0` (the identity correction), and the field + # isn't a wholesale backfill from the container's own duration (a picture + # whose packets are too sparse for libavformat to time individually). + field = _positive_seconds(stream.get("duration")) + stream_start = _finite_seconds(stream.get("start_time")) + if stream_start is None: + stream_start = 0.0 + field_correction = stream_start - origin + backfilled_from_container = ( + field is not None + and abs(field - container_seconds) <= tolerance + and not _frames_accounted_per_track(stream) + ) + if _is_zero(field_correction) and not backfilled_from_container and plausible(field): + return field + + # TIER 3 — measure E from the picture's own packets, and rebase it. + end = _measure_picture_end(video, stream.get("index"), ffprobe_path) + if end is None: + return None + duration = end - origin + return duration if duration > 0 else None + + +@dataclass +class VideoProbe: + duration_seconds: "float | None" + has_audio: bool + audio_codec: "str | None" + video_codec: "str | None" + video_duration_seconds: "float | None" + + +def probe_video(video: StrPath, ffprobe_path: str = "ffprobe") -> VideoProbe: + """Probe a video's container/stream metadata and the picture's own true + duration (see the time-model comment above `_picture_duration_seconds`). + + Raises VideoKitError when the container's duration is invalid, or when a + genuine picture stream is present but its duration cannot be determined — + never guesses from another stream's timing.""" + result = run_process( + ffprobe_path, + ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", str(video)], + ) + parsed = json.loads(result.stdout) + format_ = parsed.get("format") or {} + duration_seconds = _js_number(format_.get("duration")) + # Non-positive/unreadable duration: fail loudly rather than let ffmpeg + # exit 0 on a silent empty file. + if not math.isfinite(duration_seconds) or duration_seconds <= 0: + raise VideoKitError(f"Could not determine a valid duration for {video}; refusing to render") + + streams = parsed.get("streams") or [] + audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None) + # A genuine picture, not embedded cover art (disposition.attached_pic=1, + # the standard ID3/MP4 album-art tag). Must agree with mux_video_with_audio's + # `-map 0:V` selector, which excludes exactly those streams too. + video_stream = next( + ( + s + for s in streams + if s.get("codec_type") == "video" and (s.get("disposition") or {}).get("attached_pic") != 1 + ), + None, + ) + + video_duration_seconds: "float | None" = None + if video_stream is not None: + video_duration_seconds = _picture_duration_seconds( + video, video_stream, format_, duration_seconds, ffprobe_path + ) + if video_duration_seconds is None: + raise VideoKitError( + f"Could not determine how long the picture in {video} runs (its video stream " + "carries no usable duration, no DURATION tag, and no timestamped packets). " + "Refusing to guess: the ducking API bills on this figure, and the container's " + "own duration is the longest of ALL its streams, so guessing from it can " + "overcharge you. Re-encode the file (e.g. `ffmpeg -i in -c copy out.mp4`) and " + "try again." + ) + + return VideoProbe( + duration_seconds=duration_seconds, + has_audio=audio_stream is not None, + audio_codec=audio_stream.get("codec_name") if audio_stream is not None else None, + video_codec=video_stream.get("codec_name") if video_stream is not None else None, + video_duration_seconds=video_duration_seconds, + ) + + +_LUFS_RE = re.compile(r"I:\s*(-?\d+(?:\.\d+)?)\s*LUFS") + + +def measure_integrated_lufs(audio_path: StrPath, ffmpeg_path: str = "ffmpeg") -> "float | None": + """Integrated LUFS via ebur128. NEVER raises — None means "unmeasurable".""" + try: + result = run_process( + ffmpeg_path, + ["-hide_banner", "-nostats", "-i", str(audio_path), "-af", "ebur128", "-f", "null", "-"], + ) + except Exception: + return None + matches = _LUFS_RE.findall(result.stderr) + if not matches: + return None + try: + value = float(matches[-1]) + except ValueError: + return None + return value if math.isfinite(value) else None + + +def extract_audio( + video: StrPath, + out_path: StrPath, + audio_codec: "str | None", + ffmpeg_path: str = "ffmpeg", + trim_to_seconds: "float | None" = None, +) -> None: + """Pre-extract the video's audio track to a standalone file. Stream-copies + when the source is already aac (bounded, accepted overbill of up to one + aac frame on `-t`); re-encodes otherwise, which lands exact.""" + codec_args = ["-c:a", "copy"] if audio_codec == "aac" else ["-c:a", "aac", "-b:a", "192k"] + trim_args = ( + ["-t", f"{trim_to_seconds:.3f}"] + if trim_to_seconds is not None and math.isfinite(trim_to_seconds) and trim_to_seconds > 0 + else [] + ) + run_process( + ffmpeg_path, + ["-y", "-i", str(video), "-vn", *codec_args, *trim_args, str(out_path)], + ) + + +@dataclass +class MuxFeasibility: + ok: bool + reason: "str | None" = None + + +# The silent audio the mux dry run encodes to aac. Synthesised (lavfi), never +# taken from the video itself: the question is whether the CONTAINER can hold +# the copied picture and an aac track, and silence cannot fail to decode for +# an unrelated reason. Bounded (d=0.05) so the input EOFs on its own. +_MUX_PROBE_SILENCE = "anullsrc=r=44100:cl=stereo:d=0.05" + + +def probe_mux_feasibility(video: StrPath, out_path: StrPath, ffmpeg_path: str = "ffmpeg") -> MuxFeasibility: + """Can this picture be stream-copied into this container at all? Answered + by dry-running the real mux shape (`-map 0:V`, `-c:v copy`, `-c:a aac`, + `-frames:v 1`) against synthetic silence, to a disposable output carrying + the caller's own extension. NEVER raises except when ffmpeg itself is + missing — that is not a verdict on the video.""" + try: + run_process( + ffmpeg_path, + [ + "-y", + "-v", "error", + "-i", str(video), + "-f", "lavfi", + "-i", _MUX_PROBE_SILENCE, + "-map", "0:V", + "-map", "1:a", + "-c:v", "copy", + "-c:a", "aac", + "-frames:v", "1", + str(out_path), + ], + ) + return MuxFeasibility(ok=True) + except FfmpegError as exc: + # The FIRST lines (the cause), not the usual last three (the + # consequences) — under `-v error` ffmpeg prints the cause first. + lines = [line for line in exc.stderr_tail.strip().split("\n") if line.strip()] + return MuxFeasibility(ok=False, reason=" | ".join(lines[:3])) + + +def mux_video_with_audio( + video: StrPath, + audio_path: StrPath, + out_path: StrPath, + duration_seconds: float, + ffmpeg_path: str = "ffmpeg", +) -> None: + """Replace a video's audio with `audio_path`, copying the picture + untouched. `-map 0:V` (capital V) excludes attached cover art. The audio + is trimmed and silence-padded to `duration_seconds` (never `-shortest`), + so a mix that runs short can never truncate the picture.""" + dur = f"{duration_seconds:.3f}" + run_process( + ffmpeg_path, + [ + "-y", + "-i", str(video), + "-i", str(audio_path), + "-filter_complex", + f"[1:a]atrim=end={dur},asetpts=N/SR/TB,apad=whole_dur={dur}[aout]", + "-map", "0:V", + "-map", "[aout]", + "-c:v", "copy", + "-c:a", "aac", + str(out_path), + ], + ) diff --git a/sonilo-video-kit/src/sonilo_video_kit/duck.py b/sonilo-video-kit/src/sonilo_video_kit/duck.py new file mode 100644 index 0000000..32fea36 --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/duck.py @@ -0,0 +1,344 @@ +"""duck_music_under_speech: orchestration + rescue (ported from duck.ts). + +Ducks generated/supplied music under the speech already present in a video. +The ducking itself runs on the Sonilo API — a PAID endpoint, metered on the +uploaded voice track's duration. Only the extracted audio track is uploaded +(trimmed to the picture's length, so the billed duration equals the +delivered one); the picture stays local and is copied, never re-encoded. + +Preconditions the API cannot satisfy (no audio track, no picture, a video or +music track longer than MAX_DUCKING_DURATION_SECONDS) — and preconditions +the local mux and filesystem cannot satisfy (an `output` with no extension, +or in a directory that does not exist or is not writable) — throw before +anything is uploaded, and so before anything is charged. There is +deliberately no fallback to mix_with_video: a caller who asked for ducking +must never silently receive an un-ducked file. +""" +from __future__ import annotations + +import os +import shutil +import tempfile +import time +import uuid +from pathlib import Path +from typing import NoReturn, Optional, Union + +from ._ducking_api import ( + DuckingClient, + await_ducking_result, + download_ducked_mix, + submit_ducking_job, +) +from ._ffmpeg import StrPath, extract_audio, mux_video_with_audio, probe_mux_feasibility, probe_video +from .errors import DuckingFailedError, VideoKitError + +# The ducking API rejects voice, video, and music tracks longer than this. +MAX_DUCKING_DURATION_SECONDS = 360 + +# Absolute hard ceiling on the ducked-mix download — the anti-DoS floor that a +# hostile/compromised/MITM'd R2 host CANNOT raise. The kit only ever uploads +# an extracted audio track (<= MAX_DUCKING_DURATION_SECONDS), so the finished +# artifact is always an audio wav of at most a few dozen MB; 300 MB is +# generous headroom. The server's own expected size (output_bytes) is +# additionally clamped UNDER this ceiling in effective_download_cap; this +# constant is the last-resort bound that holds even when output_bytes is +# absent or the server itself is lying. +MAX_DUCKED_MIX_BYTES = 300 * 1024 * 1024 + +# Slack allowed above the server's exact `output_bytes` before a download is +# treated as an anomaly. A FIXED 64 KB, not a percentage: the authenticated +# channel gave us the EXACT artifact size, so the only legitimate excess is +# negligible container/transfer framing overhead. +_OUTPUT_BYTES_MARGIN = 64 * 1024 + +_DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +_DEFAULT_TIMEOUT_SECONDS = 10 * 60.0 + +AudioInput = Union[bytes, bytearray, StrPath] + + +def effective_download_cap(output_bytes: Optional[int]) -> int: + """The cap actually enforced on the download. With the server's + authenticated `output_bytes`, a body more than a hair over it is an + anomaly and is rejected even though it sits under the hard ceiling; + without it (older backend, or a non-positive/wrong-typed value), only + the hard ceiling applies.""" + if isinstance(output_bytes, int) and not isinstance(output_bytes, bool) and output_bytes > 0: + return min(output_bytes + _OUTPUT_BYTES_MARGIN, MAX_DUCKED_MIX_BYTES) + return MAX_DUCKED_MIX_BYTES + + +def _default_client() -> DuckingClient: + from sonilo import Sonilo + + return Sonilo() + + +def _paid_note(task_id: str) -> str: + """Plain words for "the API has run, you have been billed, and the mix + is still yours" — the sentence every post-submit failure has to end + with.""" + return ( + f"You have ALREADY BEEN CHARGED for ducking task {task_id}: the API bills at submit, " + "and the task keeps running to completion server-side no matter what happens to this " + "call. Do not call duck_music_under_speech again for this video -- that submits a NEW " + f"task and charges you a second time. Poll GET /v1/tasks/{task_id} instead (with the " + 'same client) until it reports "succeeded", and download the output_url it hands back: ' + "that is a re-fetch of the mix you have paid for, not a new charge." + ) + + +def _rethrow_with_task_id(err: BaseException, task_id: str) -> BaseException: + """Anything that goes wrong after a successful submit has to carry the + task id out with it — the mix is paid for, finished, and sitting in R2; + losing the id would leave re-running (and re-charging) as the only way + forward. DuckingFailedError is passed through untouched: there the + SERVER's own processing failed, so there is no finished mix to re-fetch + and its own `refunded` flag already reports the charge.""" + if isinstance(err, DuckingFailedError): + return err + reason = str(err) + return VideoKitError( + f"The ducking API ran, but the finished mix could not be collected: {reason}. " + f"{_paid_note(task_id)}", + cause=err, + ) + + +def _free_rescue_path(output: Path) -> Path: + """Where to rescue the paid mix to, without ever clobbering a rescued + mix from an EARLIER failed run: `.ducked.wav` may already hold + an irreplaceable, paid-for mix this call did not write.""" + candidates = [output.with_name(f"{output.name}.ducked.wav")] + for n in range(1, 51): + candidates.append(output.with_name(f"{output.name}.ducked.{n}.wav")) + candidates.append(output.with_name(f"{output.name}.ducked.{uuid.uuid4().hex}.wav")) + for candidate in candidates: + if not candidate.exists(): + return candidate + return candidates[-1] + + +def _place_atomically(source_path: "StrPath", dest_path: Path) -> None: + """Copy `source_path` to `dest_path` without ever leaving a truncated + file at `dest_path`: copy to a sibling temp file in the same directory + (same filesystem, so the rename can't EXDEV) and rename it into place.""" + tmp_path = dest_path.parent / f".{dest_path.name}.{uuid.uuid4().hex}.tmp" + try: + shutil.copyfile(source_path, tmp_path) + os.replace(tmp_path, dest_path) + except Exception: + try: + tmp_path.unlink() + except OSError: + pass + raise + + +def _rescue_and_raise( + stage: str, cause: BaseException, ducked_path: Path, output: Path, task_id: str +) -> NoReturn: + """The ducking API call that produced `ducked_path` has already been + billed on the video's duration, so any failure between "the mix is + downloaded" and "the mix is safely at `output`" (a mux that can't hold + the video's codec, a full disk, a missing/read-only output directory) + must not also destroy that paid-for mix. Copy it next to `output` + before re-raising, and say so in the thrown error.""" + reason = str(cause) + recovered_path = output.with_name(f"{output.name}.ducked.wav") + try: + recovered_path = _free_rescue_path(output) + _place_atomically(ducked_path, recovered_path) + rescue_note = ( + f"The ducked audio was saved to {recovered_path} so you can recover it locally " + "(e.g. retry the mux, or move the file into place yourself) instead of calling " + "duck_music_under_speech again, which would incur another charge. You have already " + f"been charged for ducking task {task_id}." + ) + except Exception as rescue_err: # noqa: BLE001 - folded into the raised error below + rescue_reason = str(rescue_err) + rescue_note = ( + f"Attempting to also save the ducked audio to {recovered_path} ALSO failed " + f"({rescue_reason}), so the mix could not be recovered locally. {_paid_note(task_id)}" + ) + raise VideoKitError( + f"{stage} failed, after the ducking API had already run and been billed for this " + f"video's duration. {rescue_note} Original error: {reason}", + cause=cause, + ) from cause + + +def duck_music_under_speech( + *, + video: StrPath, + audio: AudioInput, + output: StrPath, + client: Optional[DuckingClient] = None, + poll_interval: float = _DEFAULT_POLL_INTERVAL_SECONDS, + timeout: float = _DEFAULT_TIMEOUT_SECONDS, + ffmpeg_path: str = "ffmpeg", + ffprobe_path: str = "ffprobe", +) -> str: + if not video: + raise VideoKitError("video is required") + if not output: + raise VideoKitError("output is required") + + output_path = Path(output) + # A `output` with no extension leaves ffmpeg with no way to infer a + # muxer for the temp mux target later. Knowable now — and only now is + # it free: by mux time the API call has already been billed. + output_extension = output_path.suffix + if len(output_extension) < 2: + suggestion = str(output_path).rstrip(".") + raise VideoKitError( + f'output "{output}" has no file extension, so ffmpeg cannot tell which container ' + f'to write. Give it one (e.g. "{suggestion}.mp4").' + ) + + # `output`'s directory has to exist and be writable, and this is the + # moment it is free to say so: without this guard the job would be + # submitted, the account CHARGED, the mux would succeed -- and only then + # would placement (and the rescue, which writes next to `output`) fail. + output_dir = output_path.parent + if not os.path.isdir(output_dir) or not os.access(output_dir, os.W_OK): + raise VideoKitError( + f'output "{output}" is in a directory that does not exist or cannot be written to ' + f'({output_dir}). Create it first (e.g. `mkdir -p {output_dir}`): the ducking API ' + "is billed at submit, so discovering this after the call would cost you the charge " + "AND leave nowhere to put the mix you paid for." + ) + + if isinstance(audio, (bytes, bytearray)): + if len(audio) == 0: + raise VideoKitError("audio is required") + elif not audio: + raise VideoKitError("audio is required") + + # Guards run before the client is constructed: an unusable video reports + # the real problem even when SONILO_API_KEY is unset, and nothing is + # uploaded or charged for an input the API would reject anyway. + probe = probe_video(video, ffprobe_path) + if not probe.has_audio: + raise VideoKitError( + f"{video} has no audio track, so there is no speech to duck under. " + "Use mix_with_video to lay music over a silent video." + ) + video_duration = probe.video_duration_seconds + if video_duration is None: + raise VideoKitError( + f"{video} has no video stream (embedded cover art does not count), so there is no " + "picture to mux the ducked audio back onto. Duck an audio-only file with the " + "ducking API directly, or pass a real video." + ) + # Guard, bill, and mux on the PICTURE's duration, never the container's + # (format.duration is the maximum over all streams): the server bills + # the uploaded voice track, and the deliverable is only as long as the + # picture. + if video_duration > MAX_DUCKING_DURATION_SECONDS: + raise VideoKitError( + f"{video} runs {video_duration:.1f}s; the ducking API accepts at most " + f"{MAX_DUCKING_DURATION_SECONDS}s. Use mix_with_video for longer videos." + ) + + work_dir = Path(tempfile.mkdtemp(prefix="sonilo-video-kit-duck-")) + try: + # Can the picture actually be stream-copied into the caller's + # container? probe_video succeeding does not prove it: dry-run the + # real mux shape here, BEFORE the client is constructed, so an + # unmuxable video is reported before anything is charged. + feasibility = probe_mux_feasibility( + video, work_dir / f"feasibility{output_extension}", ffmpeg_path + ) + if not feasibility.ok: + codec = probe.video_codec or "its video stream" + raise VideoKitError( + f'{video}\'s picture ({codec}) cannot be stream-copied into a ' + f'"{output_extension}" container, so muxing the ducked audio back onto it ' + "would fail. duck_music_under_speech never re-encodes your picture, so the " + "container has to be able to hold it as it is. Try a different output " + 'extension (e.g. ".mkv" or ".mp4"), or re-encode the video first ' + "(e.g. `ffmpeg -i in -c:v libx264 -c:a copy fixed.mp4`). Refused before the " + f"ducking API was called, so you have NOT been charged. ffmpeg said: " + f"{feasibility.reason}" + ) + + # The music obeys the same cap as the video (the server applies it + # too), so probe it here rather than uploading up to 300 MB the + # server would only reject. + if isinstance(audio, (str, os.PathLike)): + music_path: StrPath = audio + else: + music_path = work_dir / "music.mp3" + Path(music_path).write_bytes(bytes(audio)) + + music_probe = probe_video(music_path, ffprobe_path) + music_duration = music_probe.duration_seconds or 0.0 + if music_duration > MAX_DUCKING_DURATION_SECONDS: + raise VideoKitError( + f"The music runs {music_duration:.1f}s; the ducking API accepts at most " + f"{MAX_DUCKING_DURATION_SECONDS}s. Use a shorter music track." + ) + + active_client: DuckingClient = client if client is not None else _default_client() + + # Upload the audio track, never the picture. Trimmed to the + # picture's length: the server bills exactly what it is given, and + # the deliverable stays as long as the picture, so trimming here + # makes the billed duration equal the delivered duration. + voice_path = work_dir / "voice.m4a" + extract_audio(video, voice_path, probe.audio_codec, ffmpeg_path, video_duration) + + # THE ACCOUNT IS CHARGED HERE. From this line on, every failure + # (a poll that errors, a download that stays broken, a timeout) is a + # failure that costs the customer money while the task keeps + # running server-side. Nothing below may throw away the task id. + task_id = submit_ducking_job(active_client, voice_path, music_path) + + ducked_path = work_dir / "ducked.wav" + try: + # ONE deadline governs the WHOLE post-submit collection: the + # caller's timeout is the budget for polling AND the download + # together, not a fresh full timeout for each. + overall_deadline = time.monotonic() + timeout + result = await_ducking_result( + active_client, + task_id, + poll_interval=poll_interval, + timeout=timeout, + deadline=overall_deadline, + ) + # Only an extracted audio track is ever uploaded, so the server + # should never answer with anything but "audio". + if result.output_type != "audio": + raise VideoKitError( + f'The ducking API returned output_type "{result.output_type}" for task ' + f'{task_id}, but only "audio" is expected: this client always uploads ' + "just the extracted audio track, never the picture." + ) + download_ducked_mix( + result.output_url, + ducked_path, + max_bytes=effective_download_cap(result.output_bytes), + expected_bytes=result.output_bytes, + deadline=overall_deadline, + ) + except Exception as err: + raise _rethrow_with_task_id(err, task_id) from err + + # Mux into work_dir first, never straight to `output`: a failure + # partway through would otherwise leave a truncated file where the + # caller expects a deliverable. + muxed_path = work_dir / f"muxed{output_extension}" + stage = f"Muxing the ducked audio onto {video}" + try: + mux_video_with_audio(video, ducked_path, muxed_path, video_duration, ffmpeg_path) + stage = f"Placing the finished mix at {output}" + _place_atomically(muxed_path, output_path) + except Exception as err: + _rescue_and_raise(stage, err, ducked_path, output_path, task_id) + + return str(output_path) + finally: + shutil.rmtree(work_dir, ignore_errors=True) diff --git a/sonilo-video-kit/src/sonilo_video_kit/errors.py b/sonilo-video-kit/src/sonilo_video_kit/errors.py new file mode 100644 index 0000000..01b6abb --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/errors.py @@ -0,0 +1,44 @@ +"""Error types for sonilo-video-kit (ported from errors.ts).""" +from __future__ import annotations + +from typing import Optional + + +class VideoKitError(Exception): + """Base class for all sonilo-video-kit errors.""" + + def __init__(self, message: str, *, cause: Optional[BaseException] = None) -> None: + super().__init__(message) + self.cause = cause + + +class FfmpegNotFoundError(VideoKitError): + """Raised when the ffmpeg/ffprobe binary cannot be found on PATH.""" + + +class FfmpegError(VideoKitError): + """Raised when ffmpeg/ffprobe exits non-zero or times out.""" + + def __init__( + self, + message: str, + *, + exit_code: Optional[int] = None, + stderr_tail: str = "", + cause: Optional[BaseException] = None, + ) -> None: + super().__init__(message, cause=cause) + self.exit_code = exit_code + self.stderr_tail = stderr_tail + + +class DuckingFailedError(VideoKitError): + """Raised when the server marks a ducking task as failed.""" + + def __init__( + self, message: str, *, code: str = "", refunded: bool = False, + cause: Optional[BaseException] = None, + ) -> None: + super().__init__(message, cause=cause) + self.code = code + self.refunded = refunded diff --git a/sonilo-video-kit/src/sonilo_video_kit/generate.py b/sonilo-video-kit/src/sonilo_video_kit/generate.py new file mode 100644 index 0000000..f3eff7e --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/generate.py @@ -0,0 +1,27 @@ +"""generate_music_for_video (ported from generate.ts).""" +from __future__ import annotations + +import os +from typing import Any, List, Optional, Protocol + + +class _V2M(Protocol): + def generate(self, *, video: Any = None, prompt: Optional[str] = None, + segments: Optional[List[Any]] = None) -> Any: ... + + +class VideoMusicClient(Protocol): + video_to_music: _V2M + + +def generate_music_for_video( + video: "str | os.PathLike[str]", + *, + prompt: Optional[str] = None, + segments: Optional[List[Any]] = None, + client: Optional[VideoMusicClient] = None, +) -> Any: + if client is None: + from sonilo import Sonilo + client = Sonilo() + return client.video_to_music.generate(video=video, prompt=prompt, segments=segments) diff --git a/sonilo-video-kit/src/sonilo_video_kit/loudness.py b/sonilo-video-kit/src/sonilo_video_kit/loudness.py new file mode 100644 index 0000000..939ef04 --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/loudness.py @@ -0,0 +1,28 @@ +"""Loudness math ported 1:1 from loudness.ts (no I/O).""" +from __future__ import annotations + +FALLBACK_MUSIC_LUFS = -16.0 +OUTPUT_CEILING_DBFS = -1.0 +SLIDER_CENTER = 0.5 +SLIDER_SPAN_DB = 24.0 +GAP_BELOW_VOICE_LU = 4.0 +DELIVERY_TARGET_LUFS = -14.0 +MAX_DELIVERY_BOOST_DB = 12.0 + + +def db_to_linear(db: float) -> float: + return 10 ** (db / 20) + + +def offset_db(slider01: float) -> float: + return (slider01 - SLIDER_CENTER) * SLIDER_SPAN_DB + + +def gap_gain(bed_lufs: float, music_lufs: float, slider01: float) -> float: + if slider01 <= 0: + return 0.0 + return db_to_linear(bed_lufs + offset_db(slider01) - music_lufs) + + +def original_final_gain(slider01: float) -> float: + return max(0.0, min(slider01, 1.0)) diff --git a/sonilo-video-kit/src/sonilo_video_kit/mix.py b/sonilo-video-kit/src/sonilo_video_kit/mix.py new file mode 100644 index 0000000..e0844d8 --- /dev/null +++ b/sonilo-video-kit/src/sonilo_video_kit/mix.py @@ -0,0 +1,166 @@ +"""mix_with_video (ported from mix.ts) — local ffmpeg loudness-matched mix.""" +from __future__ import annotations + +import math +import os +import shutil +import tempfile +from pathlib import Path +from typing import Optional, Union + +from ._ffmpeg import StrPath, extract_audio, measure_integrated_lufs, probe_video, run_process +from .errors import VideoKitError +from .loudness import ( + DELIVERY_TARGET_LUFS, + FALLBACK_MUSIC_LUFS, + GAP_BELOW_VOICE_LU, + MAX_DELIVERY_BOOST_DB, + OUTPUT_CEILING_DBFS, + db_to_linear, + gap_gain, + original_final_gain, +) + +# ebur128 reports digital silence around its gate floor (-70 LUFS on modern +# ffmpeg) rather than -inf. An anchor that quiet means "no usable original +# signal" — anchoring to it would mute the music, so fall back to the same +# reference used when the video has no audio track at all. +_SILENCE_FLOOR_LUFS = -60.0 + +AudioInput = Union[StrPath, bytes, bytearray] + + +def _assert_slider(name: str, value: float) -> None: + if not math.isfinite(value) or value < 0 or value > 1: + raise VideoKitError(f"{name} must be between 0 and 1 (got {value})") + + +def mix_with_video( + *, + video: StrPath, + audio: AudioInput, + output: StrPath, + music_volume: float = 0.5, + original_volume: float = 1.0, + loudness_match: bool = True, + normalize: bool = True, + ffmpeg_path: str = "ffmpeg", + ffprobe_path: str = "ffprobe", +) -> str: + if not video: + raise VideoKitError("video is required") + if not output: + raise VideoKitError("output is required") + _assert_slider("music_volume", music_volume) + _assert_slider("original_volume", original_volume) + + output_path = Path(output) + # Placed under the output's own parent dir (not the system temp dir) so a + # leak here would show up right next to the render — and so cleanup is + # never left to a shared tmp filesystem. + work_dir = Path(tempfile.mkdtemp(prefix="sonilo-video-kit-", dir=str(output_path.parent))) + try: + # Music input: bytes are written to a temp file (ffmpeg needs a seekable input). + if isinstance(audio, (str, os.PathLike)): + music_path: StrPath = audio + else: + music_path = work_dir / "music.mp3" + Path(music_path).write_bytes(bytes(audio)) + + probe = probe_video(video, ffprobe_path) + + # Pre-extract original audio (never mix straight from the video input — + # muxer deadlock risk on large files; see _ffmpeg.py::extract_audio). + original_path: Optional[Path] = None + if probe.has_audio and original_volume > 0: + original_path = work_dir / "original.m4a" + extract_audio(video, original_path, probe.audio_codec, ffmpeg_path) + + # Gains: matched path measures LUFS; any failure degrades to legacy + # (slider = absolute gain), mirroring sonilo-web's never-throw analyzer. + music_gain = music_volume + original_gain = original_volume + if loudness_match: + music_lufs = measure_integrated_lufs(music_path, ffmpeg_path) + raw_anchor = ( + measure_integrated_lufs(original_path, ffmpeg_path) + if original_path is not None + else FALLBACK_MUSIC_LUFS + ) + anchor_lufs = ( + FALLBACK_MUSIC_LUFS + if raw_anchor is not None and raw_anchor <= _SILENCE_FLOOR_LUFS + else raw_anchor + ) + if music_lufs is not None and anchor_lufs is not None: + music_gain = gap_gain(anchor_lufs - GAP_BELOW_VOICE_LU, music_lufs, music_volume) + original_gain = original_final_gain(original_volume) + + ceiling = f"{db_to_linear(OUTPUT_CEILING_DBFS):.6f}" + limiter = f"alimiter=limit={ceiling}:level=disabled" + mixed_path = work_dir / "mixed.mp4" if normalize else output_path + # Cap audio to the probed video duration instead of -shortest: -shortest + # truncates the picture to whichever input is shorter, which cuts the + # video down to a too-short music track. atrim/apad instead pad short + # music with silence and let ffmpeg cut only excess audio. + dur = f"{probe.duration_seconds:.3f}" + + if original_path is not None: + inputs = ["-i", str(video), "-i", str(music_path), "-i", str(original_path)] + filter_ = ( + f"[1:a]volume={music_gain:.6f}[m];" + f"[2:a]volume={original_gain:.6f}[o];" + f"[m][o]amix=inputs=2:duration=longest:normalize=0," + f"atrim=end={dur},asetpts=N/SR/TB,apad=whole_dur={dur},{limiter}[aout]" + ) + else: + inputs = ["-i", str(video), "-i", str(music_path)] + filter_ = ( + f"[1:a]volume={music_gain:.6f},atrim=end={dur},asetpts=N/SR/TB," + f"apad=whole_dur={dur},{limiter}[aout]" + ) + + run_process( + ffmpeg_path, + [ + "-y", *inputs, + "-filter_complex", filter_, + "-map", "0:v", "-map", "[aout]", + "-c:v", "copy", "-c:a", "aac", + str(mixed_path), + ], + ) + + if normalize: + _delivery_normalize(mixed_path, output_path, ffmpeg_path) + return str(output_path) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def _delivery_normalize(in_path: Path, out_path: Path, ffmpeg_path: str) -> None: + """One static gain pass to land the finished file on DELIVERY_TARGET_LUFS. + Static volume, never dynamic loudnorm (it would breathe). Best-effort: + any failure keeps the un-normalized render.""" + try: + lufs = measure_integrated_lufs(in_path, ffmpeg_path) + if lufs is None: + shutil.copyfile(in_path, out_path) + return + gain_db = min(DELIVERY_TARGET_LUFS - lufs, MAX_DELIVERY_BOOST_DB) + if abs(gain_db) < 0.1: + shutil.copyfile(in_path, out_path) + return + ceiling = f"{db_to_linear(OUTPUT_CEILING_DBFS):.6f}" + run_process( + ffmpeg_path, + [ + "-y", "-i", str(in_path), + "-c:v", "copy", + "-af", f"volume={gain_db:.2f}dB,alimiter=limit={ceiling}:level=disabled", + "-c:a", "aac", + str(out_path), + ], + ) + except Exception: + shutil.copyfile(in_path, out_path) diff --git a/sonilo-video-kit/tests/__init__.py b/sonilo-video-kit/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sonilo-video-kit/tests/test_duck.py b/sonilo-video-kit/tests/test_duck.py new file mode 100644 index 0000000..e77b602 --- /dev/null +++ b/sonilo-video-kit/tests/test_duck.py @@ -0,0 +1,24 @@ +import pytest +from sonilo_video_kit import ( + duck_music_under_speech, MAX_DUCKING_DURATION_SECONDS, MAX_DUCKED_MIX_BYTES, +) +from sonilo_video_kit.duck import effective_download_cap + + +def test_constants(): + assert MAX_DUCKING_DURATION_SECONDS == 360 + assert MAX_DUCKED_MIX_BYTES == 300 * 1024 * 1024 + + +def test_effective_cap_clamps_to_server_bytes(): + # min(output_bytes + 64KB, MAX) + assert effective_download_cap(1000) == 1000 + 64 * 1024 + assert effective_download_cap(None) == MAX_DUCKED_MIX_BYTES + assert effective_download_cap(10**12) == MAX_DUCKED_MIX_BYTES + + +def test_missing_output_extension_rejected_before_charge(tmp_path): + v = tmp_path / "in.mp4"; v.write_bytes(b"x") + with pytest.raises(Exception): + duck_music_under_speech(video=v, audio=b"music", output=tmp_path / "noext", + client=object()) # must fail on validation, never touch client diff --git a/sonilo-video-kit/tests/test_ducking_api.py b/sonilo-video-kit/tests/test_ducking_api.py new file mode 100644 index 0000000..7002222 --- /dev/null +++ b/sonilo-video-kit/tests/test_ducking_api.py @@ -0,0 +1,166 @@ +import time + +import httpx +import pytest +import respx +from sonilo import Sonilo +from sonilo_video_kit import _ducking_api as ducking_api_module +from sonilo_video_kit._ducking_api import ( + assert_safe_download_url, submit_ducking_job, await_ducking_result, download_ducked_mix, +) +from sonilo_video_kit.errors import DuckingFailedError, VideoKitError + + +def _client(): + return Sonilo(api_key="test-key") # no network at construction + + +@respx.mock +def test_submit_returns_task_id(tmp_path): + route = respx.post("https://api.sonilo.com/v1/audio-ducking").mock( + return_value=httpx.Response(200, json={"task_id": "t_123"}) + ) + voice = tmp_path / "v.wav"; voice.write_bytes(b"RIFFvoice") + music = tmp_path / "m.wav"; music.write_bytes(b"RIFFmusic") + tid = submit_ducking_job(_client(), voice, music) + assert tid == "t_123" + assert route.called + + +@respx.mock +def test_poll_until_succeeded(): + respx.get("https://api.sonilo.com/v1/tasks/t_1").mock( + side_effect=[ + httpx.Response(200, json={"status": "processing"}), + httpx.Response(200, json={"status": "succeeded", + "output_url": "https://cdn.example.com/x.wav", + "output_type": "audio", "output_bytes": 10}), + ] + ) + res = await_ducking_result(_client(), "t_1", poll_interval=0.0, timeout=5.0, + sleep=lambda *_: None) + assert res.output_type == "audio" + assert res.output_url == "https://cdn.example.com/x.wav" + assert res.output_bytes == 10 + + +@respx.mock +def test_poll_failed_raises(): + respx.get("https://api.sonilo.com/v1/tasks/t_2").mock( + return_value=httpx.Response(200, json={"status": "failed", "code": "QUOTA", + "message": "QUOTA: no credits", + "refunded": True}) + ) + with pytest.raises(DuckingFailedError) as ei: + await_ducking_result(_client(), "t_2", poll_interval=0.0, timeout=5.0, + sleep=lambda *_: None) + assert ei.value.code == "QUOTA" + assert ei.value.refunded is True + + +@pytest.mark.parametrize("bad", [ + "http://cdn.example.com/x.wav", # not https + "https://127.0.0.1/x.wav", # IP literal + "https://localhost/x.wav", + "https://thing.local/x.wav", + "https://svc.internal/x.wav", + "https://0x7f000001/x", # hex-integer 127.0.0.1 + "https://2130706433/x", # decimal-integer 127.0.0.1 + "https://017700000001/x", # legacy-octal-integer 127.0.0.1 + "https://127.0.0.1./x", # trailing-dot dotted-quad + "https://0xA9FEA9FE/x", # hex-integer 169.254.169.254 (metadata) + "https://0x7f.0.0.1/x", # dotted, first label hex + "https://0177.0.0.1/x", # dotted, first label legacy octal +]) +def test_ssrf_guard_rejects(bad): + with pytest.raises(VideoKitError): + assert_safe_download_url(bad) + + +def test_ssrf_guard_allows_normal_https(): + assert_safe_download_url("https://cdn.example.com/mix.wav") # no raise + + +def test_ssrf_guard_allows_domain_with_numeric_labels(): + # A domain whose labels merely CONTAIN digits must not be misclassified + # as an IP-literal encoding just because _parse_ipv4_number is lenient + # about hex/octal-looking labels. + assert_safe_download_url("https://sub.domain-with-numbers123.com/x") # no raise + + +@respx.mock +def test_download_enforces_byte_cap(tmp_path): + respx.get("https://cdn.example.com/big.wav").mock( + return_value=httpx.Response(200, content=b"x" * 5000) + ) + dest = tmp_path / "out.wav" + with pytest.raises(VideoKitError): + download_ducked_mix("https://cdn.example.com/big.wav", dest, max_bytes=1000, + sleep=lambda *_: None) + + +@respx.mock +def test_download_rejects_redirect_response(tmp_path): + # follow_redirects=False means a 3xx comes back as an ordinary response, + # not an exception; it must still be treated as a download failure (like + # JS's redirect:"error"), never written to disk as the "mix". + respx.get("https://cdn.example.com/mix.wav").mock( + return_value=httpx.Response( + 302, headers={"Location": "https://internal.example/secret"} + ) + ) + dest = tmp_path / "out.wav" + with pytest.raises(VideoKitError): + download_ducked_mix("https://cdn.example.com/mix.wav", dest, max_bytes=1000, + sleep=lambda *_: None) + assert not dest.exists() + + +def test_download_timeout_guard_trips_on_slow_stream(tmp_path, monkeypatch): + """A server dribbling small chunks slower than the per-attempt deadline + (but each one arriving, so httpx's own inter-chunk read timeout never + fires) must still be bounded by the wall-clock check inside the + streaming loop -- not stream forever. Uses a custom transport (rather + than respx, which buffers mocked content) so each chunk is actually + yielded with a small real sleep between them, and a near-zero per-attempt + `timeout` so the guard trips almost immediately -- fast and deterministic, + no real hang.""" + + class _SlowStream(httpx.SyncByteStream): + def __iter__(self): + for _ in range(50): + time.sleep(0.01) + yield b"x" + + def close(self) -> None: + pass + + class _SlowTransport(httpx.BaseTransport): + def handle_request(self, request): # noqa: ANN001 + return httpx.Response(200, stream=_SlowStream()) + + real_client_cls = httpx.Client + + def _client_factory(*args, **kwargs): + kwargs["transport"] = _SlowTransport() + return real_client_cls(*args, **kwargs) + + monkeypatch.setattr(ducking_api_module.httpx, "Client", _client_factory) + + dest = tmp_path / "out.wav" + start = time.monotonic() + with pytest.raises(VideoKitError): + download_ducked_mix( + "https://cdn.example.com/slow.wav", + dest, + max_bytes=10_000, + timeout=0.02, # near-zero per-attempt wall-clock cap + sleep=lambda *_: None, + ) + elapsed = time.monotonic() - start + # If the guard didn't trip, all 50 chunks * 0.01s ~= 0.5s per attempt, + # times up to 4 retried attempts ~= 2s. Bounding elapsed well under that + # proves the wall-clock check -- not the stream simply finishing -- is + # what stopped the download. + assert elapsed < 1.0 + assert not dest.exists() diff --git a/sonilo-video-kit/tests/test_errors.py b/sonilo-video-kit/tests/test_errors.py new file mode 100644 index 0000000..23a42e2 --- /dev/null +++ b/sonilo-video-kit/tests/test_errors.py @@ -0,0 +1,30 @@ +import pytest +from sonilo_video_kit.errors import ( + VideoKitError, FfmpegNotFoundError, FfmpegError, DuckingFailedError, +) + + +def test_hierarchy(): + assert issubclass(FfmpegNotFoundError, VideoKitError) + assert issubclass(FfmpegError, VideoKitError) + assert issubclass(DuckingFailedError, VideoKitError) + assert issubclass(VideoKitError, Exception) + + +def test_ffmpeg_error_carries_context(): + e = FfmpegError("boom", exit_code=1, stderr_tail="last lines") + assert e.exit_code == 1 + assert e.stderr_tail == "last lines" + assert isinstance(e, VideoKitError) + + +def test_ducking_failed_carries_code_and_refund(): + e = DuckingFailedError("nope", code="QUOTA", refunded=True) + assert e.code == "QUOTA" + assert e.refunded is True + + +def test_cause_is_optional(): + inner = ValueError("x") + e = VideoKitError("wrap", cause=inner) + assert e.cause is inner diff --git a/sonilo-video-kit/tests/test_ffmpeg_media.py b/sonilo-video-kit/tests/test_ffmpeg_media.py new file mode 100644 index 0000000..c9c807b --- /dev/null +++ b/sonilo-video-kit/tests/test_ffmpeg_media.py @@ -0,0 +1,53 @@ +import subprocess +import pytest +from sonilo_video_kit._ffmpeg import ( + measure_integrated_lufs, extract_audio, probe_mux_feasibility, mux_video_with_audio, +) + + +def _ff(args): + subprocess.run(["ffmpeg", "-y", *args], check=True, capture_output=True) + + +def test_measure_lufs_returns_float(tmp_path): + a = tmp_path / "a.wav" + _ff(["-f", "lavfi", "-i", "sine=frequency=440:duration=3", str(a)]) + val = measure_integrated_lufs(a) + assert isinstance(val, float) + + +def test_measure_lufs_none_on_bad_input(tmp_path): + bad = tmp_path / "nope.wav" + bad.write_bytes(b"not audio") + assert measure_integrated_lufs(bad) is None + + +def test_extract_audio(tmp_path): + v = tmp_path / "av.mp4" + _ff(["-f", "lavfi", "-i", "testsrc=duration=2:size=128x128:rate=15", + "-f", "lavfi", "-i", "sine=frequency=440:duration=2", + "-shortest", "-pix_fmt", "yuv420p", str(v)]) + out = tmp_path / "out.m4a" + extract_audio(v, out, "aac") + assert out.exists() and out.stat().st_size > 0 + + +def test_mux_feasibility_ok(tmp_path): + v = tmp_path / "av.mp4" + _ff(["-f", "lavfi", "-i", "testsrc=duration=1:size=128x128:rate=15", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-shortest", "-pix_fmt", "yuv420p", str(v)]) + res = probe_mux_feasibility(v, tmp_path / "probe.mp4") + assert res.ok is True + + +def test_mux_video_with_audio(tmp_path): + v = tmp_path / "av.mp4" + _ff(["-f", "lavfi", "-i", "testsrc=duration=2:size=128x128:rate=15", + "-f", "lavfi", "-i", "sine=frequency=440:duration=2", + "-shortest", "-pix_fmt", "yuv420p", str(v)]) + a = tmp_path / "music.wav" + _ff(["-f", "lavfi", "-i", "sine=frequency=220:duration=2", str(a)]) + out = tmp_path / "final.mp4" + mux_video_with_audio(v, a, out, 2.0) + assert out.exists() and out.stat().st_size > 0 diff --git a/sonilo-video-kit/tests/test_generate.py b/sonilo-video-kit/tests/test_generate.py new file mode 100644 index 0000000..0b7ec6f --- /dev/null +++ b/sonilo-video-kit/tests/test_generate.py @@ -0,0 +1,30 @@ +from sonilo_video_kit import generate_music_for_video + + +class _FakeV2M: + def __init__(self): + self.calls = [] + + def generate(self, *, video=None, prompt=None, segments=None): + self.calls.append({"video": video, "prompt": prompt, "segments": segments}) + return "TRACK" + + +class _FakeClient: + def __init__(self): + self.video_to_music = _FakeV2M() + + +def test_passthrough_with_prompt(): + c = _FakeClient() + out = generate_music_for_video("clip.mp4", prompt="epic", client=c) + assert out == "TRACK" + assert c.video_to_music.calls == [ + {"video": "clip.mp4", "prompt": "epic", "segments": None} + ] + + +def test_passthrough_with_segments(): + c = _FakeClient() + generate_music_for_video("clip.mp4", segments=[{"prompt": "x", "duration": 5}], client=c) + assert c.video_to_music.calls[0]["segments"] == [{"prompt": "x", "duration": 5}] diff --git a/sonilo-video-kit/tests/test_loudness.py b/sonilo-video-kit/tests/test_loudness.py new file mode 100644 index 0000000..af719e1 --- /dev/null +++ b/sonilo-video-kit/tests/test_loudness.py @@ -0,0 +1,38 @@ +import math +from sonilo_video_kit import loudness as L + + +def test_constants_exact(): + assert L.FALLBACK_MUSIC_LUFS == -16.0 + assert L.OUTPUT_CEILING_DBFS == -1.0 + assert L.SLIDER_CENTER == 0.5 + assert L.SLIDER_SPAN_DB == 24.0 + assert L.GAP_BELOW_VOICE_LU == 4.0 + assert L.DELIVERY_TARGET_LUFS == -14.0 + assert L.MAX_DELIVERY_BOOST_DB == 12.0 + + +def test_db_to_linear(): + assert L.db_to_linear(0) == 1.0 + assert math.isclose(L.db_to_linear(6), 1.9952623149688795, rel_tol=1e-9) + + +def test_offset_db_slider_endpoints(): + assert L.offset_db(0.5) == 0.0 + assert math.isclose(L.offset_db(1.0), 12.0) + assert math.isclose(L.offset_db(0.0), -12.0) + + +def test_gap_gain_zero_when_slider_nonpositive(): + assert L.gap_gain(-20.0, -16.0, 0.0) == 0.0 + + +def test_gap_gain_value(): + # bed=-20, music=-16, slider=0.5 -> offset 0 -> db_to_linear(-20+0-(-16)) = db_to_linear(-4) + assert math.isclose(L.gap_gain(-20.0, -16.0, 0.5), L.db_to_linear(-4.0)) + + +def test_original_final_gain_clamps(): + assert L.original_final_gain(1.5) == 1.0 + assert L.original_final_gain(-0.2) == 0.0 + assert L.original_final_gain(0.3) == 0.3 diff --git a/sonilo-video-kit/tests/test_mix.py b/sonilo-video-kit/tests/test_mix.py new file mode 100644 index 0000000..1d7e993 --- /dev/null +++ b/sonilo-video-kit/tests/test_mix.py @@ -0,0 +1,45 @@ +import subprocess +from pathlib import Path +from sonilo_video_kit import mix_with_video +from sonilo_video_kit._ffmpeg import probe_video + + +def _ff(args): + subprocess.run(["ffmpeg", "-y", *args], check=True, capture_output=True) + + +def _video_with_audio(p): + _ff(["-f", "lavfi", "-i", "testsrc=duration=3:size=160x120:rate=15", + "-f", "lavfi", "-i", "sine=frequency=440:duration=3", + "-shortest", "-pix_fmt", "yuv420p", str(p)]) + + +def test_mix_produces_video_with_audio(tmp_path): + v = tmp_path / "in.mp4"; _video_with_audio(v) + music = tmp_path / "music.wav" + _ff(["-f", "lavfi", "-i", "sine=frequency=220:duration=3", str(music)]) + out = tmp_path / "out.mp4" + res = mix_with_video(video=v, audio=music, output=out) + assert Path(res) == out and out.exists() + p = probe_video(out) + assert p.has_audio is True + assert p.video_codec is not None # picture preserved (stream-copied) + + +def test_mix_accepts_audio_bytes(tmp_path): + v = tmp_path / "in.mp4"; _video_with_audio(v) + music = tmp_path / "music.wav" + _ff(["-f", "lavfi", "-i", "sine=frequency=330:duration=3", str(music)]) + out = tmp_path / "out2.mp4" + mix_with_video(video=v, audio=music.read_bytes(), output=out) + assert out.exists() + + +def test_temp_files_cleaned(tmp_path): + v = tmp_path / "in.mp4"; _video_with_audio(v) + music = tmp_path / "m.wav" + _ff(["-f", "lavfi", "-i", "sine=frequency=200:duration=3", str(music)]) + out = tmp_path / "o.mp4" + mix_with_video(video=v, audio=music.read_bytes(), output=out) + leftover = [x for x in tmp_path.iterdir() if x.name not in {"in.mp4", "m.wav", "o.mp4"}] + assert leftover == [] diff --git a/sonilo-video-kit/tests/test_probe_video.py b/sonilo-video-kit/tests/test_probe_video.py new file mode 100644 index 0000000..09b29a5 --- /dev/null +++ b/sonilo-video-kit/tests/test_probe_video.py @@ -0,0 +1,28 @@ +import subprocess +import pytest +from sonilo_video_kit._ffmpeg import probe_video + + +def _make(path, args): + subprocess.run(["ffmpeg", "-y", *args, str(path)], check=True, + capture_output=True) + + +def test_probe_video_with_audio(tmp_path): + v = tmp_path / "av.mp4" + _make(v, ["-f", "lavfi", "-i", "testsrc=duration=2:size=128x128:rate=15", + "-f", "lavfi", "-i", "sine=frequency=440:duration=2", + "-shortest", "-pix_fmt", "yuv420p"]) + p = probe_video(v) + assert p.has_audio is True + assert p.video_codec is not None + assert p.audio_codec is not None + assert p.video_duration_seconds == pytest.approx(2.0, abs=0.5) + + +def test_probe_video_without_audio(tmp_path): + v = tmp_path / "v.mp4" + _make(v, ["-f", "lavfi", "-i", "testsrc=duration=1:size=128x128:rate=15", + "-pix_fmt", "yuv420p"]) + p = probe_video(v) + assert p.has_audio is False diff --git a/sonilo-video-kit/tests/test_public_surface.py b/sonilo-video-kit/tests/test_public_surface.py new file mode 100644 index 0000000..f3db34a --- /dev/null +++ b/sonilo-video-kit/tests/test_public_surface.py @@ -0,0 +1,18 @@ +import sonilo_video_kit as vk + + +def test_public_exports_present(): + expected = { + "generate_music_for_video", "mix_with_video", "duck_music_under_speech", + "VideoKitError", "FfmpegError", "FfmpegNotFoundError", "DuckingFailedError", + "DELIVERY_TARGET_LUFS", "FALLBACK_MUSIC_LUFS", "GAP_BELOW_VOICE_LU", + "OUTPUT_CEILING_DBFS", "MAX_DUCKING_DURATION_SECONDS", "MAX_DUCKED_MIX_BYTES", + "VideoMusicClient", + } + assert expected <= set(vk.__all__) + for name in expected: + assert hasattr(vk, name) + + +def test_all_is_sorted(): + assert vk.__all__ == sorted(vk.__all__) diff --git a/sonilo-video-kit/tests/test_run_process.py b/sonilo-video-kit/tests/test_run_process.py new file mode 100644 index 0000000..b9d5815 --- /dev/null +++ b/sonilo-video-kit/tests/test_run_process.py @@ -0,0 +1,26 @@ +import sys +import pytest +from sonilo_video_kit._ffmpeg import run_process +from sonilo_video_kit.errors import FfmpegNotFoundError, FfmpegError + + +def test_success_captures_stdout(): + r = run_process(sys.executable, ["-c", "print('hello')"]) + assert "hello" in r.stdout + + +def test_missing_binary_raises_not_found(): + with pytest.raises(FfmpegNotFoundError): + run_process("definitely-not-a-real-binary-xyz", ["--version"]) + + +def test_nonzero_exit_raises_ffmpeg_error(): + with pytest.raises(FfmpegError) as ei: + run_process(sys.executable, ["-c", "import sys; sys.stderr.write('bad\\n'); sys.exit(3)"]) + assert ei.value.exit_code == 3 + assert "bad" in ei.value.stderr_tail + + +def test_timeout_raises_ffmpeg_error(): + with pytest.raises(FfmpegError): + run_process(sys.executable, ["-c", "import time; time.sleep(5)"], timeout=0.2) diff --git a/sonilo-video-kit/tests/test_smoke.py b/sonilo-video-kit/tests/test_smoke.py new file mode 100644 index 0000000..44e058f --- /dev/null +++ b/sonilo-video-kit/tests/test_smoke.py @@ -0,0 +1,5 @@ +import sonilo_video_kit + + +def test_import(): + assert hasattr(sonilo_video_kit, "__all__")