From 59f8ce7759f01ab4e4afbd160d78b608b67a488e Mon Sep 17 00:00:00 2001 From: mohit-twelvelabs Date: Thu, 25 Jun 2026 09:43:14 -0700 Subject: [PATCH 1/3] docs: add TwelveLabs Pegasus video understanding example --- examples/README.md | 1 + .../twelvelabs_video_understanding/.gitignore | 9 ++ .../twelvelabs_video_understanding/README.md | 88 ++++++++++++ .../requirements.txt | 4 + .../twelvelabs_example.py | 132 ++++++++++++++++++ 5 files changed, 234 insertions(+) create mode 100644 examples/twelvelabs_video_understanding/.gitignore create mode 100644 examples/twelvelabs_video_understanding/README.md create mode 100644 examples/twelvelabs_video_understanding/requirements.txt create mode 100644 examples/twelvelabs_video_understanding/twelvelabs_example.py diff --git a/examples/README.md b/examples/README.md index d29b48519..09593098c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,3 +10,4 @@ For more information and examples, visit our [documentation](https://supervision - [speed estimation](./speed_estimation) by [@SkalskiP](https://github.com/SkalskiP) - [time in zone](./time_in_zone) by [@SkalskiP](https://github.com/SkalskiP) - [heatmap and track](./heatmap_and_track/) by [@HinePo](https://github.com/HinePo) +- [twelvelabs video understanding](./twelvelabs_video_understanding/) by [@mohit-twelvelabs](https://github.com/mohit-twelvelabs) diff --git a/examples/twelvelabs_video_understanding/.gitignore b/examples/twelvelabs_video_understanding/.gitignore new file mode 100644 index 000000000..34efd9e06 --- /dev/null +++ b/examples/twelvelabs_video_understanding/.gitignore @@ -0,0 +1,9 @@ +data/ +venv*/ +*.pt +*.pth +*.mp4 +*.mov +*.png +*.jpg +*.jpeg diff --git a/examples/twelvelabs_video_understanding/README.md b/examples/twelvelabs_video_understanding/README.md new file mode 100644 index 000000000..d886640d5 --- /dev/null +++ b/examples/twelvelabs_video_understanding/README.md @@ -0,0 +1,88 @@ +# twelvelabs video understanding + +## 👋 hello + +This script pairs per-frame Supervision detections with a whole-video +understanding generated by [TwelveLabs](https://twelvelabs.io) Pegasus. + +Supervision answers "what object is where, in each frame" with precise, +box-level detections. Pegasus answers "what is this video about" with a +natural-language summary of the whole clip. Running both on the same video +combines fine-grained detections with a high-level narrative — useful for +captioning, search, moderation, or building a quick description alongside your +annotated output. + +## 💻 install + +- clone repository and navigate to example directory + + ```bash + git clone --depth 1 -b develop https://github.com/roboflow/supervision.git + cd supervision/examples/twelvelabs_video_understanding + ``` + +- setup python environment and activate it [optional] + + ```bash + uv venv + source .venv/bin/activate + ``` + +- install required dependencies + + ```bash + uv pip install -r requirements.txt + ``` + +- set your TwelveLabs API key (grab a free one at [twelvelabs.io](https://twelvelabs.io)) + + ```bash + export TWELVELABS_API_KEY="" + ``` + +## 🛠️ script arguments + +- `--source_video_path`: Required. Path to the source video file to analyze. + +- `--prompt` (optional): Instruction passed to Pegasus for the video-level + summary. Default is `"Summarize what happens in this video."`. + +- `--model_id` (optional): Ultralytics YOLO model id or weights path used for + per-frame detection. Default is `"yolov8n.pt"`. + +- `--confidence_threshold` (optional): Confidence threshold for detections + (0 to 1). Default is `0.3`. + +- `--pegasus_model_name` (optional): TwelveLabs Pegasus model to use. Default + is `"pegasus1.5"`. + +- `--max_tokens` (optional): Maximum number of tokens in the Pegasus response. + Default is `512`. + +- `--target_video_path` (optional): Path to save the annotated video. If not + specified, annotated frames are displayed in real time. + +- `--api_key` (optional): TwelveLabs API key. Falls back to the + `TWELVELABS_API_KEY` environment variable when omitted. + +## ⚙️ run + +```bash +python twelvelabs_example.py \ + --source_video_path data/video.mp4 \ + --prompt "Summarize what happens in this video." \ + --confidence_threshold 0.3 \ + --target_video_path data/video_result.mp4 +``` + +## © license + +This demo integrates two main components, each with its own licensing: + +- TwelveLabs: Video understanding is provided by the [TwelveLabs](https://twelvelabs.io) + Pegasus model through the `twelvelabs` Python SDK and requires a TwelveLabs + API key. + +- supervision: The detection and annotation code in this demo is based on the + Supervision library, which is licensed under the + [MIT license](https://github.com/roboflow/supervision/blob/develop/LICENSE.md). diff --git a/examples/twelvelabs_video_understanding/requirements.txt b/examples/twelvelabs_video_understanding/requirements.txt new file mode 100644 index 000000000..a1737e2e3 --- /dev/null +++ b/examples/twelvelabs_video_understanding/requirements.txt @@ -0,0 +1,4 @@ +supervision +ultralytics +twelvelabs +jsonargparse[signatures] diff --git a/examples/twelvelabs_video_understanding/twelvelabs_example.py b/examples/twelvelabs_video_understanding/twelvelabs_example.py new file mode 100644 index 000000000..a280c1581 --- /dev/null +++ b/examples/twelvelabs_video_understanding/twelvelabs_example.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import os + +import cv2 +import numpy as np + +import supervision as sv + +COLORS = sv.ColorPalette.DEFAULT +BOX_ANNOTATOR = sv.BoxAnnotator(color=COLORS) +LABEL_ANNOTATOR = sv.LabelAnnotator(color=COLORS) + + +def analyze_video( + source_video_path: str, + prompt: str, + model_name: str = "pegasus1.5", + max_tokens: int = 512, + api_key: str = "", +) -> str: + """ + Generate a natural-language understanding of a whole video with TwelveLabs + Pegasus. + + Args: + source_video_path: Path to the source video file to analyze. + prompt: Instruction passed to Pegasus, e.g. "Summarize this video". + model_name: TwelveLabs Pegasus model to use. + max_tokens: Maximum number of tokens in the generated response. + api_key: TwelveLabs API key. Falls back to the `TWELVELABS_API_KEY` + environment variable when empty. + + Returns: + The text produced by Pegasus describing the video. + """ + import base64 + + from twelvelabs import TwelveLabs + from twelvelabs.types.video_context import VideoContext_Base64String + + api_key = api_key or os.environ.get("TWELVELABS_API_KEY", "") + if not api_key: + raise ValueError( + "A TwelveLabs API key is required. Pass --api_key or set the " + "TWELVELABS_API_KEY environment variable. Grab a free key at " + "https://twelvelabs.io." + ) + + with open(source_video_path, "rb") as video_file: + encoded_video = base64.b64encode(video_file.read()).decode("utf-8") + + client = TwelveLabs(api_key=api_key) + response = client.analyze( + model_name=model_name, + video=VideoContext_Base64String(base_64_string=encoded_video), + prompt=prompt, + max_tokens=max_tokens, + ) + return response.data or "" + + +def main( + source_video_path: str, + prompt: str = "Summarize what happens in this video.", + model_id: str = "yolov8n.pt", + confidence_threshold: float = 0.3, + pegasus_model_name: str = "pegasus1.5", + max_tokens: int = 512, + target_video_path: str | None = None, + api_key: str = "", +) -> None: + """ + Combine per-frame Supervision detections with a whole-video TwelveLabs + Pegasus understanding. + + Supervision answers "what object is where, in each frame" while Pegasus + answers "what is this video about". Running both on the same clip pairs + precise, frame-level detections with a high-level narrative summary. + + Args: + source_video_path: Path to the source video file. + prompt: Instruction passed to Pegasus for the video-level summary. + model_id: Ultralytics YOLO model id or weights path for detection. + confidence_threshold: Confidence level for detections (0 to 1). + pegasus_model_name: TwelveLabs Pegasus model to use. + max_tokens: Maximum number of tokens in the Pegasus response. + target_video_path: Optional path to save the annotated video. When + omitted, annotated frames are displayed in a window. + api_key: TwelveLabs API key. Falls back to the `TWELVELABS_API_KEY` + environment variable when empty. + """ + from ultralytics import YOLO + + summary = analyze_video( + source_video_path=source_video_path, + prompt=prompt, + model_name=pegasus_model_name, + max_tokens=max_tokens, + api_key=api_key, + ) + print("TwelveLabs Pegasus summary:") + print(summary) + + model = YOLO(model_id) + video_info = sv.VideoInfo.from_video_path(video_path=source_video_path) + frames_generator = sv.get_video_frames_generator(source_video_path) + + def annotate(frame: np.ndarray) -> np.ndarray: + result = model(frame, conf=confidence_threshold, verbose=False)[0] + detections = sv.Detections.from_ultralytics(result) + annotated_frame = BOX_ANNOTATOR.annotate(frame.copy(), detections) + annotated_frame = LABEL_ANNOTATOR.annotate(annotated_frame, detections) + return annotated_frame + + if target_video_path is not None: + with sv.VideoSink(target_video_path, video_info) as sink: + for frame in frames_generator: + sink.write_frame(annotate(frame)) + else: + for frame in frames_generator: + cv2.imshow("Processed Video", annotate(frame)) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + cv2.destroyAllWindows() + + +if __name__ == "__main__": + from jsonargparse import auto_cli, set_parsing_settings + + set_parsing_settings(parse_optionals_as_positionals=True) + auto_cli(main, as_positional=False) From c3662637d72c3534c3a4deaf5b528d01091b5f34 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:50:41 +0000 Subject: [PATCH 2/3] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto=20?= =?UTF-8?q?format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../twelvelabs_video_understanding/README.md | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/examples/twelvelabs_video_understanding/README.md b/examples/twelvelabs_video_understanding/README.md index d886640d5..88f00c8a4 100644 --- a/examples/twelvelabs_video_understanding/README.md +++ b/examples/twelvelabs_video_understanding/README.md @@ -2,15 +2,9 @@ ## 👋 hello -This script pairs per-frame Supervision detections with a whole-video -understanding generated by [TwelveLabs](https://twelvelabs.io) Pegasus. +This script pairs per-frame Supervision detections with a whole-video understanding generated by [TwelveLabs](https://twelvelabs.io) Pegasus. -Supervision answers "what object is where, in each frame" with precise, -box-level detections. Pegasus answers "what is this video about" with a -natural-language summary of the whole clip. Running both on the same video -combines fine-grained detections with a high-level narrative — useful for -captioning, search, moderation, or building a quick description alongside your -annotated output. +Supervision answers "what object is where, in each frame" with precise, box-level detections. Pegasus answers "what is this video about" with a natural-language summary of the whole clip. Running both on the same video combines fine-grained detections with a high-level narrative — useful for captioning, search, moderation, or building a quick description alongside your annotated output. ## 💻 install @@ -44,26 +38,19 @@ annotated output. - `--source_video_path`: Required. Path to the source video file to analyze. -- `--prompt` (optional): Instruction passed to Pegasus for the video-level - summary. Default is `"Summarize what happens in this video."`. +- `--prompt` (optional): Instruction passed to Pegasus for the video-level summary. Default is `"Summarize what happens in this video."`. -- `--model_id` (optional): Ultralytics YOLO model id or weights path used for - per-frame detection. Default is `"yolov8n.pt"`. +- `--model_id` (optional): Ultralytics YOLO model id or weights path used for per-frame detection. Default is `"yolov8n.pt"`. -- `--confidence_threshold` (optional): Confidence threshold for detections - (0 to 1). Default is `0.3`. +- `--confidence_threshold` (optional): Confidence threshold for detections (0 to 1). Default is `0.3`. -- `--pegasus_model_name` (optional): TwelveLabs Pegasus model to use. Default - is `"pegasus1.5"`. +- `--pegasus_model_name` (optional): TwelveLabs Pegasus model to use. Default is `"pegasus1.5"`. -- `--max_tokens` (optional): Maximum number of tokens in the Pegasus response. - Default is `512`. +- `--max_tokens` (optional): Maximum number of tokens in the Pegasus response. Default is `512`. -- `--target_video_path` (optional): Path to save the annotated video. If not - specified, annotated frames are displayed in real time. +- `--target_video_path` (optional): Path to save the annotated video. If not specified, annotated frames are displayed in real time. -- `--api_key` (optional): TwelveLabs API key. Falls back to the - `TWELVELABS_API_KEY` environment variable when omitted. +- `--api_key` (optional): TwelveLabs API key. Falls back to the `TWELVELABS_API_KEY` environment variable when omitted. ## ⚙️ run @@ -79,10 +66,6 @@ python twelvelabs_example.py \ This demo integrates two main components, each with its own licensing: -- TwelveLabs: Video understanding is provided by the [TwelveLabs](https://twelvelabs.io) - Pegasus model through the `twelvelabs` Python SDK and requires a TwelveLabs - API key. +- TwelveLabs: Video understanding is provided by the [TwelveLabs](https://twelvelabs.io) Pegasus model through the `twelvelabs` Python SDK and requires a TwelveLabs API key. -- supervision: The detection and annotation code in this demo is based on the - Supervision library, which is licensed under the - [MIT license](https://github.com/roboflow/supervision/blob/develop/LICENSE.md). +- supervision: The detection and annotation code in this demo is based on the Supervision library, which is licensed under the [MIT license](https://github.com/roboflow/supervision/blob/develop/LICENSE.md). From 9fc198982054a254537214fcdcc20f5ed4ebc3d2 Mon Sep 17 00:00:00 2001 From: mohit Date: Fri, 26 Jun 2026 02:43:15 -0700 Subject: [PATCH 3/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/twelvelabs_video_understanding/twelvelabs_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/twelvelabs_video_understanding/twelvelabs_example.py b/examples/twelvelabs_video_understanding/twelvelabs_example.py index a280c1581..392d6917d 100644 --- a/examples/twelvelabs_video_understanding/twelvelabs_example.py +++ b/examples/twelvelabs_video_understanding/twelvelabs_example.py @@ -23,6 +23,9 @@ def analyze_video( Generate a natural-language understanding of a whole video with TwelveLabs Pegasus. + This example base64-encodes the entire video file in memory before sending it to + the API, so keep clips short to avoid high memory usage. + Args: source_video_path: Path to the source video file to analyze. prompt: Instruction passed to Pegasus, e.g. "Summarize this video".