From 8decb818d456462fecb58e94fa7204ff512d2b1a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 8 Jun 2026 12:40:32 +0000 Subject: [PATCH] DX-596-together-fine-tuning: sync file validation flow with mintlify-docs#892 Mirror the updated client/server-side file validation guidance from togethercomputer/mintlify-docs#892: - references/data-formats.md: split validation into client-side structural check and server-side schema validation, document processing_status states (COMPLETED / INVALID_FORMAT / FAILED) and validation_report payloads, refresh CLI examples. - scripts/*_workflow.py: poll client.files.retrieve(...).processing_status after upload before starting fine-tuning so bad datasets fail fast instead of consuming training time. --- .../references/data-formats.md | 54 +++++++++++++++++-- .../scripts/dpo_workflow.py | 23 ++++++++ .../scripts/finetune_workflow.py | 26 +++++++++ .../scripts/function_calling_finetune.py | 22 ++++++++ .../scripts/reasoning_finetune.py | 22 ++++++++ .../scripts/vlm_finetune.py | 22 ++++++++ 6 files changed, 165 insertions(+), 4 deletions(-) diff --git a/skills/together-fine-tuning/references/data-formats.md b/skills/together-fine-tuning/references/data-formats.md index 94ffa36..7d2c8d2 100644 --- a/skills/together-fine-tuning/references/data-formats.md +++ b/skills/together-fine-tuning/references/data-formats.md @@ -254,14 +254,58 @@ For preference fine-tuning with function calling, the `tools` field goes inside ## Data Validation +Validation runs in two stages: + +1. **Client-side structural check** (local). Runs by default inside `client.files.upload(..., check=True)` or with `together files check`. Verifies only basic formatting: UTF-8 encoding, one JSON object per line, minimum sample count, and maximum file size. Pass `check=False` to skip (useful for very large files). +2. **Server-side schema validation** (during ingestion). Runs after upload and performs the full fine-tuning schema check (conversation roles, tool calls, required fields, etc.). The file is only usable for fine-tuning once `processing_status` becomes `COMPLETED`. If validation rejects the dataset, `processing_status` becomes `INVALID_FORMAT` and `validation_report.error` carries a user-facing reason. + ```python +import time from together import Together client = Together() -# Upload with validation enabled +# 1. Upload with the local structural check enabled (default). file = client.files.upload(file="my_data.jsonl", purpose="fine-tune", check=True) print(file.id) # file-abc123 + +# 2. Poll until server-side validation finishes before creating a fine-tuning job. +while True: + meta = client.files.retrieve(file.id) + if meta.processing_status == "COMPLETED": + break + if meta.processing_status == "INVALID_FORMAT": + # meta.validation_report["error"] carries a user-facing reason. + raise ValueError( + f"file is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file processing did not complete: {meta.validation_report}" + ) + time.sleep(5) +``` + +Treat `processing_status` as the authoritative readiness signal; the `validation_report` schema may evolve. A successful response looks like: + +```json +{ + "processing_status": "COMPLETED", + "validation_report": {"valid": true, "dataset_format": "conversation", "nlines": 7199} +} +``` + +A user-correctable failure looks like: + +```json +{ + "processing_status": "INVALID_FORMAT", + "validation_report": { + "valid": false, + "error_type": "INVALID_FORMAT", + "error": "Line 7: `messages[1]` must contain a `role` field" + } +} ``` ```shell @@ -269,12 +313,14 @@ print(file.id) # file-abc123 together files check my_data.jsonl together files upload my_data.jsonl -# Upload without format checking +# Upload without the local structural check together files upload my_data.jsonl --no-check -# List and manage files -together files list +# Inspect server-side validation status (processing_status / validation_report) together files retrieve + +# List and download files +together files list together files retrieve-content ``` diff --git a/skills/together-fine-tuning/scripts/dpo_workflow.py b/skills/together-fine-tuning/scripts/dpo_workflow.py index 77dcfe6..3666249 100644 --- a/skills/together-fine-tuning/scripts/dpo_workflow.py +++ b/skills/together-fine-tuning/scripts/dpo_workflow.py @@ -26,6 +26,23 @@ client = Together() +def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None: + """Block until server-side fine-tuning validation finishes for ``file_id``.""" + while True: + meta = client.files.retrieve(file_id) + if meta.processing_status == "COMPLETED": + return + if meta.processing_status == "INVALID_FORMAT": + raise ValueError( + f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file {file_id} processing did not complete: {meta.validation_report}" + ) + time.sleep(poll_interval) + + def sample_sft_data() -> list[dict]: """Return a small SFT dataset for the DPO warm-up stage.""" return [ @@ -160,6 +177,12 @@ def main() -> None: print(f"SFT file: {sft_file.id}") print(f"DPO file: {dpo_file.id}") + # Wait for server-side validation on both files before starting training. + print("Waiting for server-side validation...") + wait_for_file_ready(sft_file.id) + wait_for_file_ready(dpo_file.id) + print("Files ready for fine-tuning.") + # --- 4. Step 1: Run SFT job first --- print("\n--- Step 1: SFT Training ---") sft_job = client.fine_tuning.create( diff --git a/skills/together-fine-tuning/scripts/finetune_workflow.py b/skills/together-fine-tuning/scripts/finetune_workflow.py index 1473ddd..8f173fc 100644 --- a/skills/together-fine-tuning/scripts/finetune_workflow.py +++ b/skills/together-fine-tuning/scripts/finetune_workflow.py @@ -25,6 +25,27 @@ client = Together() +def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None: + """Block until server-side fine-tuning validation finishes for ``file_id``. + + Raises ``ValueError`` if the dataset is rejected (``INVALID_FORMAT``) and + ``RuntimeError`` for any other terminal failure. + """ + while True: + meta = client.files.retrieve(file_id) + if meta.processing_status == "COMPLETED": + return + if meta.processing_status == "INVALID_FORMAT": + raise ValueError( + f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file {file_id} processing did not complete: {meta.validation_report}" + ) + time.sleep(poll_interval) + + def sample_training_data() -> list[dict]: """Return a small conversational dataset for demonstration.""" return [ @@ -129,6 +150,11 @@ def main() -> None: file_id = file_response.id print(f"Uploaded file: {file_id}") + # Wait for server-side validation before spending tokens on training. + print("Waiting for server-side validation...") + wait_for_file_ready(file_id) + print("File ready for fine-tuning.") + # --- 3. Create LoRA fine-tuning job --- job = client.fine_tuning.create( training_file=file_id, diff --git a/skills/together-fine-tuning/scripts/function_calling_finetune.py b/skills/together-fine-tuning/scripts/function_calling_finetune.py index fc83fc9..a088bed 100644 --- a/skills/together-fine-tuning/scripts/function_calling_finetune.py +++ b/skills/together-fine-tuning/scripts/function_calling_finetune.py @@ -25,6 +25,23 @@ client = Together() +def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None: + """Block until server-side fine-tuning validation finishes for ``file_id``.""" + while True: + meta = client.files.retrieve(file_id) + if meta.processing_status == "COMPLETED": + return + if meta.processing_status == "INVALID_FORMAT": + raise ValueError( + f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file {file_id} processing did not complete: {meta.validation_report}" + ) + time.sleep(poll_interval) + + def build_tools() -> list[dict]: """Return sample tool definitions used by the demo dataset and test prompt.""" return [ @@ -222,6 +239,11 @@ def main() -> None: data_path.unlink(missing_ok=True) print(f"Uploaded file: {file_resp.id}") + # Wait for server-side validation before starting training. + print("Waiting for server-side validation...") + wait_for_file_ready(file_resp.id) + print("File ready for fine-tuning.") + # --- 3. Start LoRA fine-tuning --- job = client.fine_tuning.create( training_file=file_resp.id, diff --git a/skills/together-fine-tuning/scripts/reasoning_finetune.py b/skills/together-fine-tuning/scripts/reasoning_finetune.py index 56b7b3e..90257f0 100644 --- a/skills/together-fine-tuning/scripts/reasoning_finetune.py +++ b/skills/together-fine-tuning/scripts/reasoning_finetune.py @@ -33,6 +33,23 @@ client = Together() +def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None: + """Block until server-side fine-tuning validation finishes for ``file_id``.""" + while True: + meta = client.files.retrieve(file_id) + if meta.processing_status == "COMPLETED": + return + if meta.processing_status == "INVALID_FORMAT": + raise ValueError( + f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file {file_id} processing did not complete: {meta.validation_report}" + ) + time.sleep(poll_interval) + + def sample_training_data() -> list[dict]: """Return a small reasoning dataset.""" return [ @@ -166,6 +183,11 @@ def main() -> None: data_path.unlink(missing_ok=True) print(f"Uploaded file: {file_resp.id}") + # Wait for server-side validation before starting training. + print("Waiting for server-side validation...") + wait_for_file_ready(file_resp.id) + print("File ready for fine-tuning.") + # --- 3. Start LoRA fine-tuning on a reasoning-capable model --- job = client.fine_tuning.create( training_file=file_resp.id, diff --git a/skills/together-fine-tuning/scripts/vlm_finetune.py b/skills/together-fine-tuning/scripts/vlm_finetune.py index 5415fc1..9224ee2 100644 --- a/skills/together-fine-tuning/scripts/vlm_finetune.py +++ b/skills/together-fine-tuning/scripts/vlm_finetune.py @@ -28,6 +28,23 @@ client = Together() +def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None: + """Block until server-side fine-tuning validation finishes for ``file_id``.""" + while True: + meta = client.files.retrieve(file_id) + if meta.processing_status == "COMPLETED": + return + if meta.processing_status == "INVALID_FORMAT": + raise ValueError( + f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}" + ) + if meta.processing_status == "FAILED": + raise RuntimeError( + f"file {file_id} processing did not complete: {meta.validation_report}" + ) + time.sleep(poll_interval) + + def url_to_base64(url: str, mime_type: str = "image/jpeg") -> str: """Download an image URL and return a base64 data URI.""" response = requests.get(url, timeout=60) @@ -138,6 +155,11 @@ def main() -> None: data_path.unlink(missing_ok=True) print(f"Uploaded file: {file_resp.id}") + # Wait for server-side validation before starting training. + print("Waiting for server-side validation...") + wait_for_file_ready(file_resp.id) + print("File ready for fine-tuning.") + # --- 3. Start VLM LoRA fine-tuning --- job = client.fine_tuning.create( training_file=file_resp.id,