DX-596-together-fine-tuning: sync with mintlify-docs#892

skills/together-fine-tuning/references/data-formats.md

@@ -254,27 +254,73 @@ For preference fine-tuning with function calling, the `tools` field goes inside
 
 ## Data Validation
 
+Validation runs in two stages:
+
+1. **Client-side structural check** (local). Runs by default inside `client.files.upload(..., check=True)` or with `together files check`. Verifies only basic formatting: UTF-8 encoding, one JSON object per line, minimum sample count, and maximum file size. Pass `check=False` to skip (useful for very large files).
+2. **Server-side schema validation** (during ingestion). Runs after upload and performs the full fine-tuning schema check (conversation roles, tool calls, required fields, etc.). The file is only usable for fine-tuning once `processing_status` becomes `COMPLETED`. If validation rejects the dataset, `processing_status` becomes `INVALID_FORMAT` and `validation_report.error` carries a user-facing reason.
+
 ```python
+import time
 from together import Together
 
 client = Together()
 
-# Upload with validation enabled
+# 1. Upload with the local structural check enabled (default).
 file = client.files.upload(file="my_data.jsonl", purpose="fine-tune", check=True)
 print(file.id)  # file-abc123
+
+# 2. Poll until server-side validation finishes before creating a fine-tuning job.
+while True:
+    meta = client.files.retrieve(file.id)
+    if meta.processing_status == "COMPLETED":
+        break
+    if meta.processing_status == "INVALID_FORMAT":
+        # meta.validation_report["error"] carries a user-facing reason.
+        raise ValueError(
+            f"file is not suitable for fine-tuning: {meta.validation_report}"
+        )
+    if meta.processing_status == "FAILED":
+        raise RuntimeError(
+            f"file processing did not complete: {meta.validation_report}"
+        )
+    time.sleep(5)
+```
+
+Treat `processing_status` as the authoritative readiness signal; the `validation_report` schema may evolve. A successful response looks like:
+
+```json
+{
+  "processing_status": "COMPLETED",
+  "validation_report": {"valid": true, "dataset_format": "conversation", "nlines": 7199}
+}
+```
+
+A user-correctable failure looks like:
+
+```json
+{
+  "processing_status": "INVALID_FORMAT",
+  "validation_report": {
+    "valid": false,
+    "error_type": "INVALID_FORMAT",
+    "error": "Line 7: `messages[1]` must contain a `role` field"
+  }
+}
 ```
 
 ```shell
 # CLI: check format and upload
 together files check my_data.jsonl
 together files upload my_data.jsonl
 
-# Upload without format checking
+# Upload without the local structural check
 together files upload my_data.jsonl --no-check
 
-# List and manage files
-together files list
+# Inspect server-side validation status (processing_status / validation_report)
 together files retrieve <FILE-ID>
+
+# List and download files
+together files list
 together files retrieve-content <FILE-ID>
 ```
 

skills/together-fine-tuning/scripts/dpo_workflow.py

@@ -26,6 +26,23 @@
 client = Together()
 
 
+def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None:
+    """Block until server-side fine-tuning validation finishes for ``file_id``."""
+    while True:
+        meta = client.files.retrieve(file_id)
+        if meta.processing_status == "COMPLETED":
+            return
+        if meta.processing_status == "INVALID_FORMAT":
+            raise ValueError(
+                f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}"
+            )
+        if meta.processing_status == "FAILED":
+            raise RuntimeError(
+                f"file {file_id} processing did not complete: {meta.validation_report}"
+            )
+        time.sleep(poll_interval)
+
+
 def sample_sft_data() -> list[dict]:
     """Return a small SFT dataset for the DPO warm-up stage."""
     return [
@@ -160,6 +177,12 @@ def main() -> None:
     print(f"SFT file: {sft_file.id}")
     print(f"DPO file: {dpo_file.id}")
 
+    # Wait for server-side validation on both files before starting training.
+    print("Waiting for server-side validation...")
+    wait_for_file_ready(sft_file.id)
+    wait_for_file_ready(dpo_file.id)
+    print("Files ready for fine-tuning.")
+
     # --- 4. Step 1: Run SFT job first ---
     print("\n--- Step 1: SFT Training ---")
     sft_job = client.fine_tuning.create(

skills/together-fine-tuning/scripts/finetune_workflow.py

@@ -25,6 +25,27 @@
 client = Together()
 
 
+def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None:
+    """Block until server-side fine-tuning validation finishes for ``file_id``.
+
+    Raises ``ValueError`` if the dataset is rejected (``INVALID_FORMAT``) and
+    ``RuntimeError`` for any other terminal failure.
+    """
+    while True:
+        meta = client.files.retrieve(file_id)
+        if meta.processing_status == "COMPLETED":
+            return
+        if meta.processing_status == "INVALID_FORMAT":
+            raise ValueError(
+                f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}"
+            )
+        if meta.processing_status == "FAILED":
+            raise RuntimeError(
+                f"file {file_id} processing did not complete: {meta.validation_report}"
+            )
+        time.sleep(poll_interval)
+
+
 def sample_training_data() -> list[dict]:
     """Return a small conversational dataset for demonstration."""
     return [
@@ -129,6 +150,11 @@ def main() -> None:
     file_id = file_response.id
     print(f"Uploaded file: {file_id}")
 
+    # Wait for server-side validation before spending tokens on training.
+    print("Waiting for server-side validation...")
+    wait_for_file_ready(file_id)
+    print("File ready for fine-tuning.")
+
     # --- 3. Create LoRA fine-tuning job ---
     job = client.fine_tuning.create(
         training_file=file_id,

skills/together-fine-tuning/scripts/function_calling_finetune.py

@@ -25,6 +25,23 @@
 client = Together()
 
 
+def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None:
+    """Block until server-side fine-tuning validation finishes for ``file_id``."""
+    while True:
+        meta = client.files.retrieve(file_id)
+        if meta.processing_status == "COMPLETED":
+            return
+        if meta.processing_status == "INVALID_FORMAT":
+            raise ValueError(
+                f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}"
+            )
+        if meta.processing_status == "FAILED":
+            raise RuntimeError(
+                f"file {file_id} processing did not complete: {meta.validation_report}"
+            )
+        time.sleep(poll_interval)
+
+
 def build_tools() -> list[dict]:
     """Return sample tool definitions used by the demo dataset and test prompt."""
     return [
@@ -222,6 +239,11 @@ def main() -> None:
             data_path.unlink(missing_ok=True)
     print(f"Uploaded file: {file_resp.id}")
 
+    # Wait for server-side validation before starting training.
+    print("Waiting for server-side validation...")
+    wait_for_file_ready(file_resp.id)
+    print("File ready for fine-tuning.")
+
     # --- 3. Start LoRA fine-tuning ---
     job = client.fine_tuning.create(
         training_file=file_resp.id,

skills/together-fine-tuning/scripts/reasoning_finetune.py

@@ -33,6 +33,23 @@
 client = Together()
 
 
+def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None:
+    """Block until server-side fine-tuning validation finishes for ``file_id``."""
+    while True:
+        meta = client.files.retrieve(file_id)
+        if meta.processing_status == "COMPLETED":
+            return
+        if meta.processing_status == "INVALID_FORMAT":
+            raise ValueError(
+                f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}"
+            )
+        if meta.processing_status == "FAILED":
+            raise RuntimeError(
+                f"file {file_id} processing did not complete: {meta.validation_report}"
+            )
+        time.sleep(poll_interval)
+
+
 def sample_training_data() -> list[dict]:
     """Return a small reasoning dataset."""
     return [
@@ -166,6 +183,11 @@ def main() -> None:
             data_path.unlink(missing_ok=True)
     print(f"Uploaded file: {file_resp.id}")
 
+    # Wait for server-side validation before starting training.
+    print("Waiting for server-side validation...")
+    wait_for_file_ready(file_resp.id)
+    print("File ready for fine-tuning.")
+
     # --- 3. Start LoRA fine-tuning on a reasoning-capable model ---
     job = client.fine_tuning.create(
         training_file=file_resp.id,

skills/together-fine-tuning/scripts/vlm_finetune.py

@@ -28,6 +28,23 @@
 client = Together()
 
 
+def wait_for_file_ready(file_id: str, poll_interval: int = 5) -> None:
+    """Block until server-side fine-tuning validation finishes for ``file_id``."""
+    while True:
+        meta = client.files.retrieve(file_id)
+        if meta.processing_status == "COMPLETED":
+            return
+        if meta.processing_status == "INVALID_FORMAT":
+            raise ValueError(
+                f"file {file_id} is not suitable for fine-tuning: {meta.validation_report}"
+            )
+        if meta.processing_status == "FAILED":
+            raise RuntimeError(
+                f"file {file_id} processing did not complete: {meta.validation_report}"
+            )
+        time.sleep(poll_interval)
+
+
 def url_to_base64(url: str, mime_type: str = "image/jpeg") -> str:
     """Download an image URL and return a base64 data URI."""
     response = requests.get(url, timeout=60)
@@ -138,6 +155,11 @@ def main() -> None:
             data_path.unlink(missing_ok=True)
     print(f"Uploaded file: {file_resp.id}")
 
+    # Wait for server-side validation before starting training.
+    print("Waiting for server-side validation...")
+    wait_for_file_ready(file_resp.id)
+    print("File ready for fine-tuning.")
+
     # --- 3. Start VLM LoRA fine-tuning ---
     job = client.fine_tuning.create(
         training_file=file_resp.id,