test_seed_timing.py

#!/usr/bin/env python3
"""
Test if seed timing affects generation results.
"""

import sys
import torch
import numpy as np
from pathlib import Path

# Add src to path
sys.path.append(str(Path(__file__).parent / "src"))

from models.model_factory import ModelFactory

def test_seed_timing():
    """Test if seed timing affects generation results."""
    print("🔬 Test Seed Timing")
    print("=" * 60)
    
    # Load model
    print("🔧 Loading model...")
    model = ModelFactory.create_model(
        model_type="deepseek",
        model_name="/raid/LLM/llama3.1-8b-instruct",
        config={"gpu_id": 0}
    )
    model.load_model()
    
    # Test problem
    problem = "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?"
    prompt = f"Q: {problem}\nA:"
    
    print(f"Problem: {problem}")
    print()
    
    # Test 1: Set seed before each generation
    print("🔧 Test 1: Set seed before each generation")
    print("-" * 50)
    
    results = []
    for i in range(3):
        print(f"Generation {i+1}:")
        
        # Set seed before each generation
        torch.manual_seed(42)
        np.random.seed(42)
        if torch.cuda.is_available():
            torch.cuda.manual_seed(42)
            torch.cuda.manual_seed_all(42)
        
        generated = model.generate(
            prompt,
            max_tokens=512,
            temperature=0.0,
            top_p=1.0,
            do_sample=False,
            num_return_sequences=1
        )
        
        results.append(generated)
        print(f"  Length: {len(generated)}")
        print(f"  First 100 chars: {generated[:100]}")
        print()
    
    # Check if all results are identical
    all_identical = all(result == results[0] for result in results)
    print(f"All results identical: {all_identical}")
    
    # Test 2: Set seed once at the beginning
    print("\n🔧 Test 2: Set seed once at the beginning")
    print("-" * 50)
    
    # Set seed once at the beginning
    torch.manual_seed(42)
    np.random.seed(42)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(42)
        torch.cuda.manual_seed_all(42)
    
    results2 = []
    for i in range(3):
        print(f"Generation {i+1}:")
        
        generated = model.generate(
            prompt,
            max_tokens=512,
            temperature=0.0,
            top_p=1.0,
            do_sample=False,
            num_return_sequences=1
        )
        
        results2.append(generated)
        print(f"  Length: {len(generated)}")
        print(f"  First 100 chars: {generated[:100]}")
        print()
    
    # Check if all results are identical
    all_identical2 = all(result == results2[0] for result in results2)
    print(f"All results identical: {all_identical2}")
    
    # Test 3: Compare results between the two approaches
    print("\n🔧 Test 3: Compare approaches")
    print("-" * 50)
    
    print(f"Test 1 first result: {results[0][:100]}")
    print(f"Test 2 first result: {results2[0][:100]}")
    print(f"Results identical: {results[0] == results2[0]}")
    
    # Test 4: Check if the issue is in our framework vs direct generation
    print("\n🔧 Test 4: Framework vs Direct Generation")
    print("-" * 50)
    
    # Simulate our framework approach (set seed in generation method)
    print("Simulating our framework approach:")
    torch.manual_seed(42)
    np.random.seed(42)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(42)
        torch.cuda.manual_seed_all(42)
    
    # Simulate some processing that might change random state
    # (like what happens in our framework)
    dummy_tensor = torch.randn(10, 10)
    dummy_np = np.random.randn(10, 10)
    
    # Now generate
    framework_result = model.generate(
        prompt,
        max_tokens=512,
        temperature=0.0,
        top_p=1.0,
        do_sample=False,
        num_return_sequences=1
    )
    
    print(f"Framework result: {framework_result[:100]}")
    
    # Simulate direct generation approach (set seed right before generation)
    print("\nSimulating direct generation approach:")
    torch.manual_seed(42)
    np.random.seed(42)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(42)
        torch.cuda.manual_seed_all(42)
    
    # Generate immediately
    direct_result = model.generate(
        prompt,
        max_tokens=512,
        temperature=0.0,
        top_p=1.0,
        do_sample=False,
        num_return_sequences=1
    )
    
    print(f"Direct result: {direct_result[:100]}")
    print(f"Results identical: {framework_result == direct_result}")

if __name__ == "__main__":
    test_seed_timing()