split_large_files.py

"""
分割大文件脚本 - 将超过100MB的文件分割成多个小块
用于GitHub Pages部署（GitHub限制单个文件不能超过100MB）
"""

import json
import os
import math
from pathlib import Path

# 文件大小限制（MB）
MAX_FILE_SIZE_MB = 100
MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024


def get_file_size_mb(filepath):
    """获取文件大小（MB）"""
    return os.path.getsize(filepath) / (1024 * 1024)


def split_json_file(filepath, chunk_size_mb=90):
    """
    分割JSON文件为多个小块
    
    Args:
        filepath: JSON文件路径
        chunk_size_mb: 每个块的大小（MB），默认90MB，留一些余量
    """
    filepath = Path(filepath)
    if not filepath.exists():
        print(f"错误: 文件不存在: {filepath}")
        return
    
    file_size_mb = get_file_size_mb(filepath)
    print(f"文件大小: {file_size_mb:.2f} MB")
    
    if file_size_mb < MAX_FILE_SIZE_MB:
        print(f"文件小于 {MAX_FILE_SIZE_MB}MB，无需分割")
        return
    
    print(f"开始分割文件...")
    
    # 读取原始JSON文件
    with open(filepath, 'r', encoding='utf-8') as f:
        data = json.load(f)
    
    # 根据文件结构决定如何分割
    if isinstance(data, dict):
        # 如果是字典，按键分割
        keys = list(data.keys())
        chunk_size_bytes = chunk_size_mb * 1024 * 1024
        total_size = len(json.dumps(data, ensure_ascii=False).encode('utf-8'))
        
        # 估算每个键的大小
        avg_key_size = total_size / len(keys) if keys else 0
        keys_per_chunk = max(1, int(chunk_size_bytes / avg_key_size)) if avg_key_size > 0 else len(keys)
        
        num_chunks = math.ceil(len(keys) / keys_per_chunk)
        print(f"将分割为 {num_chunks} 个文件")
        
        base_name = filepath.stem
        output_dir = filepath.parent / f"{base_name}_chunks"
        output_dir.mkdir(exist_ok=True)
        
        # 创建索引文件
        index_data = {
            "original_file": filepath.name,
            "num_chunks": num_chunks,
            "chunk_files": []
        }
        
        for i in range(num_chunks):
            start_idx = i * keys_per_chunk
            end_idx = min((i + 1) * keys_per_chunk, len(keys))
            chunk_keys = keys[start_idx:end_idx]
            
            chunk_data = {key: data[key] for key in chunk_keys}
            chunk_filename = f"{base_name}_part{i+1:03d}.json"
            chunk_path = output_dir / chunk_filename
            
            with open(chunk_path, 'w', encoding='utf-8') as f:
                json.dump(chunk_data, f, ensure_ascii=False, indent=2)
            
            chunk_size = get_file_size_mb(chunk_path)
            print(f"  创建块 {i+1}/{num_chunks}: {chunk_filename} ({chunk_size:.2f} MB)")
            
            index_data["chunk_files"].append({
                "filename": chunk_filename,
                "keys": chunk_keys,
                "size_mb": chunk_size
            })
        
        # 保存索引文件
        index_path = output_dir / f"{base_name}_index.json"
        with open(index_path, 'w', encoding='utf-8') as f:
            json.dump(index_data, f, ensure_ascii=False, indent=2)
        
        print(f"\n分割完成！")
        print(f"输出目录: {output_dir}")
        print(f"索引文件: {index_path}")
        
    elif isinstance(data, list):
        # 如果是列表，按元素分割
        chunk_size_bytes = chunk_size_mb * 1024 * 1024
        total_size = len(json.dumps(data, ensure_ascii=False).encode('utf-8'))
        
        avg_item_size = total_size / len(data) if data else 0
        items_per_chunk = max(1, int(chunk_size_bytes / avg_item_size)) if avg_item_size > 0 else len(data)
        
        num_chunks = math.ceil(len(data) / items_per_chunk)
        print(f"将分割为 {num_chunks} 个文件")
        
        base_name = filepath.stem
        output_dir = filepath.parent / f"{base_name}_chunks"
        output_dir.mkdir(exist_ok=True)
        
        # 创建索引文件
        index_data = {
            "original_file": filepath.name,
            "num_chunks": num_chunks,
            "chunk_files": []
        }
        
        for i in range(num_chunks):
            start_idx = i * items_per_chunk
            end_idx = min((i + 1) * items_per_chunk, len(data))
            chunk_data = data[start_idx:end_idx]
            
            chunk_filename = f"{base_name}_part{i+1:03d}.json"
            chunk_path = output_dir / chunk_filename
            
            with open(chunk_path, 'w', encoding='utf-8') as f:
                json.dump(chunk_data, f, ensure_ascii=False, indent=2)
            
            chunk_size = get_file_size_mb(chunk_path)
            print(f"  创建块 {i+1}/{num_chunks}: {chunk_filename} ({chunk_size:.2f} MB)")
            
            index_data["chunk_files"].append({
                "filename": chunk_filename,
                "start_index": start_idx,
                "end_index": end_idx,
                "size_mb": chunk_size
            })
        
        # 保存索引文件
        index_path = output_dir / f"{base_name}_index.json"
        with open(index_path, 'w', encoding='utf-8') as f:
            json.dump(index_data, f, ensure_ascii=False, indent=2)
        
        print(f"\n分割完成！")
        print(f"输出目录: {output_dir}")
        print(f"索引文件: {index_path}")
    else:
        print(f"错误: 不支持的JSON格式（必须是dict或list）")


def check_and_split_all():
    """检查并分割所有需要分割的文件"""
    current_dir = Path('.')
    
    # 检查需要分割的文件
    files_to_check = [
        'graph_data.json',
        'graph_index.json'
    ]
    
    print("=" * 50)
    print("检查大文件...")
    print("=" * 50)
    
    for filename in files_to_check:
        filepath = current_dir / filename
        if filepath.exists():
            file_size_mb = get_file_size_mb(filepath)
            print(f"\n检查文件: {filename}")
            print(f"大小: {file_size_mb:.2f} MB")
            
            if file_size_mb >= MAX_FILE_SIZE_MB:
                print(f"⚠️  文件超过 {MAX_FILE_SIZE_MB}MB，需要分割")
                split_json_file(filepath)
            else:
                print(f"✓ 文件大小正常，无需分割")
        else:
            print(f"⚠️  文件不存在: {filename}")
    
    print("\n" + "=" * 50)
    print("检查完成！")
    print("=" * 50)


if __name__ == '__main__':
    import sys
    
    if len(sys.argv) > 1:
        # 如果提供了文件路径，分割指定文件
        filepath = sys.argv[1]
        split_json_file(filepath)
    else:
        # 否则检查所有文件
        check_and_split_all()