diff --git a/cookbook/client/twinkle/self_congnition.py b/cookbook/client/twinkle/self_congnition.py index 6b907653..dc318dcf 100644 --- a/cookbook/client/twinkle/self_congnition.py +++ b/cookbook/client/twinkle/self_congnition.py @@ -21,8 +21,7 @@ logger = get_logger() -# Whether to use Megatron for training -use_megatron = True + # Step 2: Initialize the Twinkle client to communicate with the remote server. # - base_url: the address of the running Twinkle server # - api_key: authentication token (loaded from environment variable) @@ -88,8 +87,7 @@ def train(): model.set_optimizer('Adam', lr=1e-4) # Use a linear learning rate scheduler (Do not support LR scheduler if server use megatron) - if not use_megatron: - model.set_lr_scheduler('LinearLR') + model.set_lr_scheduler('LinearLR') # Step 6: Optionally resume from a previous checkpoint if resume_path: diff --git a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md index 0ddf490f..2f67e37b 100644 --- a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md +++ b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md @@ -1,6 +1,6 @@ -# Qwen3.5-4B Training Best Practices +# Qwen3.5 Training Best Practices -Using Qwen3.5-4B as an example, this guide demonstrates the core capability of the Twinkle framework: **one codebase, from single GPU training to Client-Server mode**. +Using Qwen3.5-4B as an example, this guide demonstrates the core capability of the Twinkle framework: **one component-based code, used from single GPU training to Client-Server mode**. --- @@ -364,7 +364,6 @@ Twinkle Client provides an API nearly identical to local training, ideal for sce import dotenv dotenv.load_dotenv('.env') -import os from peft import LoraConfig from twinkle import get_logger @@ -376,9 +375,6 @@ from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() -# Whether to use the Megatron backend -use_megatron = True - # Initialize the Twinkle client client = init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') @@ -410,11 +406,8 @@ def train(): model.set_template('Template') model.set_processor('InputProcessor', padding_side='right') model.set_loss('CrossEntropyLoss') - model.set_optimizer('Adam', lr=1e-4) - - # LR scheduler not supported with Megatron backend - if not use_megatron: - model.set_lr_scheduler('LinearLR') + model.set_optimizer('AdamW', lr=1e-4) + model.set_lr_scheduler('LinearLR') # Resume from checkpoint if available if resume_path: @@ -473,7 +466,7 @@ init_tinker_client() from tinker import ServiceClient # Base model -base_model = 'Qwen/Qwen3-30B-A3B-Instruct-2507' +base_model = 'Qwen/Qwen3.5-4B' base_url = 'http://www.modelscope.cn/twinkle' @@ -481,7 +474,7 @@ def train(): # Prepare dataset dataset = Dataset(dataset_meta=DatasetMeta('ms://swift/self-cognition', data_slice=range(500))) dataset.set_template('Template', model_id=f'ms://{base_model}', max_length=256) - dataset.map(SelfCognitionProcessor('twinkle model', 'twinkle team'), load_from_cache_file=False) + dataset.map(SelfCognitionProcessor('Twinkle Model', 'ModelScope Team'), load_from_cache_file=False) dataset.encode(batched=True, load_from_cache_file=False) dataloader = DataLoader(dataset=dataset, batch_size=8) diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" index 191d4dcc..8b86b9b0 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" @@ -1,6 +1,6 @@ -# Qwen3.5-4B 训练最佳实践 +# Qwen3.5 训练最佳实践 -本文以 Qwen3.5-4B 为例,演示 Twinkle 框架的核心能力:**一套代码,从单卡训练到Client-Server环境**。 +本文以 Qwen3.5-4B 为例,演示 Twinkle 框架的核心能力:**一套组件化代码,从单卡训练到Client-Server环境**。 --- @@ -364,7 +364,6 @@ Twinkle Client 提供与本地训练几乎完全一致的 API,适合需要精 import dotenv dotenv.load_dotenv('.env') -import os from peft import LoraConfig from twinkle import get_logger @@ -376,9 +375,6 @@ from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() -# 是否使用 Megatron 后端 -use_megatron = True - # 初始化 Twinkle 客户端 client = init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') @@ -410,11 +406,8 @@ def train(): model.set_template('Template') model.set_processor('InputProcessor', padding_side='right') model.set_loss('CrossEntropyLoss') - model.set_optimizer('Adam', lr=1e-4) - - # Megatron 后端暂不支持 LR 调度器 - if not use_megatron: - model.set_lr_scheduler('LinearLR') + model.set_optimizer('AdamW', lr=1e-4) + model.set_lr_scheduler('LinearLR') # 恢复训练(如有检查点) if resume_path: @@ -473,7 +466,7 @@ init_tinker_client() from tinker import ServiceClient # 基座模型 -base_model = 'Qwen/Qwen3-30B-A3B-Instruct-2507' +base_model = 'Qwen/Qwen3.5-4B' base_url = 'http://www.modelscope.cn/twinkle' @@ -481,7 +474,7 @@ def train(): # 准备数据集 dataset = Dataset(dataset_meta=DatasetMeta('ms://swift/self-cognition', data_slice=range(500))) dataset.set_template('Template', model_id=f'ms://{base_model}', max_length=256) - dataset.map(SelfCognitionProcessor('twinkle模型', 'twinkle团队'), load_from_cache_file=False) + dataset.map(SelfCognitionProcessor('Twinkle模型', 'ModelScope团队'), load_from_cache_file=False) dataset.encode(batched=True, load_from_cache_file=False) dataloader = DataLoader(dataset=dataset, batch_size=8)