Hugging Face SFT Trainer

概述

使用 Hugging Face TRL SFTTrainer 进行监督微调。本教程提供了一个完整、可运行的实现。

前提条件

bash
安装所需包
pip install transformers datasets peft trl accelerate bitsandbytes
pip install trl
验证 GPU 访问
python -c "import torch; print(torch.cuda.is_available())"

数据集准备

python
from datasets import Dataset, load_dataset
import json
def prepare_dataset(examples: list[dict]) -> Dataset:
    """
    准备用于监督微调的数据集。
    
    预期格式：
    [{"instruction": "...", "input": "...", "output": "..."}]
    """
    
    def format_example(example):
        instruction = example.get("instruction", "")
        input_text = example.get("input", "")
        output = example.get("output", "")
        
        if input_text:
            prompt = f"### Instruction:\n{instruction}\n\n### Input:\n{input_text}\n\n### Response:\n{output}"
        else:
            prompt = f"### Instruction:\n{instruction}\n\n### Response:\n{output}"
        
        return {"text": prompt}
    
    formatted = [format_example(ex) for ex in examples]
    return Dataset.from_list(formatted)
加载或创建你的数据集
示例：从 HuggingFace 加载
dataset = load_dataset("your-org/your-dataset", split="train")
或从自己的数据创建
examples = [
    {
        "instruction": "分类这段文本",
        "input": "示例文本",
        "output": "类别：正面"
    }
]
dataset = prepare_dataset(examples)
print(f"数据集大小：{len(dataset)}")
print(f"样本：{dataset[0]['text'][:200]}")

使用 TRL 设置模型

python
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, TaskType
模型配置
MODEL_ID = "meta-llama/Llama-3.2-1B-Instruct"  # 或你的基础模型
OUTPUT_DIR = "./fine-tuned-model"
加载分词器
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"
QLoRA：4 位量化配置
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,
)
加载基础模型
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
配置 LoRA
lora_config = LoraConfig(
    r=16,                          # 秩 - 越高参数越多
    lora_alpha=32,                 # 缩放因子
    target_modules=[               # 要适配的层
        "q_proj", "v_proj",
        "k_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    lora_dropout=0.05,
    bias="none",
    task_type=TaskType.CAUSAL_LM,
)
应用 LoRA
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
输出：trainable params: 6.7M || all params: 1.24B || trainable%: 0.54%

训练配置

python
from transformers import TrainingArguments
from trl import SFTTrainer
训练参数
training_args = TrainingArguments(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,      # 有效批次 = 16
    gradient_checkpointing=True,        # 节省内存
    optim="paged_adamw_32bit",
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    weight_decay=0.001,
    max_grad_norm=0.3,
    logging_steps=25,
    save_steps=500,
    eval_steps=500,
    fp16=True,                          # 对于 Ampere GPU 使用 bf16=True
    report_to="mlflow",                 # 使用 MLflow 跟踪
    run_name="fine-tuning-run-1",
)
初始化训练器
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    tokenizer=tokenizer,
    args=training_args,
    dataset_text_field="text",
    max_seq_length=2048,
    packing=False,
)
开始训练！
trainer.train()
保存微调后的适配器
trainer.model.save_pretrained(OUTPUT_DIR)
tokenizer.save_pretrained(OUTPUT_DIR)
print(f"模型已保存至 {OUTPUT_DIR}")

使用微调模型进行推理

python
from peft import PeftModel
加载基础模型 + 适配器
base_model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, device_map="auto", torch_dtype=torch.float16
)
model = PeftModel.from_pretrained(base_model, OUTPUT_DIR)
model.eval()
def generate(instruction: str, input_text: str = "") -> str:
    """使用微调模型生成。"""
    if input_text:
        prompt = f"### Instruction:\n{instruction}\n\n### Input:\n{input_text}\n\n### Response:\n"
    else:
        prompt = f"### Instruction:\n{instruction}\n\n### Response:\n"
    
    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=512,
            temperature=0.1,
            do_sample=True,
            eos_token_id=tokenizer.eos_token_id,
        )
    
    response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
    return response.strip()
测试
response = generate("用简单的话解释监督学习")
print(response)

评估

python
from evaluate import load
import numpy as np
加载评估指标
rouge = load("rouge")
bleu = load("bleu")
def evaluate_model(test_examples: list[dict], model_fn) -> dict:
    """评估微调模型质量。"""
    predictions = []
    references = []
    
    for ex in test_examples:
        pred = model_fn(ex["instruction"], ex.get("input", ""))
        predictions.append(pred)
        references.append(ex["output"])
    
    rouge_scores = rouge.compute(predictions=predictions, references=references)
    
    return {
        "rouge1": rouge_scores["rouge1"],
        "rouge2": rouge_scores["rouge2"],
        "rougeL": rouge_scores["rougeL"],
        "num_examples": len(predictions)
    }
results = evaluate_model(test_examples, generate)
print(f"评估结果：{results}")

GPU 内存需求

模型大小技术所需显存

1B 参数全量微调~6GB 1B 参数QLoRA~3GB 7B 参数QLoRA~10GB 13B 参数QLoRA~14GB 70B 参数QLoRA~45GB

最佳实践

资源