AI Knowledge Distillation: Big Models Teaching Small Ones

AI Knowledge Distillation: Big Models Teaching Small Ones

💡 What You Will Learn

AI Knowledge Distillation: Big Models Teaching Small Ones

import json
from openai import OpenAI

client = OpenAI()

def generate_training_data(seed_questions, teacher_model="gpt-4o"):
    """Generate QA pairs using LLM as distillation training data"""
    training_pairs = []
    for q in seed_questions:
        # Generate
        resp = client.chat.completions.create(
            model=teacher_model,
            messages=[{"role": "user", "content": q}],
            temperature=0.7  # 
        )
        answer = resp.choices[0].message.content

        # GenerateChain-of-Thought
        cot_resp = client.chat.completions.create(
            model=teacher_model,
            messages=[{"role": "user", "content": f"{q}"}],
            temperature=0.3
        )
        reasoning = cot_resp.choices[0].message.content

        training_pairs.append({
            "question": q,
            "answer": answer,
            "reasoning": reasoning,
            "source": teacher_model
        })
    return training_pairs

# Generate1000
seed = ["", "", ""]
dataset = generate_training_data(seed)
print(f"{len(dataset)}")

# UseLLaMA-FactoryUnsloth
# Qwen3-1.8B

from datasets import Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer

# 
def format_for_training(pair):
    return {
        "text": f"{pair['question']}\n{pair['answer']}\n{pair['reasoning']}"
    }

train_dataset = Dataset.from_list([format_for_training(p) for p in dataset])

# Qwen3-1.8B1.8BParameter
model_name = "Qwen/Qwen3-1.8B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)

training_args = TrainingArguments(
    output_dir="./distilled_qwen",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    learning_rate=2e-5,
    save_steps=500,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)

trainer.train()

|:----|:-----|:--------|:-----------------|:------| || 671B || $1,500 | 95% | || 7B ||| 88% | || 1.8B ||| 82% |

def evaluate_distillation(student_model, teacher_model, test_questions):
    """"""
    scores = {"exact_match": 0, "semantic_similar": 0, "total": len(test_questions)}
    for q in test_questions:
        student_ans = student_model.chat(q)
        teacher_ans = teacher_model.chat(q)
        # GPT-4o
        eval_prompt = f"{student_ans}\n{teacher_ans}\n0-10"
        score = float(gpt_eval(eval_prompt))
        if score >= 9:
            scores['exact_match'] += 1
        if score >= 7:
            scores['semantic_similar'] += 1
    return scores

# 92%1/50