Small Language Models (SLMs)

Models under 10B parameters: distillation, quantization, edge deployment, and specialization.

Why SLMs?

SLM Landscape (2024)

General Purpose

Specialized

Creating SLMs

1. Distillation (Knowledge Distillation)

# Teacher → Student
class DistillationTrainer:
    def __init__(self, teacher: Model, student: Model, temperature=2.0):
        self.teacher = teacher
        self.student = student
        self.T = temperature

    def loss(self, student_logits, teacher_logits, labels):
        # Soft targets
        soft_loss = F.kl_div(
            F.log_softmax(student_logits / self.T, dim=-1),
            F.softmax(teacher_logits / self.T, dim=-1),
            reduction='batchmean'
        ) * (self.T ** 2)

        # Hard targets
        hard_loss = F.cross_entropy(student_logits, labels)

        return 0.5 * soft_loss + 0.5 * hard_loss

2. Quantization-Aware Training (QAT)

# Train with quantization simulation
def quantize_model(model, bits=4):
    for module in model.modules():
        if isinstance(module, nn.Linear):
            # Fake quantization during training
            module.weight = FakeQuantize(module.weight, bits=bits)
            if module.bias is not None:
                module.bias = FakeQuantize(module.bias, bits=bits)

# Post-training quantization (PTQ)
def ptq_quantize(model, calib_data, bits=4):
    # GPTQ, AWQ, SpQR
    quantizer = GPTQQuantizer(bits=bits, group_size=128)
    quantizer.quantize(model, calib_data)
    return quantizer.get_quantized_model()

3. Pruning + Distillation

# Structured pruning + retrain
def prune_and_distill(model, teacher, sparsity=0.5):
    # 1. Magnitude pruning
    for module in model.modules():
        if isinstance(module, nn.Linear):
            mask = (module.weight.abs() > threshold).float()
            module.weight.data *= mask

    # 2. Distillation to recover accuracy
    trainer = DistillationTrainer(teacher, model)
    trainer.train(pruned_data)

    return model

Quantization Methods

GGUF (llama.cpp) - Edge Deployment

# Convert
llama-export model --format gguf --outfile model.gguf

# Quantize
llama-quantize model.gguf model-q4_k_m.gguf q4_k_m

# Run on CPU
./llama-cli -m model-q4_k_m.gguf -p "Hello"

# Python bindings
from llama_cpp import Llama

llm = Llama(
    model_path="model-q4_k_m.gguf",
    n_ctx=4096,
    n_threads=8,
    n_gpu_layers=0  # CPU only
)

response = llm("Hello", max_tokens=512)

Fine-Tuning SLMs

LoRA (Low-Rank Adaptation)

from peft import LoraConfig, get_peft_model

config = LoraConfig(
    r=16,                    # Rank
    lora_alpha=32,           # Scaling
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.1,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(base_model, config)
model.print_trainable_parameters()  # ~0.1% params

trainer = Trainer(model=model, args=training_args, train_dataset=dataset)
trainer.train()

QLoRA (Quantized LoRA)

from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True
)

model = AutoModelForCausalLM.from_pretrained(
    "model-name",
    quantization_config=bnb_config,
    device_map="auto"
)

# Then apply LoRA as above

Full Fine-Tuning (Small Models)

# For 1-3B models, full fine-tuning feasible
trainer = Trainer(
    model=model,
    args=TrainingArguments(
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        learning_rate=2e-5,
        num_train_epochs=3,
        bf16=True,
        gradient_checkpointing=True
    ),
    train_dataset=dataset
)

Specialization Techniques

1. Domain-Adaptive Pretraining (DAPT)

# Continue pretraining on domain corpus
def dapt(model, domain_corpus, steps=10000):
    trainer = Trainer(
        model=model,
        args=TrainingArguments(
            learning_rate=1e-5,
            max_steps=steps,
            per_device_train_batch_size=8
        ),
        train_dataset=domain_corpus
    )
    trainer.train()
    return model

2. Task-Adaptive Fine-Tuning (TAFT)

# Fine-tune on task-specific data
def taft(model, task_data, epochs=3):
    trainer = Trainer(
        model=model,
        args=TrainingArguments(
            learning_rate=2e-5,
            num_train_epochs=epochs,
            per_device_train_batch_size=4
        ),
        train_dataset=task_data
    )
    trainer.train()
    return model

3. Synthetic Data Generation

# Use large model to generate training data for SLM
async def generate_synthetic_data(teacher, domain, n_samples=10000):
    prompts = generate_prompts(domain, n_samples)
    responses = await asyncio.gather(*[
        teacher.generate(p) for p in prompts
    ])
    return [{"input": p, "output": r} for p, r in zip(prompts, responses)]

Edge Deployment

Mobile (iOS/Android)

// iOS: Core ML
import CoreML

let config = MLModelConfiguration()
config.computeUnits = .all  // CPU + GPU + Neural Engine
let model = try MyLLM(configuration: config)

let input = MyLLMInput(prompt: "Hello")
let output = try model.prediction(input: input)

// Android: MediaPipe LLM Inference
val options = LlmInference.LlmInferenceOptions.builder()
    .setModelPath("model.task")
    .setMaxTokens(512)
    .setTemperature(0.7f)
    .build()

val llm = LlmInference.createFromOptions(context, options)
val result = llm.generate("Hello")

Web (WebGPU/WASM)

// WebLLM / Transformers.js
import { CreateMLCEngine } from "@mlc-ai/web-llm";

const engine = await CreateMLCEngine(
    "Llama-3-8B-q4f16_1-MLC",
    { initProgressCallback: (report) => console.log(report) }
);

const reply = await engine.chat.completions.create({
    messages: [{ role: "user", content: "Hello!" }],
    temperature: 0.7
});

Serverless (Edge Functions)

# Cloudflare Workers / Vercel Edge / AWS Lambda
# Using ONNX Runtime Web or WASM

import onnxruntime_web as ort

session = ort.InferenceSession("model.onnx")
outputs = session.run(None, {"input_ids": input_ids})

SLM Evaluation

Benchmarks for SLMs

Evaluation Framework

def evaluate_slm(model, benchmarks=["mmlu", "gsm8k", "humaneval"]):
    results = {}
    for bench in benchmarks:
        dataset = load_dataset(bench)
        score = evaluate(model, dataset)
        results[bench] = score
    return results

# Cost-quality Pareto frontier
def pareto_frontier(models):
    results = [(m.name, m.size, evaluate(m), m.latency, m.cost) for m in models]
    # Plot: quality vs cost/latency
    return pareto_optimal(results)

When to Use SLMs

Related Notes

• Mixture of Experts — MoE vs SLM
• Inference Optimization — SLM inference
• Model Routing — Routing to SLMs
• Cost Optimization — SLM cost benefits
• Scaling Strategies — SLM scaling
• Deployment Architecture — Edge deployment
• Agent Computer Interface — On-device agents
• Coding Agents — Code SLMs
• Research Directions — SLM research

Sources: Llama 3/Gemma/Qwen/Phi release notes, HuggingFace distillation guides, GPTQ/AWQ/SpQR papers, llama.cpp/GGUF docs, MediaPipe LLM, WebLLM, Core ML, ONNX Runtime Web, Peft/QLoRA docs, distillation surveys (Hinton et al., 2015), QLoRA (Dettmers et al.).