Lab Specification — Module FT09: DoRA, rsLoRA, and Modern PEFT

# prep_data.py
from datasets import load_dataset

ds = load_dataset("HuggingFaceTB/smoltalk", "all", split="train")
# Keep only the standard system/user/assistant conversations.
ds = ds.filter(lambda x: len(x["messages"]) >= 2 and
               x["messages"][-1]["role"] == "assistant")
ds = ds.shuffle(seed=42).select(range(2000))

# 90/10 train/eval split — eval is held-out, never trained on.
split = ds.train_test_split(test_size=200, seed=42)
train_ds, eval_ds = split["train"], split["test"]

train_ds.save_to_disk("./data/train")
eval_ds.save_to_disk("./data/eval")
print(f"train={len(train_ds)}  eval={len(eval_ds)}")

# eval_harness.py
import torch, evaluate
from datasets import load_from_disk
from transformers import AutoTokenizer
from peft import PeftModel

BASE_MODEL = "HuggingFaceTB/SmolLM2-135M-Instruct"

def load_eval():
    return load_from_disk("./data/eval")

def generate_predictions(adapter_path, eval_ds, max_new_tokens=64):
    """Merge the adapter into the base and greedy-decode the assistant turn."""
    tok = AutoTokenizer.from_pretrained(BASE_MODEL)
    base = AutoModelForCausalLM.from_pretrained(
        BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto")
    model = PeftModel.from_pretrained(base, adapter_path)
    model = model.merge_and_unload()  # zero-overhead merged inference
    model.eval()

    preds, refs = [], []
    for ex in eval_ds:
        msgs = ex["messages"][:-1]  # everything except the gold assistant turn
        prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
        gold = ex["messages"][-1]["content"]
        inputs = tok(prompt, return_tensors="pt").to(model.device)
        with torch.no_grad():
            out = model.generate(**inputs, max_new_tokens=max_new_tokens,
                                 do_sample=False,
                                 pad_token_id=tok.eos_token_id)
        gen = tok.decode(out[0][inputs["input_ids"].shape[1]:],
                         skip_special_tokens=True).strip()
        preds.append(gen); refs.append(gold)
    return preds, refs

def score(preds, refs):
    rouge = evaluate.load("rouge")
    exact = sum(int(p.strip() == r.strip()) for p, r in zip(preds, refs)) / len(refs)
    return {"exact_match": round(exact, 4),
            "rougeL": round(rouge.compute(predictions=preds, references=refs)["rougeL"], 4)}

# run_a_lora.py
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_from_disk

BASE = "HuggingFaceTB/SmolLM2-135M-Instruct"
tok = AutoTokenizer.from_pretrained(BASE)
tok.pad_token = tok.eos_token

model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype="bfloat16", device_map="auto")

# Vanilla LoRA — the FT08 config. No DoRA, no rsLoRA.
config = LoraConfig(
    r=16,                          # LOW RANK on purpose — DoRA's lead is largest here
    lora_alpha=32,                 # alpha = 2*r is a common vanilla recipe
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
)

model = get_peft_model(model, config)
model.print_trainable_parameters()  # confirm the adapter size

train_ds = load_from_disk("./data/train")
cfg = SFTConfig(
    output_dir="./out/run_a_lora",
    num_train_epochs=1,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    logging_steps=10,
    bf16=True,
    packing=False,
    dataset_text_field=None,        # use the messages list directly
)

trainer = SFTTrainer(
    model=model,
    args=cfg,
    train_dataset=train_ds,
    processing_class=tok,
)
trainer.train()
model.save_pretrained("./out/run_a_lora_adapter")
print("Run A (vanilla LoRA) complete.")

# run_b_dora_rslora.py
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_from_disk

BASE = "HuggingFaceTB/SmolLM2-135M-Instruct"
tok = AutoTokenizer.from_pretrained(BASE)
tok.pad_token = tok.eos_token

model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype="bfloat16", device_map="auto")

# THE FT09 SWEET SPOT — two flags change everything.
config = LoraConfig(
    r=16,                          # SAME RANK as Run A — controlled comparison
    lora_alpha=16,                 # with use_rslora, effective scale = alpha/sqrt(r)
    use_dora=True,                 # magnitude/direction decomposition
    use_rslora=True,               # alpha/sqrt(r) scaling
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
)

model = get_peft_model(model, config)
model.print_trainable_parameters()  # expect ~the LoRA count + tiny magnitude vectors

train_ds = load_from_disk("./data/train")
cfg = SFTConfig(
    output_dir="./out/run_b_dora_rslora",
    num_train_epochs=1,            # SAME epochs
    per_device_train_batch_size=4, # SAME batch
    gradient_accumulation_steps=4,
    learning_rate=2e-4,            # SAME LR — only the adapter method differs
    lr_scheduler_type="cosine",
    logging_steps=10,
    bf16=True,
    packing=False,
    dataset_text_field=None,
)

trainer = SFTTrainer(
    model=model,
    args=cfg,
    train_dataset=train_ds,
    processing_class=tok,
)
trainer.train()
model.save_pretrained("./out/run_b_dora_rslora_adapter")
print("Run B (DoRA + rsLoRA) complete.")

# run_eval.py
from eval_harness import load_eval, generate_predictions, score

eval_ds = load_eval()

print("=== Run A: vanilla LoRA ===")
preds_a, refs = generate_predictions("./out/run_a_lora_adapter", eval_ds)
print(score(preds_a, refs))

print("\n=== Run B: DoRA + rsLoRA ===")
preds_b, _ = generate_predictions("./out/run_b_dora_rslora_adapter", eval_ds)
print(score(preds_b, refs))

# Show 3 sample deltas
import random
random.seed(0)
for i in random.sample(range(len(eval_ds)), 3):
    ex = eval_ds[i]
    gold = ex["messages"][-1]["content"]
    print(f"\n--- example {i} ---")
    print(f"GOLD:       {gold[:120]}")
    print(f"LoRA:       {preds_a[i][:120]}")
    print(f"DoRA+rslora:{preds_b[i][:120]}")

# Lab Specification — Module FT09: DoRA, rsLoRA, and Modern PEFT

**Course**: Course 3 — LLM Fine-Tuning Masterclass
**Module**: FT09 — DoRA, rsLoRA, and Modern PEFT
**Duration**: 60–90 minutes (two training runs + eval; consumer-GPU friendly)
**Environment**: A single consumer GPU (8–24 GB VRAM). The lab is written for a small, fast base model so it runs end-to-end in a reasonable time, and the same recipe scales to 7B-class models via QDoRA on a 12 GB card. No multi-GPU, no rented A100 required.

> **Setup (one time):**
> ```bash
> pip install "transformers>=4.46" "peft>=0.13" "trl>=0.11" "accelerate>=0.34" \
>             "datasets>=2.20" "bitsandbytes>=0.43" torch evaluate
> ```
> A CUDA GPU is expected. If you are on Apple Silicon or CPU-only, see the *No-GPU fallback* note at the bottom — you can still complete the inspection/config portions, but the training runs require a GPU.

---

## Learning objectives

By the end of this lab you will have:

1. **Run a controlled A/B between vanilla LoRA and DoRA + rsLoRA** on the *same* base model, the *same* data, the *same* eval set — the only valid basis for adopting a new PEFT method.
2. **Configured DoRA + rsLoRA in PEFT** using the two flags (`use_dora=True`, `use_rslora=True`) and confirmed the parameter counts / scaling behavior.
3. **Measured the quality delta** on a held-out set and observed that DoRA leads — especially at low rank, where the magnitude/direction decoupling buys the most headroom.
4. **Internalized the FT09 discipline**: never adopt a PEFT method on a paper's benchmark alone; measure it on your own data with a controlled A/B. The lab is the embodiment of that discipline.

---

## The setup

We use a small, fast base so the lab runs in minutes, not hours. The recipe is identical for any causal LM — swap the model name to scale up.

- **Base model**: `HuggingFaceTB/SmolLM2-135M-Instruct` (fast, small; swap for `Qwen/Qwen2.5-0.5B-Instruct` if you prefer).
- **Dataset**: a small instruction dataset. We use a 2k-row slice of `HuggingFaceTB/smoltalk` (the "all" subset), held-out 200 rows for eval. Any small instruction dataset works; the point is that both adapters see the *same* data.
- **Eval**: held-out exact-match + ROUGE-L on the assistant turn. The metric is less important than that it is *the same* for both adapters.

### Data prep (shared by both runs)

```python
# prep_data.py
from datasets import load_dataset

ds = load_dataset("HuggingFaceTB/smoltalk", "all", split="train")
# Keep only the standard system/user/assistant conversations.
ds = ds.filter(lambda x: len(x["messages"]) >= 2 and
               x["messages"][-1]["role"] == "assistant")
ds = ds.shuffle(seed=42).select(range(2000))

# 90/10 train/eval split — eval is held-out, never trained on.
split = ds.train_test_split(test_size=200, seed=42)
train_ds, eval_ds = split["train"], split["test"]

train_ds.save_to_disk("./data/train")
eval_ds.save_to_disk("./data/eval")
print(f"train={len(train_ds)}  eval={len(eval_ds)}")
```

Run once: `python prep_data.py`. Both training runs consume `./data/train` and are scored on `./data/eval`.

---

## The eval harness (shared by both runs)

```python
# eval_harness.py
import torch, evaluate
from datasets import load_from_disk
from transformers import AutoTokenizer
from peft import PeftModel

BASE_MODEL = "HuggingFaceTB/SmolLM2-135M-Instruct"

def load_eval():
    return load_from_disk("./data/eval")

def generate_predictions(adapter_path, eval_ds, max_new_tokens=64):
    """Merge the adapter into the base and greedy-decode the assistant turn."""
    tok = AutoTokenizer.from_pretrained(BASE_MODEL)
    base = AutoModelForCausalLM.from_pretrained(
        BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto")
    model = PeftModel.from_pretrained(base, adapter_path)
    model = model.merge_and_unload()  # zero-overhead merged inference
    model.eval()

    preds, refs = [], []
    for ex in eval_ds:
        msgs = ex["messages"][:-1]  # everything except the gold assistant turn
        prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
        gold = ex["messages"][-1]["content"]
        inputs = tok(prompt, return_tensors="pt").to(model.device)
        with torch.no_grad():
            out = model.generate(**inputs, max_new_tokens=max_new_tokens,
                                 do_sample=False,
                                 pad_token_id=tok.eos_token_id)
        gen = tok.decode(out[0][inputs["input_ids"].shape[1]:],
                         skip_special_tokens=True).strip()
        preds.append(gen); refs.append(gold)
    return preds, refs

def score(preds, refs):
    rouge = evaluate.load("rouge")
    exact = sum(int(p.strip() == r.strip()) for p, r in zip(preds, refs)) / len(refs)
    return {"exact_match": round(exact, 4),
            "rougeL": round(rouge.compute(predictions=preds, references=refs)["rougeL"], 4)}
```

The crucial property: `eval_harness.py` is *identical* for both adapters. The only difference between the two runs is the adapter on disk. That is what makes the comparison controlled.

(`AutoModelForCausalLM` import omitted for brevity — add `from transformers import AutoModelForCausalLM` at the top.)

---

## Run A — Vanilla LoRA (the FT08 baseline)

```python
# run_a_lora.py
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_from_disk

BASE = "HuggingFaceTB/SmolLM2-135M-Instruct"
tok = AutoTokenizer.from_pretrained(BASE)
tok.pad_token = tok.eos_token

model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype="bfloat16", device_map="auto")

# Vanilla LoRA — the FT08 config. No DoRA, no rsLoRA.
config = LoraConfig(
    r=16,                          # LOW RANK on purpose — DoRA's lead is largest here
    lora_alpha=32,                 # alpha = 2*r is a common vanilla recipe
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
)

model = get_peft_model(model, config)
model.print_trainable_parameters()  # confirm the adapter size

train_ds = load_from_disk("./data/train")
cfg = SFTConfig(
    output_dir="./out/run_a_lora",
    num_train_epochs=1,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    logging_steps=10,
    bf16=True,
    packing=False,
    dataset_text_field=None,        # use the messages list directly
)

trainer = SFTTrainer(
    model=model,
    args=cfg,
    train_dataset=train_ds,
    processing_class=tok,
)
trainer.train()
model.save_pretrained("./out/run_a_lora_adapter")
print("Run A (vanilla LoRA) complete.")
```

Run it: `python run_a_lora.py`. Note the trainable parameter count printed by `print_trainable_parameters()`. Record it.

---

## Run B — DoRA + rsLoRA (the FT09 default)

```python
# run_b_dora_rslora.py
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_from_disk

BASE = "HuggingFaceTB/SmolLM2-135M-Instruct"
tok = AutoTokenizer.from_pretrained(BASE)
tok.pad_token = tok.eos_token

model = AutoModelForCausalLM.from_pretrained(BASE, torch_dtype="bfloat16", device_map="auto")

# THE FT09 SWEET SPOT — two flags change everything.
config = LoraConfig(
    r=16,                          # SAME RANK as Run A — controlled comparison
    lora_alpha=16,                 # with use_rslora, effective scale = alpha/sqrt(r)
    use_dora=True,                 # magnitude/direction decomposition
    use_rslora=True,               # alpha/sqrt(r) scaling
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
)

model = get_peft_model(model, config)
model.print_trainable_parameters()  # expect ~the LoRA count + tiny magnitude vectors

train_ds = load_from_disk("./data/train")
cfg = SFTConfig(
    output_dir="./out/run_b_dora_rslora",
    num_train_epochs=1,            # SAME epochs
    per_device_train_batch_size=4, # SAME batch
    gradient_accumulation_steps=4,
    learning_rate=2e-4,            # SAME LR — only the adapter method differs
    lr_scheduler_type="cosine",
    logging_steps=10,
    bf16=True,
    packing=False,
    dataset_text_field=None,
)

trainer = SFTTrainer(
    model=model,
    args=cfg,
    train_dataset=train_ds,
    processing_class=tok,
)
trainer.train()
model.save_pretrained("./out/run_b_dora_rslora_adapter")
print("Run B (DoRA + rsLoRA) complete.")
```

Run it: `python run_b_dora_rslora.py`. Note the trainable parameter count — it should be very close to Run A (the magnitude vectors add a negligible `d`-per-matrix overhead). The point of this lab is that the *quality* differs more than the *cost* does.

### Why the same rank, LR, and epochs?

Because otherwise you are not measuring DoRA — you are measuring a confounded change. The discipline of a PEFT A/B: change exactly one thing (the adapter method), hold everything else fixed. If you also bump the rank or the LR, you no longer know which change caused the delta.

---

## Eval — score both adapters on the held-out set

```python
# run_eval.py
from eval_harness import load_eval, generate_predictions, score

eval_ds = load_eval()

print("=== Run A: vanilla LoRA ===")
preds_a, refs = generate_predictions("./out/run_a_lora_adapter", eval_ds)
print(score(preds_a, refs))

print("\n=== Run B: DoRA + rsLoRA ===")
preds_b, _ = generate_predictions("./out/run_b_dora_rslora_adapter", eval_ds)
print(score(preds_b, refs))

# Show 3 sample deltas
import random
random.seed(0)
for i in random.sample(range(len(eval_ds)), 3):
    ex = eval_ds[i]
    gold = ex["messages"][-1]["content"]
    print(f"\n--- example {i} ---")
    print(f"GOLD:       {gold[:120]}")
    print(f"LoRA:       {preds_a[i][:120]}")
    print(f"DoRA+rslora:{preds_b[i][:120]}")
```

Run it: `python run_eval.py`.

### What you should observe (the FT09 hypothesis)

Expect DoRA + rsLoRA to score higher than vanilla LoRA on the held-out set — most visibly on ROUGE-L (a partial-credit generation metric), sometimes on exact-match. The gap is typically a few points, not a blowout. The point is the *direction*: DoRA ahead, especially at this low rank (r=16), where the magnitude/direction decoupling buys the most headroom. If on your particular data the gap is small or even slightly negative, that is itself a finding — it tells you your task is close enough to the base distribution that the DoRA advantage is muted (see teaching doc section 9.1, "Where DoRA wins most").

If the LoRA run fails to improve at all, or DoRA produces NaN, check: tokenizer template (FT07), EOS handling (FT07), and that `use_rslora=True` is actually set (a common copy-paste miss). The most common "DoRA didn't help" report traces to a config mistake, not the method.

---

## Deliverables

Submit `ft09-lab-report.md` containing:

- [ ] **The two configs** (Run A and Run B), annotated. Highlight exactly which lines differ.
- [ ] **Trainable parameter counts** for both adapters (from `print_trainable_parameters()`). Confirm DoRA's overhead is negligible (~the magnitude vectors).
- [ ] **The eval table** — exact-match and ROUGE-L for Run A and Run B on the 200-row held-out set.
- [ ] **3 sample generations** from each adapter, side by side with the gold, with a one-line note on what differs.
- [ ] **Your verdict**: did DoRA + rsLoRA win on your data? By how much? Was the gap larger or smaller than you expected, and why (relate to "where DoRA wins most" in the teaching doc)?
- [ ] **A 4–6 sentence reflection**: why is a controlled same-base/same-data/same-eval A/B the *only* valid way to adopt a new PEFT method? Why is benchmark-shopping (switching because a paper claimed a win) the anti-pattern?

---

## Solution key

These are *defensible* results and interpretations, not the only possible numbers. Your exact scores will depend on the dataset slice, seed, and base model version. The structure of the answer is what matters.

### Expected parameter counts

For SmolLM2-135M-Instruct with the 7 target modules above at r=16:
- **Vanilla LoRA**: roughly ~0.5–1.0 M trainable params (depends on hidden size / layer count).
- **DoRA + rsLoRA**: the same LoRA matrices, plus a magnitude vector of length `hidden_size` per target module — typically +1–3% over the vanilla LoRA count. The `print_trainable_parameters()` line confirms this; the magnitude overhead is small.

### Expected quality delta

On smoltalk-style instruction-following at r=16, DoRA + rsLoRA typically beats vanilla LoRA by **1–4 ROUGE-L points** and a smaller exact-match delta. The gap is larger at lower ranks (try r=8 to amplify it) and smaller at higher ranks (try r=64, where LoRA has more room and rsLoRA's scaling matters more than DoRA's decomposition). On tasks that are a large domain shift from the base, the DoRA gap grows.

### Verdict template

> On my held-out 200-row eval, DoRA + rsLoRA scored ROUGE-L of **X.XX** vs vanilla LoRA's **Y.YY**, a **+Z.Z** point lift, at ~negligible extra parameter cost. The gap is [larger / smaller / comparable] than expected because [my task is close to / far from the base distribution]. On this evidence I would [adopt DoRA + rsLoRA as the default / keep vanilla LoRA for this specific task / re-run at a different rank to confirm].

### Reflection (model answer)

A controlled A/B — same base, same data, same eval, only the adapter method changed — is the only way to isolate the effect of the PEFT method itself. If you change two things (e.g., method and rank, or method and dataset), you cannot attribute the delta. Benchmark-shopping is the anti-pattern because a paper's benchmark is never your data: a method that wins on a generic benchmark may lose on your distribution, and the only way to know is to measure. This is why FT09 frames DoRA not as "always better" but as "the 2026 default you confirm on your own held-out set before adopting." The lab is that confirmation step, miniaturized.

---

## Stretch goals

1. **Sweep the rank.** Re-run the A/B at r=8, r=32, r=64. Plot ROUGE-L vs rank for vanilla LoRA and DoRA + rsLoRA on the same axes. Expect: (a) DoRA's lead shrinks as rank grows; (b) vanilla LoRA stalls or degrades above r=32 unless you also set `use_rslora=True` — that is the rsLoRA effect, demonstrated.
2. **Isolate DoRA from rsLoRA.** Run a 2×2: `{use_dora: True/False} × {use_rslora: True/False}` at r=64. Confirm the two improvements are additive — the DoRA+rsLoRA cell beats either single-flag cell, which both beat vanilla LoRA.
3. **QDoRA on a bigger model.** Swap the base for `Qwen/Qwen2.5-1.5B-Instruct`, load it in 4-bit (`BitsAndBytesConfig(load_in_4bit=True)`), and run DoRA + rsLoRA on top — that is QDoRA. Confirm it fits on a 12 GB card and produces quality above what QLoRA gave you on the same model. This is the 2026 quality/cost frontier from the teaching doc.
4. **PiSSA initialization.** Add PiSSA initialization to the DoRA + rsLoRA run (PEFT supports `init_lora_weights="pissa"`). Measure whether convergence is faster (fewer steps to a target loss) on your data.

---

## No-GPU fallback

If you have no CUDA GPU: you can still complete the *inspection* portion of the lab — load each `LoraConfig`, call `get_peft_model`, run `print_trainable_parameters()`, and confirm the magnitude overhead and the `use_rslora` flag take effect. You can also verify that `merge_and_unload()` produces a weight tensor of the same shape as the base (the zero-inference-overhead property). The training runs themselves require a GPU; substitute the parameter-count and config-inspection deliverables for the eval table if you are CPU-only, and note this in your report.