Fine-tuning

from datasets import Dataset
from trl import SFTTrainer, SFTConfig
from transformers import AutoModelForCausalLM, AutoTokenizer

# SFT training data: (instruction, response) pairs
data = [
    {
        "messages": [
            {"role": "user", "content": "Summarize this text in 2 sentences: [article text]"},
            {"role": "assistant", "content": "The article discusses..."}
        ]
    },
    # ... thousands more high-quality examples
]
dataset = Dataset.from_list(data)

model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.1-8B")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B")

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=SFTConfig(
        output_dir="./llama-sft",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,   # effective batch = 16
        learning_rate=2e-5,
        warmup_ratio=0.1,
        lr_scheduler_type="cosine",
        bf16=True,                        # bfloat16 — faster, same quality as fp16
        logging_steps=10,
        save_strategy="epoch",
    ),
)
trainer.train()

# pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
# pip install --no-deps trl peft accelerate bitsandbytes

from unsloth import FastLanguageModel
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset
import torch

# ── 1. Load model (4-bit QLoRA via Unsloth) ──────────────────────────────────
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Meta-Llama-3.1-8B-Instruct",
    max_seq_length=2048,
    dtype=None,           # auto-detect: bfloat16 on Ampere, float16 on older
    load_in_4bit=True,    # QLoRA — ~5GB VRAM vs ~16GB for FP16
)

# ── 2. Add LoRA adapters via Unsloth (optimized PEFT) ────────────────────────
model = FastLanguageModel.get_peft_model(
    model,
    r=16,                 # LoRA rank
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    lora_alpha=16,
    lora_dropout=0,       # Unsloth: 0 dropout is optimized for speed
    bias="none",
    use_gradient_checkpointing="unsloth",   # 30% more context length
    random_state=42,
)

# ── 3. Format dataset in ChatML / Alpaca format ───────────────────────────────
dataset = load_dataset("yahma/alpaca-cleaned", split="train")

alpaca_prompt = """Below is an instruction. Write a response.
### Instruction: {}
### Response: {}"""

def format_prompts(examples):
    texts = []
    for instr, output in zip(examples["instruction"], examples["output"]):
        texts.append(alpaca_prompt.format(instr, output) + tokenizer.eos_token)
    return {"text": texts}

dataset = dataset.map(format_prompts, batched=True)

# ── 4. Train ──────────────────────────────────────────────────────────────────
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
    args=SFTConfig(
        output_dir="outputs",
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_steps=5,
        max_steps=60,          # full run: set num_train_epochs=3
        learning_rate=2e-4,
        fp16=not torch.cuda.is_bf16_supported(),
        bf16=torch.cuda.is_bf16_supported(),
        logging_steps=1,
        optim="adamw_8bit",    # Unsloth: 8-bit Adam — less memory, same quality
        lr_scheduler_type="linear",
    ),
)

trainer.train()

# ── 5. Save and merge ─────────────────────────────────────────────────────────
model.save_pretrained("lora_model")         # saves LoRA weights only (~300MB)
model.save_pretrained_merged("merged_model", tokenizer,
                             save_method="merged_16bit")  # merge + save full model

from openai import OpenAI
import json

client = OpenAI()

# ── Step 1: Prepare JSONL training data ─────────────────────────────────────
# Each line: one complete conversation with the ideal response
training_examples = [
    {
        "messages": [
            {"role": "system", "content": "You are a concise SQL expert. Always return SQL in a fenced code block. Never explain unless asked."},
            {"role": "user", "content": "Get all users who signed up last month"},
            {"role": "assistant", "content": "```sql
SELECT * FROM users
WHERE created_at >= date_trunc('month', now() - interval '1 month')
  AND created_at < date_trunc('month', now());
```"}
        ]
    },
    # ... 50+ more high-quality examples (minimum 10, recommended 50–200)
]

with open("training_data.jsonl", "w") as f:
    for ex in training_examples:
        f.write(json.dumps(ex) + "
")

# ── Step 2: Upload training file ──────────────────────────────────────────────
upload = client.files.create(
    file=open("training_data.jsonl", "rb"),
    purpose="fine-tune"
)

# ── Step 3: Create fine-tuning job ────────────────────────────────────────────
job = client.fine_tuning.jobs.create(
    training_file=upload.id,
    model="gpt-4o-mini-2024-07-18",    # cheapest; also supports gpt-4o
    hyperparameters={
        "n_epochs": 3,                  # 3 epochs default; increase for <50 examples
    }
)

print(f"Job ID: {job.id}")  # monitor at platform.openai.com/finetune

# ── Step 4: Use the fine-tuned model ─────────────────────────────────────────
# (after job status = "succeeded", typically 15–60 minutes)
response = client.chat.completions.create(
    model=job.fine_tuned_model,   # e.g. "ft:gpt-4o-mini:org:name:abc123"
    messages=[{"role": "user", "content": "Count users by country"}]
)
print(response.choices[0].message.content)

# Cost: ~$3 per 1M training tokens.  
# A dataset of 100 examples × 500 tokens each × 3 epochs ≈ 150K tokens ≈ $0.45 total.