Fine-Tune an LLM with LoRA

{"messages": [{"role": "user", "content": "What's your refund policy?"}, {"role": "assistant", "content": "We offer full refunds within 30 days of purchase. After 30 days, we provide store credit."}]}
{"messages": [{"role": "user", "content": "How do I track my order?"}, {"role": "assistant", "content": "Go to Orders in your account dashboard and click on the order number. You'll see real-time tracking."}]}

runcrate instances create --name finetune --gpu A100

# Upload dataset and training script
runcrate cp ./train_data.jsonl finetune:/root/
runcrate cp ./finetune.py finetune:/root/

runcrate ssh finetune -- "pip install torch transformers datasets peft trl accelerate bitsandbytes"

from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig

model_id = "meta-llama/Llama-3.1-8B-Instruct"

# QLoRA: load model in 4-bit
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype="bfloat16",
)

tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto",
)

# LoRA config — rank 16 for domain adaptation
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_dropout=0.05,
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

# Load dataset
dataset = load_dataset("json", data_files="/root/train_data.jsonl", split="train")

# Train
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=SFTConfig(
        output_dir="/root/output",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        learning_rate=2e-4,
        logging_steps=10,
        save_strategy="epoch",
        bf16=True,
    ),
    processing_class=tokenizer,
)

trainer.train()
trainer.save_model("/root/output/final")
tokenizer.save_pretrained("/root/output/final")
print("Training complete.")

runcrate ssh finetune -- "cd /root && python finetune.py"

# Check progress
runcrate ssh finetune -- "tail -20 /root/output/training.log"

# Watch GPU utilization
runcrate ssh finetune -- "nvidia-smi"

# Download the LoRA adapter
runcrate cp -r finetune:/root/output/final/ ./my-adapter/

# Tear down
runcrate instances delete finetune

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
model = PeftModel.from_pretrained(base_model, "./my-adapter")
merged = model.merge_and_unload()

merged.save_pretrained("./merged-model")
AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct").save_pretrained("./merged-model")

runcrate ssh server -- "python -m vllm.entrypoints.openai.api_server --model /root/merged-model --port 8000 --host 0.0.0.0"

from runcrate import Runcrate
import time

client = Runcrate(api_key="rc_live_...")

# Deploy with all dependencies pre-installed
instance = client.instances.create(
    name="finetune",
    gpu_type="A100",
    gpu_count=1,
    startup_commands=[
        "pip install torch transformers datasets peft trl accelerate bitsandbytes",
    ],
)

# Wait for deployment
while True:
    status = client.instances.get_status(instance.id)
    if status.status == "deployed":
        break
    time.sleep(10)

print(f"Ready — SSH: root@{status.ip}")