Deploy a vLLM Inference Server

runcrate instances create --name llm-server --gpu A100

runcrate instances status llm-server

runcrate ssh llm-server -- "pip install vllm"

runcrate ssh llm-server -- "nohup python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 1 \
  --max-model-len 8192 \
  --port 8000 \
  --host 0.0.0.0 \
  > /root/vllm.log 2>&1 &"

# Get the instance IP
runcrate instances info llm-server

# Hit the API
curl http://<INSTANCE_IP>:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-70B-Instruct",
    "messages": [{"role": "user", "content": "What is PagedAttention?"}],
    "max_tokens": 256
  }'

from openai import OpenAI

client = OpenAI(
    base_url="http://<INSTANCE_IP>:8000/v1",
    api_key="not-needed",
)

response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-70B-Instruct",
    messages=[{"role": "user", "content": "Explain vLLM in one sentence."}],
)
print(response.choices[0].message.content)

from runcrate import Runcrate
import time

client = Runcrate(api_key="rc_live_...")

# Deploy an A100
instance = client.instances.create(
    name="llm-server",
    gpu_type="A100",
    gpu_count=1,
    startup_commands=[
        "pip install vllm",
        "nohup python -m vllm.entrypoints.openai.api_server "
        "--model meta-llama/Llama-3.1-70B-Instruct "
        "--tensor-parallel-size 1 "
        "--max-model-len 8192 "
        "--port 8000 --host 0.0.0.0 > /root/vllm.log 2>&1 &",
    ],
)

# Wait for deployment
while True:
    status = client.instances.get_status(instance.id)
    if status.status == "deployed":
        print(f"Server ready at http://{status.ip}:8000")
        break
    time.sleep(10)

runcrate instances create --name llm-server-4gpu --gpu H100 --gpu-count 4

runcrate ssh llm-server-4gpu -- "nohup python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-405B-Instruct-FP8 \
  --tensor-parallel-size 4 \
  --max-model-len 16384 \
  --port 8000 --host 0.0.0.0 > /root/vllm.log 2>&1 &"

# Check GPU memory and utilization
runcrate ssh llm-server -- nvidia-smi

# Check vLLM logs
runcrate ssh llm-server -- "tail -50 /root/vllm.log"

# Check active request count
runcrate ssh llm-server -- "curl -s localhost:8000/metrics | grep vllm_num_requests"

runcrate instances delete llm-server