Run Docker Containers on GPU Instances

runcrate instances create --name docker-gpu --gpu RTX4090 --template ubuntu-devbox
runcrate instances status docker-gpu

runcrate ssh docker-gpu -- "docker --version"
runcrate ssh docker-gpu -- "docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi"

runcrate ssh docker-gpu -- "docker run --rm --gpus all \
  -v /workspace:/workspace \
  pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime \
  python -c 'import torch; print(f\"CUDA: {torch.cuda.is_available()}, GPU: {torch.cuda.get_device_name(0)}\")'"

runcrate ssh docker-gpu -- "docker run -d --gpus all \
  --name vllm-server \
  -p 8000:8000 \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --max-model-len 8192 --host 0.0.0.0"

runcrate instances info docker-gpu

curl http://<INSTANCE_IP>:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [{"role": "user", "content": "Hello from Docker."}],
    "max_tokens": 128
  }'

runcrate cp ./Dockerfile docker-gpu:/workspace/Dockerfile
runcrate cp ./app/ docker-gpu:/workspace/app/

runcrate ssh docker-gpu -- "cd /workspace && docker build -t my-gpu-app ."
runcrate ssh docker-gpu -- "docker run --rm --gpus all my-gpu-app"

runcrate ssh docker-gpu -- "docker ps"
runcrate ssh docker-gpu -- "docker logs vllm-server --tail 50"
runcrate ssh docker-gpu -- "docker stop vllm-server"

runcrate instances delete docker-gpu