Download and Serve HuggingFace Models

runcrate instances create --name hf-serve --gpu A100 --template ubuntu-devbox
runcrate instances status hf-serve

runcrate ssh hf-serve -- "pip install huggingface_hub[cli] vllm"

runcrate ssh hf-serve -- "huggingface-cli login --token hf_YOUR_TOKEN"

runcrate ssh hf-serve -- "huggingface-cli download meta-llama/Llama-3.1-8B-Instruct \
  --local-dir /workspace/models/llama-8b"

runcrate ssh hf-serve -- "du -sh /workspace/models/*"

runcrate ssh hf-serve -- "nohup python -m vllm.entrypoints.openai.api_server \
  --model /workspace/models/llama-8b \
  --max-model-len 8192 \
  --port 8000 --host 0.0.0.0 \
  > /root/vllm.log 2>&1 &"

runcrate instances info hf-serve

curl http://<INSTANCE_IP>:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "/workspace/models/llama-8b",
    "messages": [{"role": "user", "content": "Hello from HuggingFace."}],
    "max_tokens": 128
  }'

runcrate volumes create --name hf-models --size 200
runcrate instances create --name hf-serve --gpu A100 --template ubuntu-devbox --storage hf-models

runcrate ssh hf-serve -- "huggingface-cli download meta-llama/Llama-3.1-8B-Instruct \
  --local-dir /workspace/models/llama-8b \
  --include '*.safetensors' '*.json' 'tokenizer*'"

runcrate instances delete hf-serve