LiteLLM Proxy：單一 API 接入 100+ LLM（15 分鐘 Docker 部署）

# Check Docker is installed
docker --version
docker compose version

# Export your LLM API keys (add to your shell profile for persistence)
export OPENAI_API_KEY="sk-..."
export ANTHROPIC_API_KEY="sk-ant-..."

# Optional: set a master key for your proxy (you'll need this later)
export LITELLM_MASTER_KEY="sk-master-your-secret-key"

docker run -d \
  --name litellm-proxy \
  -p 4000:4000 \
  -e OPENAI_API_KEY=$OPENAI_API_KEY \
  -e LITELLM_MASTER_KEY=$LITELLM_MASTER_KEY \
  ghcr.io/berriai/litellm:main-stable \
  --model openai/gpt-4o

curl http://localhost:4000/v1/chat/completions \
  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "openai/gpt-4o",
    "messages": [{"role": "user", "content": "Say hello from LiteLLM"}]
  }'

from openai import OpenAI

# Point the standard OpenAI SDK at your proxy
client = OpenAI(
    api_key="sk-master-your-secret-key",
    base_url="http://localhost:4000/v1"
)

response = client.chat.completions.create(
    model="openai/gpt-4o",
    messages=[{"role": "user", "content": "Say hello from LiteLLM"}]
)
print(response.choices[0].message.content)

# docker-compose.yml
version: "3.9"

services:
  litellm:
    image: ghcr.io/berriai/litellm:main-stable
    container_name: litellm-proxy
    ports:
      - "4000:4000"         # Proxy API port
    volumes:
      - ./config.yaml:/app/config.yaml   # Mount your config file
    environment:
      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
      - DATABASE_URL=postgresql://litellm:litellm_password@postgres:5432/litellm
      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY:-sk-salt-random-string}
    command: --config /app/config.yaml --detailed_debug
    depends_on:
      postgres:
        condition: service_healthy
    restart: unless-stopped

  postgres:
    image: postgres:16-alpine
    container_name: litellm-db
    environment:
      POSTGRES_DB: litellm
      POSTGRES_USER: litellm
      POSTGRES_PASSWORD: litellm_password
    volumes:
      - litellm_pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U litellm"]
      interval: 5s
      timeout: 5s
      retries: 5
    restart: unless-stopped

volumes:
  litellm_pgdata:

# Create a .env file with your keys (don't commit this to git)
echo "LITELLM_MASTER_KEY=sk-master-your-secret" > .env
echo "OPENAI_API_KEY=sk-..." >> .env
echo "ANTHROPIC_API_KEY=sk-ant-..." >> .env
echo "LITELLM_SALT_KEY=sk-salt-$(openssl rand -hex 16)" >> .env

# Start everything
docker compose up -d

# Check logs
docker compose logs -f litellm

# Health check
curl http://localhost:4000/health

# Test a request
curl http://localhost:4000/v1/chat/completions \
  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "ping"}]}'

# config.yaml -- Real multi-provider setup
model_list:
  # Primary: OpenAI GPT-4o
  - model_name: gpt-4o          # The name YOUR code uses
    litellm_params:
      model: openai/gpt-4o      # The actual provider/model
      api_key: os.environ/OPENAI_API_KEY

  # Secondary: Anthropic Claude
  - model_name: claude-sonnet
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY

  # Local: Ollama for development / cost-free testing
  - model_name: local-llama
    litellm_params:
      model: ollama/llama3.1
      api_base: http://host.docker.internal:11434

  # Fallback: route "gpt-4o" to Claude if OpenAI is down
  - model_name: gpt-4o
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY

router_settings:
  routing_strategy: least-busy    # Load balance across same-name models
  num_retries: 3
  retry_after: 5                  # Seconds between retries
  fallbacks: [{"gpt-4o": ["claude-sonnet"]}]

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL

# Create a virtual key with a $50/month budget
curl http://localhost:4000/key/generate \
  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "team_id": "frontend-team",
    "max_budget": 50.0,
    "budget_duration": "1mo",
    "models": ["gpt-4o", "claude-sonnet"],
    "metadata": {"purpose": "frontend AI features"}
  }'

# Create a key with rate limits: 100 requests/minute, 50K tokens/minute
curl http://localhost:4000/key/generate \
  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "team_id": "backend-team",
    "max_budget": 200.0,
    "budget_duration": "1mo",
    "rpm_limit": 100,
    "tpm_limit": 50000,
    "models": ["gpt-4o", "claude-sonnet", "local-llama"]
  }'

import requests

# Check a key's current spend and limits
response = requests.get(
    "http://localhost:4000/key/info",
    headers={"Authorization": f"Bearer {MASTER_KEY}"},
    params={"key": "sk-team-abc123..."}
)
info = response.json()
print(f"Spent: ${info['spend']:.2f} / ${info['max_budget']:.2f}")
print(f"RPM used: {info['rpm_limit_used']} / {info['rpm_limit']}")

curl -X POST http://localhost:4000/key/delete \
  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{"keys": ["sk-team-abc123..."]}'

# Set Claude Code to use your LiteLLM proxy
export ANTHROPIC_BASE_URL=http://localhost:4000/v1
export ANTHROPIC_API_KEY=sk-team-your-virtual-key

{
  "openai.apiBaseUrl": "http://localhost:4000/v1",
  "openai.apiKey": "sk-team-your-virtual-key"
}

{
  "models": [
    {
      "title": "GPT-4o via LiteLLM",
      "provider": "openai",
      "model": "gpt-4o",
      "apiBase": "http://localhost:4000/v1",
      "apiKey": "sk-team-your-virtual-key"
    }
  ]
}

# Check the file exists where you think it does
ls -la ./config.yaml

# The volume mount in docker-compose.yml should match
# volumes:
#   - ./config.yaml:/app/config.yaml

# Check if the port is listening
docker port litellm-proxy
# Should show: 4000/tcp -> 0.0.0.0:4000