LiteLLM on the Homelab: One Proxy, Every Model, Waterfall Routing Included

# docker run command for LiteLLM
docker run -d \
  --name litellm \
  --restart unless-stopped \
  -p 4000:4000 \
  -v /mnt/nvme-pool/apps/litellm:/litellm \
  -e OPENAI_API_KEY=${OPENAI_API_KEY} \
  -e ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} \
  -e GROQ_API_KEY=${GROQ_API_KEY} \
  -v /mnt/Storage_Pool/models:/models \
  ghcr.io/berriai/litellm:main \
  --config /litellm/config.yaml \
  --port 4000 \
  --host 0.0.0.0

# Alternative: Using docker-compose
# Create docker-compose.yml:
version: '3.8'
services:
  litellm:
    image: ghcr.io/berriai/litellm:main
    container_name: litellm
    restart: unless-stopped
    ports:
      - "4000:4000"
    volumes:
      - ./config:/litellm
      - /mnt/Storage_Pool/models:/models
    environment:
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
      - GROQ_API_KEY=${GROQ_API_KEY}
    command: >
      --config /litellm/config.yaml
      --port 4000
      --host 0.0.0.0

# Install LiteLLM directly
pip install litellm

# Create a basic config file (config.yaml)
cat > config.yaml << 'EOF'
model_list:
  - model_name: ollama_qwen
    litellm_provider: ollama
    model_name: qwen2.5-coder:14b
    api_base: http://localhost:11434

  - model_name: openai_backup
    litellm_provider: openai
    model: gpt-4o-mini
    api_key: ${OPENAI_API_KEY}

# Start LiteLLM
litellm --config config.yaml --port 4000 --host 0.0.0.0

# Add LiteLLM helm repo
helm repo add litellm https://berriai.github.io/litellm-helm-chart
helm repo update

# Install with custom values
helm install litellm litellm/litellm \
  --namespace ai \
  --create-namespace \
  --set service.type=LoadBalancer \
  --set service.port=4000 \
  --set env.OPENAI_API_KEY=${OPENAI_API_KEY} \
  --set env.ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} \
  --set config.modelList="[
    {model_name: 'ollama_qwen', litellm_provider: 'ollama', model_name: 'qwen2.5-coder:14b', api_base: 'http://localhost:11434'},
    {model_name: 'openai_backup', litellm_provider: 'openai', model: 'gpt-4o-mini', api_key: env.OPENAI_API_KEY}
  ]"

# config.yaml for development assistant
model_list:
  # Primary: Fast local model for coding help
  - model_name: dev_primary
    litellm_provider: ollama
    model_name: qwen2.5-coder:14b
    api_base: http://localhost:11434
    tpm: 8000
    rpm: 100

  # Secondary: Capable local model for complex reasoning
  - model_name: dev_secondary
    litellm_provider: ollama
    model_name: deepseek-r1:14b
    api_base: http://localhost:11434
    tpm: 4000
    rpm: 60

  # Tertiary: Small local model for simple tasks (faster, less capable)
  - model_name: dev_tertialy
    litellm_provider: ollama
    model_name: llama3.1:8b
    api_base: http://localhost:11434
    tpm: 10000
    rpm: 200

  # Emergency fallback: Only use when local is truly inadequate
  - model_name: dev_emergency
    litellm_provider: openai
    model: gpt-3.5-turbo
    api_key: ${OPENAI_API_KEY}
    tpm: 500
    rpm: 20

# Use LLMSingleAgent routing strategy (try in order until success)
# This creates a waterfall: try dev_primary -> dev_secondary -> dev_tertialy -> dev_emergency

# config.yaml for content generation workflow
model_list:
  # Drafting: Use capable local models
  - model_name: content_draft
    litellm_provider: ollama
    model_name: qwen2.5-coder:14b
    api_base: http://localhost:11434
    tpm: 6000
    rpm: 80

  - model_name: content_draft_alt
    litellm_provider: ollama
    model_name: deepseek-r1:7b
    api_base: http://localhost:11434
    tpm: 4000
    rpm: 60

  # Polishing: Use better models for final output
  - model_name: content_polish
    litellm_provider: openai
    model: gpt-4o
    api_key: ${OPENAI_API_KEY}
    tpm: 2000
    rpm: 50

  # Budget-conscious alternative for polishing
  - model_name: content_polish_alt
    litellm_provider: openai
    model: gpt-3.5-turbo
    api_key: ${OPENAI_API_KEY}
    tpm: 3000
    rpm: 80

  # Specialized: For creative writing tasks
  - model_name: content_creative
    litellm_provider: anthropic
    model: claude-3-sonnet-20240229
    api_key: ${ANTHROPIC_API_KEY}
    tpm: 3000
    rpm: 40

# Different routing strategies for different workflow stages
# Drafting: LLMSingleAgent (try local models first)
# Polishing: LLMSingleAgent with cost preference (try cheaper OpenAI first)
# Creative: LLMSingleAgent (try claude sonnet then fall back)

# config.yaml for budget-conscious service
model_list:
  # Local models: Free to run (after hardware investment)
  - model_name: local_fast
    litellm_provider: ollama
    model_name: qwen2.5-coder:14b
    api_base: http://localhost:11434
    tpm: 10000
    rpm: 120

  - model_name: local_capable
    litellm_provider: ollama
    model_name: llama3.1:70b
    api_base: http://localhost:11434
    tpm: 3000
    rpm: 30

  # Remote models: Tracked against budget
  - model_name: remote_tier1
    litellm_provider: openai
    model: gpt-3.5-turbo
    api_key: ${OPENAI_API_KEY}
    tpm: 2000
    rpm: 100

  - model_name: remote_tier2
    litellm_provider: openai
    model: gpt-4o-mini
    api_key: ${OPENAI_API_KEY}
    tpm: 1000
    rpm: 50

  - model_name: remote_tier3
    litellm_provider: anthropic
    model: claude-3-haiku-20240307
    api_key: ${ANTHROPIC_API_KEY}
    tpm: 1500
    rpm: 75

# Global budget settings: $50/month maximum
budget:
  duration: monthly
  max_budget: 50.0
  # Exclude local models from budget tracking (they're "free")
  excludes: ["local_fast", "local_capable"]
  # Optional: send alert when 80% of budget is used
  # callbacks: [budget_alert_webhook]

# Use LLMPriority routing with cost weights
# Assign lower weights (higher priority) to cheaper/freer models
model_aliases:
  # Local models get highest priority (lowest weight numbers)
  fast_local: local_fast
  capable_local: local_capable

  # Remote models get lower priority (higher weight numbers)
  tier1_remote: remote_tier1
  tier2_remote: remote_tier2
  tier3_remote: remote_tier3

# The routing will try models in priority order:
# 1. local_fast (weight 1 - highest priority)
# 2. local_capable (weight 2)
# 3. remote_tier1 (weight 3)
# 4. remote_tier2 (weight 4)
# 5. remote_tier3 (weight 5 - lowest priority)

# Python example using official OpenAI SDK
from openai import OpenAI

# Configure the client to point to your LiteLLM instance
client = OpenAI(
    base_url="http://homelab.local:4000/v1",  # Point to LiteLLM
    api_key="sk-12345"  # Can be any non-empty string for LiteLLM
)

# Now use it exactly as you would the real OpenAI API
def get_code_explanation(code_snippet):
    """Get explanation of code using locally-hosted models when possible"""
    try:
        response = client.chat.completions.create(
            model="ollama_qwen_coder",  # This gets routed by LiteLLM
            messages=[
                {"role": "system", "content": "You are an expert SAP ABAP developer."},
                {"role": "user", "content": f"Explain this ABAP code snippet:\n\n{code_snippet}"}
            ],
            temperature=0.3,
            max_tokens=500
        )

        return response.choices[0].message.content

    except Exception as e:
        # LiteLLM will have already tried fallbacks, so this means all options failed
        return f"Error generating explanation: {str(e)}"

def generate_content_outline(topic, audience="technical"):
    """Generate content outline using potentially different models"""
    try:
        response = client.chat.completions.create(
            model="content_draft",  # Routes to drafting model
            messages=[
                {"role": "system", "content": "You are an expert technical writer and content strategist."},
                {"role": "user", "content": f"Create a detailed outline for a {audience}-focused article about {topic}. Include sections, subsections, and key points to cover."}
            ],
            temperature=0.7,
            max_tokens=800
        )

        return response.choices[0].message.content

    except Exception as e:
        return f"Error generating outline: {str(e)}"

# Example usage
if __name__ == "__main__":
    sap_code = '''
    REPORT z_sales_analysis.

    SELECT vbeln, erdat, netwr, kitae
      FROM vbap
      INTO TABLE @DATA(items)
      WHERE erdat >= @sy-date-30
        AND venda IN ('DE', 'FR', 'UK').

    LOOP AT items ASSIGNING FIELD-SYMBOL().
      WRITE: / -vbeln, -erdat, -netwr, -kitae.
    ENDLOOP.
    '''

    explanation = get_code_explanation(sap_code)
    print("Code Explanation:")
    print(explanation)
    print("\n" + "="*50 + "\n")

    outline = generate_content_outline("SAP AI Units pricing model")
    print("Content Outline:")
    print(outline)

// JavaScript/TypeScript example
// Using the official OpenAI npm package
import { OpenAI } from "openai";

// Configure for LiteLLM endpoint
const client = new OpenAI({
  baseURL: "http://homelab.local:4000/v1",
  apiKey: "sk-12345" // Any non-empty string works with LiteLLM
});

// Function to generate SAP-inspired content
async function generateSAPBlogPost(topic) {
  try {
    const response = await client.chat.completions.create({
      model: "content_draft", // Routes to appropriate local model
      messages: [
        {
          role: "system",
          content: "You are an expert technical writer specializing in SAP and enterprise AI. Write in a practical, skeptical-yet-hopeful tone."
        },
        {
          role: "user",
          content: `Write a comprehensive blog post about ${topic} for enterprise architects. Include practical examples, implementation considerations, and realistic timelines. Target length: 1200-1500 words.`
        }
      ],
      temperature: 0.6,
      max_tokens: 1500
    });

    return response.choices[0].message.content;
  } catch (error) {
    console.error("Error generating blog post:", error);
    throw new Error(`Failed to generate content: ${error.message}`);
  }
}

// Function to get quick coding help
async function getCodingHelp(language, problem) {
  try {
    const response = await client.chat.completions.create({
      model: "dev_primary", // Fast local model for coding help
      messages: [
        {
          role: "system",
          content: `You are an expert ${language} programmer. Provide clear, concise solutions to programming problems.`
        },
        {
          role: "user",
          content: `How do I solve this problem in ${language}: ${problem}`
        }
      ],
      temperature: 0.2,
      max_tokens: 400
    });

    return response.choices[0].message.content;
  } catch (error) {
    console.error("Error getting coding help:", error);
    throw new Error(`Failed to get coding help: ${error.message}`);
  }

// Example usage in a web app context
async function handleUserRequest(userInput) {
  try {
    // Determine what kind of help the user needs
    if (userInput.startsWith("/code ")) {
      const [, language, ...problemParts] = userInput.split(" ");
      const problem = problemParts.join(" ");
      const help = await getCodingHelp(language || "javascript", problem);
      return { type: "code_help", content: help };
    } else if (userInput.startsWith("/write ")) {
      const topic = userInput.substring(7).trim();
      const blogPost = await generateSAPBlogPost(topic);
      return { type: "blog_post", content: blogPost };
    } else {
      // General conversation
      const response = await client.chat.completions.create({
        model: "content_draft",
        messages: [
          { role: "system", content: "You are a helpful AI assistant." },
          { role: "user", content: userInput }
        ],
        temperature: 0.7,
        max_tokens: 800
      });

      return { type: "conversation", content: response.choices[0].message.content };
    }
  } catch (error) {
    return { type: "error", content: `Sorry, I encountered an error: ${error.message}` };
  }
}

# Basic health check
curl -s http://homelab.local:4000/health

# Should return:
# {"status": "OK"}

# List available models
curl -s http://homelab.local:4000/v1/models | jq '.data[].id'

# Generate text completion
curl -s -X POST http://homelab.local:4000/v1/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer sk-12345" \
  -d '{
    "model": "ollama_qwen_coder",
    "prompt": "Explain the concept of thermal dynamics in simple terms:",
    "max_tokens": 200,
    "temperature": 0.3
  }' | jq '.choices[0].text'

# Chat completion
curl -s -X POST http://homelab.local:4000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer sk-12345" \
  -d '{
    "model": "content_draft",
    "messages": [
      {"role": "system", "content": "You are a helpful SAP consultant."},
      {"role": "user", "content": "What are the key considerations when migrating from SAP ECC to S/4HANA?"}
    ],
    "temperature": 0.5,
    "max_tokens": 500
  }' | jq '.choices[0].message.content'

# Embedding generation (useful for RAG)
curl -s -X POST http://homelab.local:4000/v1/embeddings \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer sk-12345" \
  -d '{
    "model": "nomic_embed_text",
    "input": "How to optimize SQL queries for SAP BW"
  }' | jq '.data[0].embedding[:5]' # Show first 5 dimensions

# Streaming response (for real-time UI)
curl -s -N -X POST http://homelab.local:4000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer sk-12345" \
  -d '{
    "model": "dev_primary",
    "messages": [{"role": "user", "content": "Write a haiku about debugging code"}],
    "temperature": 0.8,
    "max_tokens": 50,
    "stream": true
  }'

# Add to config.yaml for caching
# Note: Requires redis instance running

caching:
  type: redis
  host: localhost
  port: 6379
  # Optional: requirepass: your_redis_password
  # Optional: username: default

# Cache behavior:
# - Identical requests (same model, messages, parameters) return cached result
# - TTL defaults to 1 hour but can be configured
# - Especially useful for:
#   * Frequently asked questions
#   * Code snippet explanations
#   * Standardized report sections
#   * Translation of common phrases

# Example cache key generation:
#Request: model=ollama_qwen, messages=[{"role":"user","content":"What is 2+2?"}]
#Cache key includes: model, messages hash, parameters hash
#Result: Cached for subsequent identical requests

# Create a custom transformer module (transformers.py)
# Place in your LiteLLM config directory

def modify_params(params, model_alias, api_key, optional_params):
    """Modify request parameters before sending to provider"""

    # Add system prompt for SAP-specific queries
    if model_alias in ["content_draft", "content_polish"]:
        if "messages" in params:
            # Check if we already have a system message
            has_system = any(msg.get("role") == "system" for msg in params["messages"])

            if not has_system:
                # Insert SAP-focused system message at the beginning
                params["messages"].insert(0, {
                    "role": "system",
                    "content": "You are an expert SAP technical consultant with 20+ years of experience. Provide practical, implementation-focused advice."
                })

    # Adjust temperature based on model type
    if model_alias.startswith("dev_"):
        # Lower temperature for more deterministic code help
        params["temperature"] = min(params.get("temperature", 0.7), 0.3)
    elif model_alias.startswith("content_"):
        # Slightly higher temperature for creative writing
        params["temperature"] = min(params.get("temperature", 0.7), 0.8)

    return params

def modify_response(response, model_alias, api_key, optional_params):
    """Modify response after receiving from provider"""

    # Add usage hints for coding responses
    if model_alias.startswith("dev_") and "choices" in response:
        choice = response["choices"][0]
        if choice["message"]["content"].strip():
            # Add a helpful footer to code explanations
            choice["message"]["content"] += "\n\n💡 Tip: Consider adding unit tests for this implementation."

    # Add word count for content generation
    if model_alias.startswith("content_") and "choices" in response:
        choice = response["choices"][0]
        word_count = len(choice["message"]["content"].split())
        choice["message"]["content"] += f"\n\n*Word count: {word_count}*"

    return response

# Then in config.yaml:
# custom_llm_provider:
#   custom_handler: true
#   params_modifier_fn: transformers.modify_params
#   response_modifier_fn: transformers.modify_response

# Install the LangFuse integration
pip install litellm-langfuse

# Configure LiteLLM to use LangFuse
# In config.yaml:
callbacks:
  - langfuse

# Set LangFuse credentials as environment variables
# docker run -e LANGFUSE_PUBLIC_KEY=pk-lf-... \
#            -e LANGFUSE_SECRET_KEY=sk-lf-... \
#            -e LANGFUSE_HOST=http://langfuse.local:3100 \
#            ...

# Once configured, you'll get:
# - Automatic tracing of all LLM calls
# - Token usage and latency metrics
# - Ability to trace complex chains and agents
# - Feedback collection UI
# - Cost tracking per project/user
# - Trace visualization in LangFuse dashboard

# Example of what you can do with LangFuse data:
# 1. See which models are used most frequently
# 2. Identify slow or failing requests
# 3. Track cost by project or user
# 4. Optimize routing based on actual usage patterns
# 5. Set up alerts for anomalies or budget overruns