Self-Hosting AI Models: Complete Infrastructure Guide

# Linux/macOS
curl -fsSL https://ollama.com/install.sh | sh

# Or with Docker
docker run -d --gpus all -v ollama:/root/.ollama -p 11434:11434 ollama/ollama

# Llama 3.1 8B (default)
ollama pull llama3.1

# Llama 3.1 70B (needs ~40GB VRAM with Q4)
ollama pull llama3.1:70b

# Mistral 7B
ollama pull mistral

# Code-specialized
ollama pull codellama:34b

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [{"role": "user", "content": "Hello!"}],
  "stream": false
}'

import OpenAI from 'openai';

const client = new OpenAI({
  baseURL: 'http://localhost:11434/v1',
  apiKey: 'ollama', // Required but unused
});

const response = await client.chat.completions.create({
  model: 'llama3.1',
  messages: [{ role: 'user', content: 'Hello!' }],
});

pip install vllm

# Or with Docker
docker run --runtime nvidia --gpus all \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  vllm/vllm-openai:latest \
  --model meta-llama/Meta-Llama-3.1-8B-Instruct

python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-8B-Instruct \
  --tensor-parallel-size 1 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9

python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2 \
  --max-model-len 4096 \
  --gpu-memory-utilization 0.95

# docker-compose.yml
version: '3.8'
services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - '8000:8000'
    volumes:
      - huggingface-cache:/root/.cache/huggingface
    command: >
      --model meta-llama/Meta-Llama-3.1-8B-Instruct
      --tensor-parallel-size 1
      --max-model-len 8192
    environment:
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN}
    restart: unless-stopped

volumes:
  huggingface-cache:

docker run --gpus all --shm-size 1g -p 8080:80 \
  -v /data:/data \
  ghcr.io/huggingface/text-generation-inference:latest \
  --model-id meta-llama/Meta-Llama-3.1-8B-Instruct \
  --max-input-length 4096 \
  --max-total-tokens 8192

curl http://localhost:8080/generate \
  -X POST \
  -H 'Content-Type: application/json' \
  -d '{"inputs": "What is AI?", "parameters": {"max_new_tokens": 100}}'

┌─────────────────────────────────────────────────────────┐
│                    Your Server                           │
│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐     │
│  │   Nginx     │  │   vLLM/TGI  │  │   Redis     │     │
│  │  (Reverse   │->│  (Model     │->│  (Queue/    │     │
│  │   Proxy)    │  │   Server)   │  │   Cache)    │     │
│  └─────────────┘  └─────────────┘  └─────────────┘     │
│         │                │                              │
│         │         ┌──────┴──────┐                      │
│         │         │   GPU(s)    │                      │
│         │         └─────────────┘                      │
└─────────│───────────────────────────────────────────────┘
          │
    ┌─────┴─────┐
    │ Cloudflare│
    │   (CDN)   │
    └───────────┘

# /etc/nginx/sites-available/llm
upstream vllm {
    server 127.0.0.1:8000;
    keepalive 32;
}

server {
    listen 443 ssl http2;
    server_name llm.yourdomain.com;

    ssl_certificate /etc/letsencrypt/live/llm.yourdomain.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/llm.yourdomain.com/privkey.pem;

    location / {
        proxy_pass http://vllm;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;

        # For streaming
        proxy_buffering off;
        proxy_cache off;
    }
}

# 70B model on 2x A100
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2

# docker-compose.yml for load balancing
version: '3.8'
services:
  nginx:
    image: nginx:alpine
    ports:
      - '8000:8000'
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
    depends_on:
      - vllm1
      - vllm2

  vllm1:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    # ... config

  vllm2:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    # ... config

# nginx.conf
upstream vllm_cluster {
    least_conn;
    server vllm1:8000;
    server vllm2:8000;
}

server {
    listen 8000;
    location / {
        proxy_pass http://vllm_cluster;
    }
}

# prometheus.yml
scrape_configs:
  - job_name: 'vllm'
    static_configs:
      - targets: ['localhost:8000']

{
  "panels": [
    {
      "title": "Requests per Second",
      "targets": [
        {
          "expr": "rate(vllm:num_requests_total[1m])"
        }
      ]
    },
    {
      "title": "GPU Memory Usage",
      "targets": [
        {
          "expr": "vllm:gpu_cache_usage_perc"
        }
      ]
    }
  ]
}

# Add to vLLM startup
--api-key your-secret-key

# Only allow internal access
ufw allow from 10.0.0.0/8 to any port 8000
ufw deny 8000

# Use HuggingFace token for gated models
export HUGGING_FACE_HUB_TOKEN=hf_...

# Reduce context length
--max-model-len 4096

# Use quantization
--quantization awq

# Reduce batch size
--max-num-seqs 32

# Enable Flash Attention
--enable-flashinfer

# Use continuous batching
--disable-log-requests  # Reduces overhead

# Check disk space
df -h

# Verify GPU access
nvidia-smi

# Check HuggingFace cache
ls ~/.cache/huggingface/hub/