Sovereign AI: Run Models 100% On Your Own Hardware

┌─────────────────────────────────────────────────────────────────┐
│                    SOVEREIGN AI STACK                            │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ APPLICATION LAYER                                           ││
│  │ Your App → API Gateway → Load Balancer                      ││
│  └─────────────────────────────────────────────────────────────┘│
│                           │                                      │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ INFERENCE LAYER                                             ││
│  │ vLLM / Ollama / TGI (OpenAI-compatible API)                ││
│  └─────────────────────────────────────────────────────────────┘│
│                           │                                      │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ MODEL LAYER                                                 ││
│  │ Llama 3.1 / Qwen 2.5 / Mistral / DeepSeek                  ││
│  └─────────────────────────────────────────────────────────────┘│
│                           │                                      │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ RUNTIME LAYER                                               ││
│  │ Docker + NVIDIA Container Toolkit                           ││
│  └─────────────────────────────────────────────────────────────┘│
│                           │                                      │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ OS LAYER                                                    ││
│  │ Ubuntu 22.04 LTS + NVIDIA Drivers                          ││
│  └─────────────────────────────────────────────────────────────┘│
│                           │                                      │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │ HARDWARE LAYER                                              ││
│  │ GPU(s) + CPU + RAM + NVMe Storage                          ││
│  └─────────────────────────────────────────────────────────────┘│
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

# Fresh Ubuntu 22.04 LTS install
# Update system
sudo apt update && sudo apt upgrade -y

# Install essential tools
sudo apt install -y build-essential git curl wget htop nvtop

# Add NVIDIA repository
sudo add-apt-repository ppa:graphics-drivers/ppa -y
sudo apt update

# Install latest driver (check nvidia.com for current version)
sudo apt install -y nvidia-driver-550

# Reboot
sudo reboot

# Verify installation
nvidia-smi

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 550.xx    Driver Version: 550.xx    CUDA Version: 12.4          |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |
|  0%   35C    P8    10W / 450W |      0MiB / 24576MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

# Install Docker
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER

# Install NVIDIA Container Toolkit
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
  sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt update
sudo apt install -y nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# Verify GPU access in Docker
docker run --rm --gpus all nvidia/cuda:12.4.0-base-ubuntu22.04 nvidia-smi

# Install huggingface-cli
pip install huggingface-hub

# Download models to a portable drive
huggingface-cli download meta-llama/Meta-Llama-3.1-8B-Instruct \
  --local-dir /mnt/portable/models/llama-3.1-8b

huggingface-cli download meta-llama/Meta-Llama-3.1-70B-Instruct \
  --local-dir /mnt/portable/models/llama-3.1-70b

# For Ollama (GGUF format)
wget https://huggingface.co/TheBloke/Llama-3.1-8B-Instruct-GGUF/resolve/main/llama-3.1-8b-instruct.Q4_K_M.gguf \
  -O /mnt/portable/models/llama-3.1-8b-q4.gguf

# Install Ollama
curl -fsSL https://ollama.com/install.sh | sh

# For air-gapped: copy the binary and models manually
# Create model from downloaded GGUF
cat > Modelfile << 'EOF'
FROM /path/to/llama-3.1-8b-q4.gguf
PARAMETER temperature 0.7
PARAMETER num_ctx 8192
SYSTEM You are a helpful AI assistant.
EOF

ollama create llama3.1 -f Modelfile

# Start server
ollama serve

# Test
curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [{"role": "user", "content": "Hello!"}],
  "stream": false
}'

# docker-compose.yml
version: '3.8'
services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - "8000:8000"
    volumes:
      - /path/to/models:/models
    command: >
      --model /models/llama-3.1-8b
      --tensor-parallel-size 1
      --max-model-len 8192
      --gpu-memory-utilization 0.9
    restart: unless-stopped

docker compose up -d

# Test (OpenAI-compatible)
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "/models/llama-3.1-8b",
    "messages": [{"role": "user", "content": "Hello!"}]
  }'

docker run --gpus all --shm-size 1g -p 8080:80 \
  -v /path/to/models:/data \
  ghcr.io/huggingface/text-generation-inference:latest \
  --model-id /data/llama-3.1-8b \
  --max-input-length 4096 \
  --max-total-tokens 8192

# vLLM with 2 GPUs for 70B model
docker run --gpus all -p 8000:8000 \
  -v /path/to/models:/models \
  vllm/vllm-openai:latest \
  --model /models/llama-3.1-70b \
  --tensor-parallel-size 2 \
  --max-model-len 4096

# docker-compose.yml
version: '3.8'
services:
  llama-8b:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    environment:
      - CUDA_VISIBLE_DEVICES=0
    ports:
      - '8001:8000'
    volumes:
      - /models:/models
    command: --model /models/llama-3.1-8b

  mistral-7b:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    environment:
      - CUDA_VISIBLE_DEVICES=1
    ports:
      - '8002:8000'
    volumes:
      - /models:/models
    command: --model /models/mistral-7b

# 1. Download all Docker images
docker pull vllm/vllm-openai:latest
docker pull nginx:alpine
docker pull prom/prometheus:latest
docker pull grafana/grafana:latest

# Save to tar files
docker save vllm/vllm-openai:latest > vllm.tar
docker save nginx:alpine > nginx.tar

# 2. Download models (see Part 2, Step 4)

# 3. Download all dependencies
# Create requirements.txt with all Python packages
pip download -r requirements.txt -d ./packages/

# 4. Transfer everything via USB/secure transfer

# 1. Load Docker images
docker load < vllm.tar
docker load < nginx.tar

# 2. Copy models to /opt/models

# 3. Start services
docker compose up -d

# 4. Verify (no external calls)
tcpdump -i any -c 100  # Should show only local traffic

# Block all outbound traffic except local
iptables -A OUTPUT -o lo -j ACCEPT
iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT
iptables -A OUTPUT -d 172.16.0.0/12 -j ACCEPT
iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT
iptables -A OUTPUT -j DROP

# docker-compose.prod.yml
version: '3.8'

services:
  # Reverse proxy with SSL
  nginx:
    image: nginx:alpine
    ports:
      - '443:443'
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
      - ./certs:/etc/nginx/certs
    depends_on:
      - vllm
    restart: unless-stopped

  # Main inference server
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - /opt/models:/models
    command: >
      --model /models/llama-3.1-8b
      --api-key ${API_KEY}
      --max-model-len 8192
      --gpu-memory-utilization 0.9
    restart: unless-stopped

  # Monitoring
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    ports:
      - '9090:9090'
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    volumes:
      - grafana_data:/var/lib/grafana
    ports:
      - '3000:3000'
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# nginx.conf
events {
    worker_connections 1024;
}

http {
    upstream vllm {
        server vllm:8000;
        keepalive 32;
    }

    server {
        listen 443 ssl http2;
        server_name ai.internal;

        ssl_certificate /etc/nginx/certs/server.crt;
        ssl_certificate_key /etc/nginx/certs/server.key;

        # API key validation
        location / {
            if ($http_authorization != "Bearer ${API_KEY}") {
                return 401;
            }

            proxy_pass http://vllm;
            proxy_http_version 1.1;
            proxy_set_header Connection "";

            # For streaming
            proxy_buffering off;
            proxy_cache off;
            chunked_transfer_encoding on;
        }
    }
}

Daily tokens: 10,000,000
API cost (GPT-4o @ $2.50/1M input, $10/1M output):
  - Assuming 50/50 split: ~$62.50/day = $1,875/month

Self-hosted cost:
  - Hardware: $8,000 (one-time)
  - Monthly: $400

Break-even: $8,000 / ($1,875 - $400) = 5.4 months
After break-even: Save $1,475/month

# Check for new model versions
# (On internet-connected machine)
huggingface-cli repo-info meta-llama/Meta-Llama-3.1-8B-Instruct

# Download new version
huggingface-cli download meta-llama/Meta-Llama-3.2-8B-Instruct \
  --local-dir /mnt/portable/models/llama-3.2-8b

# Transfer and deploy
# Update docker-compose.yml to point to new model
# Rolling restart
docker compose up -d --no-deps vllm

# Security updates (do regularly)
sudo apt update
sudo apt upgrade -y

# Driver updates (test first!)
# Download driver on separate machine
# Test in staging before production

# prometheus/alerts.yml
groups:
  - name: ai-server
    rules:
      - alert: GPUMemoryHigh
        expr: nvidia_gpu_memory_used_bytes / nvidia_gpu_memory_total_bytes > 0.95
        for: 5m
        labels:
          severity: warning

      - alert: GPUTemperatureHigh
        expr: nvidia_gpu_temperature_celsius > 85
        for: 2m
        labels:
          severity: critical

      - alert: InferenceLatencyHigh
        expr: histogram_quantile(0.99, rate(vllm_request_latency_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning

# Disable root SSH
sudo sed -i 's/PermitRootLogin yes/PermitRootLogin no/' /etc/ssh/sshd_config

# SSH key only
sudo sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config

# Firewall
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow from 10.0.0.0/8 to any port 22  # SSH from internal only
sudo ufw allow from 10.0.0.0/8 to any port 443 # API from internal only
sudo ufw enable

# Add to your application
import hashlib
import time

def validate_request(api_key: str, timestamp: str, signature: str, body: str) -> bool:
    """Validate HMAC signature for API requests."""
    # Check timestamp freshness (prevent replay)
    if abs(time.time() - float(timestamp)) > 300:  # 5 min window
        return False

    # Verify signature
    expected = hashlib.sha256(
        f"{api_key}{timestamp}{body}".encode()
    ).hexdigest()

    return signature == expected

# Log all inference requests
import logging
from datetime import datetime

audit_logger = logging.getLogger('audit')
audit_logger.setLevel(logging.INFO)
handler = logging.FileHandler('/var/log/ai-audit.log')
audit_logger.addHandler(handler)

def log_request(user_id: str, model: str, tokens_in: int, tokens_out: int):
    audit_logger.info(f"{datetime.utcnow().isoformat()} | {user_id} | {model} | {tokens_in} | {tokens_out}")