Distinguished Architect. CEO.
Securing Local Generative AI for Production.

# model_router.py — FastAPI gateway routing chat completions across
# private inference pools. Nothing leaves the boundary: every upstream is
# an internal vLLM endpoint or a private-link cloud pool.
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import httpx

UPSTREAMS = {
    "fast":  "http://vllm-l40s.internal:8000/v1",   # Llama-3-8B, low latency
    "smart": "http://vllm-h100.internal:8000/v1",   # Llama-3-70B, hard questions
    "batch": "https://gpu-pool.oci.internal/v1",    # OCI pool via private link
}
ROUTE_BY_BUDGET_MS = [(350, "fast"), (2500, "smart")]

app = FastAPI(title="model-router")

class ChatReq(BaseModel):
    messages: list[dict]
    model: str = "auto"
    latency_budget_ms: int = 2500
    max_tokens: int = 512

def pick(req: ChatReq) -> str:
    if req.model != "auto":
        if req.model not in UPSTREAMS:
            raise HTTPException(400, f"unknown model tier {req.model!r}")
        return UPSTREAMS[req.model]
    for budget, tier in ROUTE_BY_BUDGET_MS:
        if req.latency_budget_ms <= budget:
            return UPSTREAMS[tier]
    return UPSTREAMS["batch"]

@app.post("/v1/chat/completions")
async def route(req: ChatReq):
    upstream = pick(req)
    async with httpx.AsyncClient(timeout=30.0) as client:
        r = await client.post(f"{upstream}/chat/completions",
                              json=req.model_dump(exclude={"latency_budget_ms"}))
        r.raise_for_status()
        return r.json()

# llama3-qlora.yaml — Axolotl QLoRA fine-tune that fits one 24 GB GPU.
# Train on your own transcripts; weights never leave the building.
base_model: meta-llama/Meta-Llama-3-8B-Instruct
load_in_4bit: true
adapter: qlora

lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_modules: [q_proj, k_proj, v_proj, o_proj]

datasets:
  - path: data/support-transcripts.jsonl
    type: chat_template

sequence_len: 4096
sample_packing: true
micro_batch_size: 2
gradient_accumulation_steps: 8
num_epochs: 3
learning_rate: 2e-4
lr_scheduler: cosine
warmup_ratio: 0.05
bf16: auto
gradient_checkpointing: true

output_dir: ./outputs/llama3-support-lora

# data_plane.tf — zero-trust split: the control plane orchestrates,
# but customer data never leaves the customer\'s private VPC boundary.
module "customer_data_plane" {
  source   = "./modules/data-plane"
  for_each = var.customers

  vpc_cidr             = each.value.cidr
  allow_public_ingress = false            # no exceptions
  db_subnet_tier       = "isolated"       # no route to an IGW/NAT

  # Control plane reaches in via PrivateLink only — never the reverse.
  control_plane_endpoint_service = aws_vpc_endpoint_service.control.id
}

resource "aws_security_group_rule" "db_ingress" {
  for_each          = var.customers
  type              = "ingress"
  from_port         = 5432
  to_port           = 5432
  protocol          = "tcp"
  security_group_id = module.customer_data_plane[each.key].db_sg_id
  # Only the customer\'s own app tier — no 0.0.0.0/0, ever.
  source_security_group_id = module.customer_data_plane[each.key].app_sg_id
}

# vllm.service — production inference serving, OpenAI-compatible API.
# High throughput via continuous batching + tensor parallelism.
[Unit]
Description=vLLM inference server (Llama-3-70B-Instruct)
After=network-online.target

[Service]
User=vllm
Restart=always
RestartSec=5
Environment=HF_HUB_OFFLINE=1
ExecStart=/opt/vllm/bin/vllm serve meta-llama/Meta-Llama-3-70B-Instruct \
  --host 127.0.0.1 --port 8000 \
  --tensor-parallel-size 4 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.92 \
  --api-key-file /etc/vllm/api-keys

[Install]
WantedBy=multi-user.target