Oobabooga: The Only LLM UI You Actually Need

# Clone repository
git clone https://github.com/oobabooga/text-generation-webui
cd text-generation-webui

# Windows: One-click installer
# Download from https://github.com/oobabooga/install-windows-main

# macOS/Linux: Manual install
# Install CUDA toolkit first (if using GPU)
# Check: https://developer.nvidia.com/cuda-downloads

# Create conda environment
conda env create -f environment.yml
conda activate textgen

# Or with pip (if not using conda)
pip install -r requirements.txt

# Optional: Install specific loaders
pip install bitsandbytes  # For 4-bit loading
pip install exllamav2  # For ExLlamaV2 (fastest)
pip install llama-cpp-python  # For GGUF

# Download GGUF model
mkdir models
cd models

# Using HuggingFace CLI
huggingface-cli download TheBloke/Mistral-7B-Instruct-v0.2-GGUF mistral-7b-instruct-v0.2.Q4_K_M.gguf --local-dir .

# Or direct download
wget https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf

# Start webui with GGUF
cd text-generation-webui
python server.py \
  --loader llama.cpp \
  --model ../models/mistral-7b-instruct-v0.2.Q4_K_M.gguf \
  --n-gpu-layers -1  # All layers on GPU (or 0 for CPU)

# Install ExLlamaV2
pip install exllamav2

# Download ExLlamaV2-converted model
huggingface-cli download turboderp/Mistral-7B-instruct-exl2 \
  Mistral-7B-instruct-exl2-4.0bpw --local-dir .

# Start with ExLlamaV2 (fastest option)
python server.py \
  --loader exllamav2 \
  --model ../models/Mistral-7B-instruct-exl2-4.0bpw \
  --gpu-memory 10  # GB VRAM to use (auto-detect if omitted)

# ExLlamaV2 features:
# - 2-3x faster than llama.cpp
# - Lowest VRAM usage
# - Only works with NVIDIA GPUs
# - Requires ExLlamaV2-converted models

# Load any HuggingFace model in 4-bit
# No conversion needed, loads directly

python server.py \
  --model TheBloke/Mistral-7B-Instruct-v0.2 \
  --load-in-4bit \
  --compute_dtype float16 \
  --quant_type nf4  # or fp4

# Quantization types:
# nf4: NormalFloat 4 (recommended, best quality)
# fp4: Float 4 (slightly faster, lower quality)

# For 8-bit (better quality, more VRAM)
python server.py \
  --model TheBloke/Mistral-7B-Instruct-v0.2 \
  --load-in-8bit

# Install OpenAI API extension
cd extensions
git clone https://github.com/oobabooga/text-generation-webui-extensions
cd text-generation-webui-extensions/openai

# Enable extension
cd ../..
python server.py --extensions openai

# Configure in web UI: Settings → OpenAI
# Or via command line:
python server.py \
  --extensions openai \
  --openai-api-port 5000 \
  --openai-api-key your-api-key

# Drop-in replacement for OpenAI API
curl http://localhost:5000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer your-api-key" \
  -d '{
    "model": "local-model",
    "messages": [{"role": "user", "content": "Hello!"}]
  }'

# Works with existing OpenAI clients
# Just change base_url to http://localhost:5000/v1

# Enable training extension
python server.py --extensions training

# Via web UI: Extensions → Training
# Upload dataset (JSONL format)
# Configure hyperparameters:
# - LoRA rank: 8-64 (higher = more capacity)
# - Learning rate: 1e-4 to 5e-4
# - Batch size: 1-4 (depending on VRAM)
# - Epochs: 1-3 (more = overfitting)

# Training dataset format (JSONL):
{"text": "User: What is X?\nAssistant: X is..."}
{"text": "User: How do I Y?\nAssistant: To Y,..."}

# Fine-tune on custom data
python train.py \
  --model mistral-7b-instruct \
  --data training.jsonl \
  --lora-rank 32 \
  --lora-alpha 64 \
  --learning-rate 2e-4 \
  --batch-size 2 \
  --epochs 2 \
  --output-dir ./loras \
  --output-name my-lora

# Load with LoRA
python server.py \
  --model mistral-7b-instruct \
  --lora_dir ./loras \
  --lora my-lora

# Enable notebook mode (like Jupyter)
python server.py --extensions notebook

# Access at http://localhost:7860/?notebook=true

# Features:
# - Code cells with markdown support
# - Variable sharing between cells
# - Export to Python script
# - Model state inspection

# Example notebook workflow:
# Cell 1: Load model
model = load_model("mistral-7b-instruct")

# Cell 2: Test generation
output = generate("Hello, world!", max_tokens=100)
print(output)

# Cell 3: Benchmark
import time
start = time.time()
generate("Long prompt...", max_tokens=1000)
print(f"Time: {time.time() - start:.2f}s")

# Run as background service
nohup python server.py \
  --loader exllamav2 \
  --model mistral-7b-exl2 \
  --api \
  --listen \
  --port 5000 \
  > logs/textgen.log 2>&1 &

# Systemd service
sudo tee /etc/systemd/system/textgen.service <