feat(llm-tui): tie GGUF models, inference presets, and mmproj together bi-directionally
This commit is contained in:
parent
0cf685a6d2
commit
ca74d9f321
3 changed files with 358 additions and 55 deletions
|
|
@ -33,7 +33,8 @@ uv run bin/llm-tui
|
||||||
```
|
```
|
||||||
|
|
||||||
Inside the TUI:
|
Inside the TUI:
|
||||||
- `F5` / **Launch**: Starts the server(s) in the background.
|
- **Tied Configurations**: GGUF model weights, vision multimodal projectors (`mmproj`), chat templates, and inference presets are automatically bound bi-directionally. Selecting an inference preset auto-selects its matching GGUF model and mmproj; selecting a GGUF model auto-selects its paired mmproj and appropriate preset tuning.
|
||||||
|
- `F5` / **Launch**: Starts the server(s) in the background with selected weights, projector, and parameters.
|
||||||
- `F8` / **Stop**: Stops server on the selected port.
|
- `F8` / **Stop**: Stops server on the selected port.
|
||||||
- `F9` / **Stop All**: Kills all running `llama-server` instances.
|
- `F9` / **Stop All**: Kills all running `llama-server` instances.
|
||||||
- `F2` / **Refresh**: Refreshes health status.
|
- `F2` / **Refresh**: Refreshes health status.
|
||||||
|
|
@ -50,6 +51,12 @@ Inside the TUI:
|
||||||
# SuperQwen 3.8 27B Solo (256K Context on port 11434)
|
# SuperQwen 3.8 27B Solo (256K Context on port 11434)
|
||||||
./bin/llm-runner superqwen
|
./bin/llm-runner superqwen
|
||||||
|
|
||||||
|
# Qwen 3.5 4B Solo (port 11434)
|
||||||
|
./bin/llm-runner qwen4b
|
||||||
|
|
||||||
|
# Custom Model Instance (with custom GGUF and vision projector)
|
||||||
|
MODEL=models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf MMPROJ=models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf ./bin/llm-runner custom
|
||||||
|
|
||||||
# Check status of running servers
|
# Check status of running servers
|
||||||
./bin/llm-runner status
|
./bin/llm-runner status
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -199,16 +199,18 @@ cmd_cluster() {
|
||||||
trap cleanup_cluster SIGINT SIGTERM EXIT
|
trap cleanup_cluster SIGINT SIGTERM EXIT
|
||||||
|
|
||||||
# SuperQwen 3.8 27B (Planner)
|
# SuperQwen 3.8 27B (Planner)
|
||||||
local planner_model="${PLANNER_MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}"
|
||||||
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
|
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
|
||||||
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
|
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
|
||||||
fi
|
fi
|
||||||
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
||||||
|
if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi
|
||||||
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
||||||
|
|
||||||
# Qwen 3.5 4B (Workers)
|
# Qwen 3.5 4B (Workers)
|
||||||
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
||||||
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
||||||
|
if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi
|
||||||
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
||||||
|
|
||||||
# 1. Planner on 11434
|
# 1. Planner on 11434
|
||||||
|
|
@ -244,6 +246,7 @@ cmd_muse() {
|
||||||
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
|
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
|
||||||
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}"
|
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}"
|
||||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}"
|
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}"
|
||||||
|
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}"
|
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}"
|
||||||
|
|
||||||
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \
|
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \
|
||||||
|
|
@ -264,6 +267,7 @@ cmd_superqwen() {
|
||||||
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
|
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
|
||||||
fi
|
fi
|
||||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
||||||
|
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
||||||
|
|
||||||
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \
|
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \
|
||||||
|
|
@ -281,6 +285,7 @@ cmd_qwen4b() {
|
||||||
log_info "Starting Qwen 3.5 4B Solo Server..."
|
log_info "Starting Qwen 3.5 4B Solo Server..."
|
||||||
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
||||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
||||||
|
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
||||||
|
|
||||||
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \
|
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \
|
||||||
|
|
@ -294,6 +299,29 @@ cmd_qwen4b() {
|
||||||
wait
|
wait
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cmd_custom() {
|
||||||
|
log_info "Starting Custom Server Instance..."
|
||||||
|
local model="${MODEL:-}"
|
||||||
|
if [ -z "${model}" ] || [ ! -f "${model}" ]; then
|
||||||
|
log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
local mmproj="${MMPROJ:-}"
|
||||||
|
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||||
|
local tmpl="${TEMPLATE:-}"
|
||||||
|
local name="custom_solo"
|
||||||
|
|
||||||
|
launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \
|
||||||
|
"$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
|
||||||
|
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
|
||||||
|
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log"
|
||||||
|
echo "Press Ctrl+C to stop."
|
||||||
|
wait
|
||||||
|
}
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Help / Usage
|
# Help / Usage
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -306,11 +334,15 @@ Commands:
|
||||||
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
|
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
|
||||||
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
|
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
|
||||||
qwen4b Launch Qwen 3.5 4B Solo (port 11434)
|
qwen4b Launch Qwen 3.5 4B Solo (port 11434)
|
||||||
|
custom Launch custom model instance (requires MODEL=/path/to/model.gguf)
|
||||||
status Show current server health and active ports
|
status Show current server health and active ports
|
||||||
stop [target] Stop servers (target: "all" or port number like 11434)
|
stop [target] Stop servers (target: "all" or port number like 11434)
|
||||||
help Display this help message
|
help Display this help message
|
||||||
|
|
||||||
Environment Variables:
|
Environment Variables:
|
||||||
|
MODEL Path to GGUF model file
|
||||||
|
MMPROJ Path to mmproj file (or "none" for text-only)
|
||||||
|
TEMPLATE Path to chat template file
|
||||||
HOST Bind address (default: 0.0.0.0)
|
HOST Bind address (default: 0.0.0.0)
|
||||||
PORT Main port override (default: 11434)
|
PORT Main port override (default: 11434)
|
||||||
CTX Context size override
|
CTX Context size override
|
||||||
|
|
@ -331,6 +363,7 @@ case "${ACTION}" in
|
||||||
muse) cmd_muse "$@" ;;
|
muse) cmd_muse "$@" ;;
|
||||||
superqwen) cmd_superqwen "$@" ;;
|
superqwen) cmd_superqwen "$@" ;;
|
||||||
qwen4b) cmd_qwen4b "$@" ;;
|
qwen4b) cmd_qwen4b "$@" ;;
|
||||||
|
custom) cmd_custom "$@" ;;
|
||||||
status) cmd_status "$@" ;;
|
status) cmd_status "$@" ;;
|
||||||
stop) cmd_stop "$@" ;;
|
stop) cmd_stop "$@" ;;
|
||||||
help|--help|-h) cmd_help ;;
|
help|--help|-h) cmd_help ;;
|
||||||
|
|
|
||||||
369
bin/llm-tui
369
bin/llm-tui
|
|
@ -14,6 +14,7 @@ import signal
|
||||||
import subprocess
|
import subprocess
|
||||||
import urllib.request
|
import urllib.request
|
||||||
import json
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
|
@ -45,10 +46,22 @@ RUNNER_BIN = SCRIPT_DIR / "bin" / "llm-runner"
|
||||||
|
|
||||||
TOTAL_USABLE_VRAM_GB = 30.0
|
TOTAL_USABLE_VRAM_GB = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ModelProfile:
|
||||||
|
name: str
|
||||||
|
path: Path
|
||||||
|
size_gb: float
|
||||||
|
directory: Path
|
||||||
|
paired_mmproj: Optional[Path]
|
||||||
|
paired_template: Optional[Path]
|
||||||
|
preset_key: str
|
||||||
|
|
||||||
|
|
||||||
PRESETS = {
|
PRESETS = {
|
||||||
"cluster": {
|
"cluster": {
|
||||||
"name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)",
|
"name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)",
|
||||||
"desc": "Planner on 11434, Worker 1 on 11435, Worker 2 on 11436. Optimized for Pi & Hermes agents.",
|
"desc": "Planner on 11434 (SuperQwen 27B + mmproj), Worker 1 on 11435, Worker 2 on 11436 (Qwen 4B + mmproj). Optimized for Pi & Hermes agents.",
|
||||||
"port": 11434,
|
"port": 11434,
|
||||||
"ctx": 65536,
|
"ctx": 65536,
|
||||||
"temp": 0.6,
|
"temp": 0.6,
|
||||||
|
|
@ -59,10 +72,12 @@ PRESETS = {
|
||||||
"flash_attn": True,
|
"flash_attn": True,
|
||||||
"kv_quant": "q8_0",
|
"kv_quant": "q8_0",
|
||||||
"command": "cluster",
|
"command": "cluster",
|
||||||
|
"target_model_match": "SuperQwen",
|
||||||
|
"is_cluster": True,
|
||||||
},
|
},
|
||||||
"muse": {
|
"muse": {
|
||||||
"name": "🔮 Muse-Glimmer 30B Solo (128K Context)",
|
"name": "🔮 Muse-Glimmer 30B Solo (128K Context)",
|
||||||
"desc": "Dedicated deep-reasoning agent with max 128K context on port 11434.",
|
"desc": "Dedicated deep-reasoning agent with max 128K context on port 11434. Auto-pairs Muse-Glimmer Q8_0 vision projector.",
|
||||||
"port": 11434,
|
"port": 11434,
|
||||||
"ctx": 131072,
|
"ctx": 131072,
|
||||||
"temp": 0.6,
|
"temp": 0.6,
|
||||||
|
|
@ -73,10 +88,12 @@ PRESETS = {
|
||||||
"flash_attn": True,
|
"flash_attn": True,
|
||||||
"kv_quant": "q8_0",
|
"kv_quant": "q8_0",
|
||||||
"command": "muse",
|
"command": "muse",
|
||||||
|
"target_model_match": "Muse-Glimmer",
|
||||||
|
"is_cluster": False,
|
||||||
},
|
},
|
||||||
"superqwen": {
|
"superqwen": {
|
||||||
"name": "🧠 SuperQwen 3.8 27B Solo (256K Context)",
|
"name": "🧠 SuperQwen 3.8 27B Solo (256K Context)",
|
||||||
"desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434.",
|
"desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434. Auto-pairs SuperQwen Q8_0 vision projector.",
|
||||||
"port": 11434,
|
"port": 11434,
|
||||||
"ctx": 262144,
|
"ctx": 262144,
|
||||||
"temp": 0.6,
|
"temp": 0.6,
|
||||||
|
|
@ -87,10 +104,12 @@ PRESETS = {
|
||||||
"flash_attn": True,
|
"flash_attn": True,
|
||||||
"kv_quant": "q8_0",
|
"kv_quant": "q8_0",
|
||||||
"command": "superqwen",
|
"command": "superqwen",
|
||||||
|
"target_model_match": "SuperQwen",
|
||||||
|
"is_cluster": False,
|
||||||
},
|
},
|
||||||
"qwen4b": {
|
"qwen4b": {
|
||||||
"name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)",
|
"name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)",
|
||||||
"desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint.",
|
"desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint. Auto-pairs BF16 vision projector.",
|
||||||
"port": 11434,
|
"port": 11434,
|
||||||
"ctx": 32768,
|
"ctx": 32768,
|
||||||
"temp": 0.7,
|
"temp": 0.7,
|
||||||
|
|
@ -101,6 +120,8 @@ PRESETS = {
|
||||||
"flash_attn": True,
|
"flash_attn": True,
|
||||||
"kv_quant": "q8_0",
|
"kv_quant": "q8_0",
|
||||||
"command": "qwen4b",
|
"command": "qwen4b",
|
||||||
|
"target_model_match": "Qwen3.5-4B",
|
||||||
|
"is_cluster": False,
|
||||||
},
|
},
|
||||||
"custom": {
|
"custom": {
|
||||||
"name": "🛠️ Custom Model & Parameters",
|
"name": "🛠️ Custom Model & Parameters",
|
||||||
|
|
@ -115,28 +136,80 @@ PRESETS = {
|
||||||
"flash_attn": True,
|
"flash_attn": True,
|
||||||
"kv_quant": "q8_0",
|
"kv_quant": "q8_0",
|
||||||
"command": "custom",
|
"command": "custom",
|
||||||
|
"target_model_match": None,
|
||||||
|
"is_cluster": False,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def find_gguf_models() -> list[tuple[str, str]]:
|
def scan_models() -> tuple[list[ModelProfile], list[tuple[str, str]], list[tuple[str, str]]]:
|
||||||
models: list[tuple[str, str]] = []
|
profiles: list[ModelProfile] = []
|
||||||
if MODELS_DIR.exists():
|
model_options: list[tuple[str, str]] = []
|
||||||
for path in sorted(MODELS_DIR.rglob("*.gguf")):
|
mmproj_options: list[tuple[str, str]] = [("None (Text Only / Vision Disabled)", "none")]
|
||||||
if "mmproj" not in path.name.lower():
|
|
||||||
rel = path.relative_to(MODELS_DIR)
|
|
||||||
models.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path)))
|
|
||||||
if not models:
|
|
||||||
models.append(("No GGUF models found in models/", ""))
|
|
||||||
return models
|
|
||||||
|
|
||||||
|
if not MODELS_DIR.exists():
|
||||||
|
return profiles, [("No GGUF models found in models/", "")], mmproj_options
|
||||||
|
|
||||||
def find_mmproj_files() -> list[tuple[str, str]]:
|
# Gather all mmproj projector files first
|
||||||
projectors = [("None (Text Only)", "")]
|
all_mmproj = sorted(MODELS_DIR.rglob("mmproj*.gguf"))
|
||||||
if MODELS_DIR.exists():
|
for p in all_mmproj:
|
||||||
for path in sorted(MODELS_DIR.rglob("mmproj*.gguf")):
|
try:
|
||||||
projectors.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path)))
|
sz = p.stat().st_size / 1e9
|
||||||
return projectors
|
except Exception:
|
||||||
|
sz = 0.0
|
||||||
|
mmproj_options.append((f"{p.name} ({sz:.1f} GB) [{p.parent.name}]", str(p)))
|
||||||
|
|
||||||
|
# Gather base model GGUF files
|
||||||
|
for p in sorted(MODELS_DIR.rglob("*.gguf")):
|
||||||
|
if "mmproj" in p.name.lower():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
sz = p.stat().st_size / 1e9
|
||||||
|
except Exception:
|
||||||
|
sz = 0.0
|
||||||
|
|
||||||
|
# Find paired mmproj in same folder
|
||||||
|
paired_mmproj = None
|
||||||
|
for m in all_mmproj:
|
||||||
|
if m.parent == p.parent:
|
||||||
|
paired_mmproj = m
|
||||||
|
break
|
||||||
|
|
||||||
|
# Find paired chat template in directory
|
||||||
|
tmpl = p.parent / "chat_template.jinja"
|
||||||
|
paired_template = tmpl if tmpl.exists() else None
|
||||||
|
|
||||||
|
# Determine preset mapping
|
||||||
|
p_lower = (p.name + " " + p.parent.name).lower()
|
||||||
|
if "muse" in p_lower:
|
||||||
|
preset_key = "muse"
|
||||||
|
tag = "Solo Reasoning (128K)"
|
||||||
|
elif "superqwen" in p_lower or "qwen3.8" in p_lower:
|
||||||
|
preset_key = "superqwen"
|
||||||
|
tag = "Solo 256K / Planner"
|
||||||
|
elif "qwen3.5" in p_lower or "4b" in p_lower:
|
||||||
|
preset_key = "qwen4b"
|
||||||
|
tag = "Fast Instruct / Worker"
|
||||||
|
else:
|
||||||
|
preset_key = "custom"
|
||||||
|
tag = "Custom"
|
||||||
|
|
||||||
|
prof = ModelProfile(
|
||||||
|
name=p.name,
|
||||||
|
path=p,
|
||||||
|
size_gb=sz,
|
||||||
|
directory=p.parent,
|
||||||
|
paired_mmproj=paired_mmproj,
|
||||||
|
paired_template=paired_template,
|
||||||
|
preset_key=preset_key,
|
||||||
|
)
|
||||||
|
profiles.append(prof)
|
||||||
|
model_options.append((f"{p.name} ({sz:.1f} GB) — {tag}", str(p)))
|
||||||
|
|
||||||
|
if not model_options:
|
||||||
|
model_options.append(("No GGUF models found in models/", ""))
|
||||||
|
|
||||||
|
return profiles, model_options, mmproj_options
|
||||||
|
|
||||||
|
|
||||||
def check_port_status(port: int) -> tuple[bool, str]:
|
def check_port_status(port: int) -> tuple[bool, str]:
|
||||||
|
|
@ -224,6 +297,12 @@ class LLMConfigApp(App):
|
||||||
text-style: bold;
|
text-style: bold;
|
||||||
border: solid #d08770;
|
border: solid #d08770;
|
||||||
}
|
}
|
||||||
|
#binding-indicator {
|
||||||
|
background: #1e2531;
|
||||||
|
border-left: thick #88c0d0;
|
||||||
|
padding: 1 2;
|
||||||
|
margin-bottom: 1;
|
||||||
|
}
|
||||||
#vram-meter {
|
#vram-meter {
|
||||||
background: #232834;
|
background: #232834;
|
||||||
border: solid #4c566a;
|
border: solid #4c566a;
|
||||||
|
|
@ -253,13 +332,35 @@ class LLMConfigApp(App):
|
||||||
]
|
]
|
||||||
|
|
||||||
selected_preset: str = "cluster"
|
selected_preset: str = "cluster"
|
||||||
models_list: list[tuple[str, str]] = []
|
profiles: list[ModelProfile] = []
|
||||||
mmproj_list: list[tuple[str, str]] = []
|
profiles_by_path: dict[str, ModelProfile] = {}
|
||||||
|
models_options: list[tuple[str, str]] = []
|
||||||
|
mmproj_options: list[tuple[str, str]] = []
|
||||||
|
_sync_lock: bool = False
|
||||||
|
_last_selected_model: str = ""
|
||||||
|
_last_selected_mmproj: str = ""
|
||||||
|
|
||||||
def compose(self) -> ComposeResult:
|
def compose(self) -> ComposeResult:
|
||||||
yield Header(show_clock=True)
|
yield Header(show_clock=True)
|
||||||
self.models_list = find_gguf_models()
|
self.profiles, self.models_options, self.mmproj_options = scan_models()
|
||||||
self.mmproj_list = find_mmproj_files()
|
self.profiles_by_path = {str(p.path): p for p in self.profiles}
|
||||||
|
|
||||||
|
# Choose default model and mmproj based on initial cluster preset
|
||||||
|
default_model = ""
|
||||||
|
default_mmproj = "none"
|
||||||
|
for p in self.profiles:
|
||||||
|
if "superqwen" in p.name.lower() or "qwen3.8" in p.name.lower():
|
||||||
|
default_model = str(p.path)
|
||||||
|
if p.paired_mmproj:
|
||||||
|
default_mmproj = str(p.paired_mmproj)
|
||||||
|
break
|
||||||
|
if not default_model and self.profiles:
|
||||||
|
default_model = str(self.profiles[0].path)
|
||||||
|
if self.profiles[0].paired_mmproj:
|
||||||
|
default_mmproj = str(self.profiles[0].paired_mmproj)
|
||||||
|
|
||||||
|
self._last_selected_model = default_model
|
||||||
|
self._last_selected_mmproj = default_mmproj
|
||||||
|
|
||||||
with TabbedContent():
|
with TabbedContent():
|
||||||
with TabPane("🚀 Model Launcher & Form", id="tab-launch"):
|
with TabPane("🚀 Model Launcher & Form", id="tab-launch"):
|
||||||
|
|
@ -280,23 +381,26 @@ class LLMConfigApp(App):
|
||||||
classes="text-muted",
|
classes="text-muted",
|
||||||
)
|
)
|
||||||
|
|
||||||
# 2. Parameters Form
|
# 2. Active Tied Binding Indicator Card
|
||||||
|
yield Static(id="binding-indicator")
|
||||||
|
|
||||||
|
# 3. Parameters Form (GGUF, MMProj, and Tuning)
|
||||||
with Container(classes="card", id="form-container"):
|
with Container(classes="card", id="form-container"):
|
||||||
yield Label("Hyperparameters & Hardware Tuning", classes="title")
|
yield Label("Hyperparameters & Hardware Tuning", classes="title")
|
||||||
|
|
||||||
with Horizontal(classes="field-row"):
|
with Horizontal(classes="field-row"):
|
||||||
yield Label("GGUF Model:", classes="field-label")
|
yield Label("GGUF Model:", classes="field-label")
|
||||||
yield Select(
|
yield Select(
|
||||||
options=[(lbl, val) for lbl, val in self.models_list],
|
options=[(lbl, val) for lbl, val in self.models_options],
|
||||||
value=self.models_list[0][1] if self.models_list else None,
|
value=default_model if default_model else None,
|
||||||
id="select-model",
|
id="select-model",
|
||||||
)
|
)
|
||||||
|
|
||||||
with Horizontal(classes="field-row"):
|
with Horizontal(classes="field-row"):
|
||||||
yield Label("Vision MMProj:", classes="field-label")
|
yield Label("Vision MMProj:", classes="field-label")
|
||||||
yield Select(
|
yield Select(
|
||||||
options=[(lbl, val) for lbl, val in self.mmproj_list],
|
options=[(lbl, val) for lbl, val in self.mmproj_options],
|
||||||
value=self.mmproj_list[0][1] if self.mmproj_list else None,
|
value=default_mmproj,
|
||||||
id="select-mmproj",
|
id="select-mmproj",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -339,7 +443,7 @@ class LLMConfigApp(App):
|
||||||
# VRAM Estimation Banner
|
# VRAM Estimation Banner
|
||||||
yield Static(id="vram-meter")
|
yield Static(id="vram-meter")
|
||||||
|
|
||||||
# 3. Execution Action Bar
|
# 4. Execution Action Bar
|
||||||
with Horizontal(classes="card"):
|
with Horizontal(classes="card"):
|
||||||
yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch")
|
yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch")
|
||||||
yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop")
|
yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop")
|
||||||
|
|
@ -355,15 +459,25 @@ class LLMConfigApp(App):
|
||||||
yield Footer()
|
yield Footer()
|
||||||
|
|
||||||
def on_mount(self) -> None:
|
def on_mount(self) -> None:
|
||||||
self.update_vram_estimation()
|
self.apply_preset(self.selected_preset)
|
||||||
self.action_refresh_status()
|
self.action_refresh_status()
|
||||||
self.set_interval(3.0, self.action_refresh_status)
|
self.set_interval(3.0, self.action_refresh_status)
|
||||||
self.set_interval(1.5, self.tail_logs)
|
self.set_interval(1.5, self.tail_logs)
|
||||||
|
|
||||||
def on_radio_set_changed(self, event: RadioSet.Changed) -> None:
|
def _set_active_radio(self, preset_key: str) -> None:
|
||||||
radio_id = event.pressed.id or ""
|
rs = self.query_one("#preset-radios", RadioSet)
|
||||||
preset_key = radio_id.replace("preset-", "")
|
target = self.query_one(f"#preset-{preset_key}", RadioButton)
|
||||||
if preset_key in PRESETS:
|
with rs.prevent(RadioButton.Changed):
|
||||||
|
if rs._pressed_button and rs._pressed_button != target:
|
||||||
|
rs._pressed_button.value = False
|
||||||
|
target.value = True
|
||||||
|
rs._pressed_button = target
|
||||||
|
|
||||||
|
def apply_preset(self, preset_key: str) -> None:
|
||||||
|
if preset_key not in PRESETS:
|
||||||
|
return
|
||||||
|
self._sync_lock = True
|
||||||
|
try:
|
||||||
self.selected_preset = preset_key
|
self.selected_preset = preset_key
|
||||||
preset = PRESETS[preset_key]
|
preset = PRESETS[preset_key]
|
||||||
self.query_one("#preset-description", Label).update(preset["desc"])
|
self.query_one("#preset-description", Label).update(preset["desc"])
|
||||||
|
|
@ -375,13 +489,134 @@ class LLMConfigApp(App):
|
||||||
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
|
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
|
||||||
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
|
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
|
||||||
self.query_one("#select-kv", Select).value = preset["kv_quant"]
|
self.query_one("#select-kv", Select).value = preset["kv_quant"]
|
||||||
|
|
||||||
|
# Automatically tie GGUF model and MMProj to preset
|
||||||
|
prof = self._find_profile_for_preset(preset_key)
|
||||||
|
if prof:
|
||||||
|
self._last_selected_model = str(prof.path)
|
||||||
|
self.query_one("#select-model", Select).value = str(prof.path)
|
||||||
|
target_mmproj = str(prof.paired_mmproj) if prof.paired_mmproj else "none"
|
||||||
|
self._last_selected_mmproj = target_mmproj
|
||||||
|
self.query_one("#select-mmproj", Select).value = target_mmproj
|
||||||
|
|
||||||
|
# Sync active RadioButton in RadioSet
|
||||||
|
self._set_active_radio(preset_key)
|
||||||
|
|
||||||
|
self.update_binding_indicator()
|
||||||
|
self.update_vram_estimation()
|
||||||
|
finally:
|
||||||
|
self._sync_lock = False
|
||||||
|
|
||||||
|
def _find_profile_for_preset(self, preset_key: str) -> Optional[ModelProfile]:
|
||||||
|
preset = PRESETS.get(preset_key)
|
||||||
|
if not preset or not preset.get("target_model_match"):
|
||||||
|
return None
|
||||||
|
target = preset["target_model_match"].lower()
|
||||||
|
for prof in self.profiles:
|
||||||
|
if target in prof.name.lower() or target in prof.directory.name.lower():
|
||||||
|
return prof
|
||||||
|
return None
|
||||||
|
|
||||||
|
def on_radio_set_changed(self, event: RadioSet.Changed) -> None:
|
||||||
|
if self._sync_lock:
|
||||||
|
return
|
||||||
|
radio_id = event.pressed.id or ""
|
||||||
|
preset_key = radio_id.replace("preset-", "")
|
||||||
|
self.apply_preset(preset_key)
|
||||||
|
|
||||||
|
def on_select_changed(self, event: Select.Changed) -> None:
|
||||||
|
if self._sync_lock:
|
||||||
|
return
|
||||||
|
|
||||||
|
if event.select.id == "select-model":
|
||||||
|
model_path_str = str(event.value or "")
|
||||||
|
if model_path_str == self._last_selected_model:
|
||||||
|
return
|
||||||
|
self._last_selected_model = model_path_str
|
||||||
|
|
||||||
|
self._sync_lock = True
|
||||||
|
try:
|
||||||
|
prof = self.profiles_by_path.get(model_path_str)
|
||||||
|
if prof:
|
||||||
|
# 1. Automatically tie matching MMProj
|
||||||
|
if prof.paired_mmproj:
|
||||||
|
self._last_selected_mmproj = str(prof.paired_mmproj)
|
||||||
|
self.query_one("#select-mmproj", Select).value = str(prof.paired_mmproj)
|
||||||
|
else:
|
||||||
|
self._last_selected_mmproj = "none"
|
||||||
|
self.query_one("#select-mmproj", Select).value = "none"
|
||||||
|
|
||||||
|
# 2. Automatically sync inference preset and sampling defaults
|
||||||
|
p_key = prof.preset_key
|
||||||
|
if self.selected_preset == "cluster" and p_key in ("superqwen", "qwen4b"):
|
||||||
|
# SuperQwen and Qwen 4B are the Planner and Workers of Cluster mode
|
||||||
|
pass
|
||||||
|
elif p_key != "custom" and p_key in PRESETS:
|
||||||
|
self.selected_preset = p_key
|
||||||
|
self._set_active_radio(p_key)
|
||||||
|
preset = PRESETS[p_key]
|
||||||
|
self.query_one("#preset-description", Label).update(preset["desc"])
|
||||||
|
self.query_one("#input-port", Input).value = str(preset["port"])
|
||||||
|
self.query_one("#input-ctx", Input).value = str(preset["ctx"])
|
||||||
|
self.query_one("#input-temp", Input).value = str(preset["temp"])
|
||||||
|
self.query_one("#input-top-p", Input).value = str(preset["top_p"])
|
||||||
|
self.query_one("#input-min-p", Input).value = str(preset["min_p"])
|
||||||
|
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
|
||||||
|
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
|
||||||
|
self.query_one("#select-kv", Select).value = preset["kv_quant"]
|
||||||
|
else:
|
||||||
|
self.selected_preset = "custom"
|
||||||
|
self._set_active_radio("custom")
|
||||||
|
self.query_one("#preset-description", Label).update(PRESETS["custom"]["desc"])
|
||||||
|
|
||||||
|
self.update_binding_indicator()
|
||||||
|
self.update_vram_estimation()
|
||||||
|
finally:
|
||||||
|
self._sync_lock = False
|
||||||
|
|
||||||
|
elif event.select.id == "select-mmproj":
|
||||||
|
mmproj_str = str(event.value or "")
|
||||||
|
if mmproj_str == self._last_selected_mmproj:
|
||||||
|
return
|
||||||
|
self._last_selected_mmproj = mmproj_str
|
||||||
|
self.update_binding_indicator()
|
||||||
|
self.update_vram_estimation()
|
||||||
|
|
||||||
|
elif event.select.id == "select-kv":
|
||||||
|
self.update_binding_indicator()
|
||||||
self.update_vram_estimation()
|
self.update_vram_estimation()
|
||||||
|
|
||||||
def on_input_changed(self, event: Input.Changed) -> None:
|
def on_input_changed(self, event: Input.Changed) -> None:
|
||||||
self.update_vram_estimation()
|
self.update_vram_estimation()
|
||||||
|
|
||||||
def on_select_changed(self, event: Select.Changed) -> None:
|
def update_binding_indicator(self) -> None:
|
||||||
self.update_vram_estimation()
|
preset_key = self.selected_preset
|
||||||
|
preset = PRESETS.get(preset_key, PRESETS["custom"])
|
||||||
|
model_val = str(self.query_one("#select-model", Select).value or "")
|
||||||
|
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
|
||||||
|
|
||||||
|
model_name = Path(model_val).name if (model_val and Path(model_val).exists()) else "None"
|
||||||
|
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
|
||||||
|
mmproj_text = f"[green]✔ {Path(mmproj_val).name} (Vision Enabled)[/]"
|
||||||
|
else:
|
||||||
|
mmproj_text = "[yellow]○ None (Text Only / Vision Disabled)[/]"
|
||||||
|
|
||||||
|
if preset.get("is_cluster"):
|
||||||
|
text = (
|
||||||
|
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
|
||||||
|
f" • [bold]Planner (Port 11434):[/] SuperQwen 3.8 27B + mmproj-SuperQwen (64K Ctx)\n"
|
||||||
|
f" • [bold]Workers (Ports 11435/11436):[/] 2x Qwen 3.5 4B + mmproj-BF16 (16K Ctx)\n"
|
||||||
|
f" • [bold green]● GGUF weights, projectors, and inference presets are coordinated across all 3 nodes[/]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
text = (
|
||||||
|
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
|
||||||
|
f" • [bold]GGUF Model:[/] [bold magenta]{model_name}[/]\n"
|
||||||
|
f" • [bold]Vision Projector:[/] {mmproj_text}\n"
|
||||||
|
f" • [bold]Chat Template:[/] Jinja template auto-paired from model directory"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.query_one("#binding-indicator", Static).update(text)
|
||||||
|
|
||||||
def update_vram_estimation(self) -> None:
|
def update_vram_estimation(self) -> None:
|
||||||
try:
|
try:
|
||||||
|
|
@ -390,25 +625,31 @@ class LLMConfigApp(App):
|
||||||
ctx = 65536
|
ctx = 65536
|
||||||
|
|
||||||
preset = self.selected_preset
|
preset = self.selected_preset
|
||||||
|
kv_choice = str(self.query_one("#select-kv", Select).value or "q8_0")
|
||||||
|
kv_mult = 1.0 if kv_choice == "q8_0" else (2.0 if kv_choice == "f16" else 0.5)
|
||||||
|
|
||||||
if preset == "cluster":
|
if preset == "cluster":
|
||||||
weights_gb = 25.9
|
weights_gb = 25.9
|
||||||
kv_gb = 3.2
|
kv_gb = 3.2 * kv_mult
|
||||||
elif preset == "muse":
|
|
||||||
weights_gb = 17.5
|
|
||||||
kv_gb = (ctx / 131072.0) * 8.5
|
|
||||||
elif preset == "superqwen":
|
|
||||||
weights_gb = 17.5
|
|
||||||
kv_gb = (ctx / 262144.0) * 10.5
|
|
||||||
elif preset == "qwen4b":
|
|
||||||
weights_gb = 3.9
|
|
||||||
kv_gb = (ctx / 32768.0) * 1.5
|
|
||||||
else:
|
else:
|
||||||
model_val = str(self.query_one("#select-model", Select).value or "")
|
model_val = str(self.query_one("#select-model", Select).value or "")
|
||||||
if model_val and Path(model_val).exists():
|
if model_val and Path(model_val).exists():
|
||||||
weights_gb = Path(model_val).stat().st_size / 1e9
|
model_gb = Path(model_val).stat().st_size / 1e9
|
||||||
else:
|
else:
|
||||||
weights_gb = 15.0
|
model_gb = 16.0
|
||||||
kv_gb = (ctx / 65536.0) * 4.0
|
|
||||||
|
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
|
||||||
|
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
|
||||||
|
mmproj_gb = Path(mmproj_val).stat().st_size / 1e9
|
||||||
|
else:
|
||||||
|
mmproj_gb = 0.0
|
||||||
|
|
||||||
|
weights_gb = model_gb + mmproj_gb
|
||||||
|
|
||||||
|
if weights_gb > 10.0:
|
||||||
|
kv_gb = (ctx / 131072.0) * 8.5 * kv_mult
|
||||||
|
else:
|
||||||
|
kv_gb = (ctx / 32768.0) * 1.5 * kv_mult
|
||||||
|
|
||||||
total_est = weights_gb + kv_gb
|
total_est = weights_gb + kv_gb
|
||||||
remaining = TOTAL_USABLE_VRAM_GB - total_est
|
remaining = TOTAL_USABLE_VRAM_GB - total_est
|
||||||
|
|
@ -454,7 +695,9 @@ class LLMConfigApp(App):
|
||||||
def action_launch(self) -> None:
|
def action_launch(self) -> None:
|
||||||
preset = self.selected_preset
|
preset = self.selected_preset
|
||||||
log_widget = self.query_one("#log-stream", Log)
|
log_widget = self.query_one("#log-stream", Log)
|
||||||
log_widget.write(f"\n[bold cyan]>>> Launching preset: {preset}...[/]\n")
|
|
||||||
|
model_val = str(self.query_one("#select-model", Select).value or "")
|
||||||
|
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
|
||||||
|
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536"
|
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536"
|
||||||
|
|
@ -467,9 +710,29 @@ class LLMConfigApp(App):
|
||||||
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
|
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
|
||||||
env["CTV"] = env["CTK"]
|
env["CTV"] = env["CTK"]
|
||||||
|
|
||||||
cmd = [str(RUNNER_BIN), preset]
|
if model_val and Path(model_val).exists():
|
||||||
|
env["MODEL"] = model_val
|
||||||
|
tmpl = Path(model_val).parent / "chat_template.jinja"
|
||||||
|
if tmpl.exists():
|
||||||
|
env["TEMPLATE"] = str(tmpl)
|
||||||
|
|
||||||
|
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
|
||||||
|
env["MMPROJ"] = mmproj_val
|
||||||
|
else:
|
||||||
|
env["MMPROJ"] = "none"
|
||||||
|
|
||||||
|
runner_cmd = PRESETS.get(preset, {}).get("command", preset)
|
||||||
|
cmd = [str(RUNNER_BIN), runner_cmd]
|
||||||
|
|
||||||
|
log_widget.write(
|
||||||
|
f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n"
|
||||||
|
f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n"
|
||||||
|
f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n"
|
||||||
|
f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n"
|
||||||
|
)
|
||||||
|
|
||||||
subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True)
|
subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True)
|
||||||
self.notify(f"Launching {preset} server in background. Checking ports...", severity="information")
|
self.notify(f"Launching {preset} server. Check Status & Live Logs tab.", severity="information")
|
||||||
self.action_refresh_status()
|
self.action_refresh_status()
|
||||||
|
|
||||||
def action_stop(self) -> None:
|
def action_stop(self) -> None:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue