feat(llm-tui): tie GGUF models, inference presets, and mmproj together bi-directionally

This commit is contained in:
Noel Bautista 2026-10-21 12:52:15 -07:00
parent 0cf685a6d2
commit ca74d9f321
No known key found for this signature in database
3 changed files with 358 additions and 55 deletions

View file

@ -33,7 +33,8 @@ uv run bin/llm-tui
``` ```
Inside the TUI: Inside the TUI:
- `F5` / **Launch**: Starts the server(s) in the background. - **Tied Configurations**: GGUF model weights, vision multimodal projectors (`mmproj`), chat templates, and inference presets are automatically bound bi-directionally. Selecting an inference preset auto-selects its matching GGUF model and mmproj; selecting a GGUF model auto-selects its paired mmproj and appropriate preset tuning.
- `F5` / **Launch**: Starts the server(s) in the background with selected weights, projector, and parameters.
- `F8` / **Stop**: Stops server on the selected port. - `F8` / **Stop**: Stops server on the selected port.
- `F9` / **Stop All**: Kills all running `llama-server` instances. - `F9` / **Stop All**: Kills all running `llama-server` instances.
- `F2` / **Refresh**: Refreshes health status. - `F2` / **Refresh**: Refreshes health status.
@ -50,6 +51,12 @@ Inside the TUI:
# SuperQwen 3.8 27B Solo (256K Context on port 11434) # SuperQwen 3.8 27B Solo (256K Context on port 11434)
./bin/llm-runner superqwen ./bin/llm-runner superqwen
# Qwen 3.5 4B Solo (port 11434)
./bin/llm-runner qwen4b
# Custom Model Instance (with custom GGUF and vision projector)
MODEL=models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf MMPROJ=models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf ./bin/llm-runner custom
# Check status of running servers # Check status of running servers
./bin/llm-runner status ./bin/llm-runner status

View file

@ -199,16 +199,18 @@ cmd_cluster() {
trap cleanup_cluster SIGINT SIGTERM EXIT trap cleanup_cluster SIGINT SIGTERM EXIT
# SuperQwen 3.8 27B (Planner) # SuperQwen 3.8 27B (Planner)
local planner_model="${PLANNER_MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}" local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}"
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
fi fi
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
# Qwen 3.5 4B (Workers) # Qwen 3.5 4B (Workers)
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
# 1. Planner on 11434 # 1. Planner on 11434
@ -244,6 +246,7 @@ cmd_muse() {
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..." log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}" local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}" local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}"
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \
@ -264,6 +267,7 @@ cmd_superqwen() {
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
fi fi
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \
@ -281,6 +285,7 @@ cmd_qwen4b() {
log_info "Starting Qwen 3.5 4B Solo Server..." log_info "Starting Qwen 3.5 4B Solo Server..."
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \
@ -294,6 +299,29 @@ cmd_qwen4b() {
wait wait
} }
cmd_custom() {
log_info "Starting Custom Server Instance..."
local model="${MODEL:-}"
if [ -z "${model}" ] || [ ! -f "${model}" ]; then
log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf"
exit 1
fi
local mmproj="${MMPROJ:-}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-}"
local name="custom_solo"
launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \
"$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}"
echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log"
echo "Press Ctrl+C to stop."
wait
}
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Help / Usage # Help / Usage
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -306,11 +334,15 @@ Commands:
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434) muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434) superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
qwen4b Launch Qwen 3.5 4B Solo (port 11434) qwen4b Launch Qwen 3.5 4B Solo (port 11434)
custom Launch custom model instance (requires MODEL=/path/to/model.gguf)
status Show current server health and active ports status Show current server health and active ports
stop [target] Stop servers (target: "all" or port number like 11434) stop [target] Stop servers (target: "all" or port number like 11434)
help Display this help message help Display this help message
Environment Variables: Environment Variables:
MODEL Path to GGUF model file
MMPROJ Path to mmproj file (or "none" for text-only)
TEMPLATE Path to chat template file
HOST Bind address (default: 0.0.0.0) HOST Bind address (default: 0.0.0.0)
PORT Main port override (default: 11434) PORT Main port override (default: 11434)
CTX Context size override CTX Context size override
@ -331,6 +363,7 @@ case "${ACTION}" in
muse) cmd_muse "$@" ;; muse) cmd_muse "$@" ;;
superqwen) cmd_superqwen "$@" ;; superqwen) cmd_superqwen "$@" ;;
qwen4b) cmd_qwen4b "$@" ;; qwen4b) cmd_qwen4b "$@" ;;
custom) cmd_custom "$@" ;;
status) cmd_status "$@" ;; status) cmd_status "$@" ;;
stop) cmd_stop "$@" ;; stop) cmd_stop "$@" ;;
help|--help|-h) cmd_help ;; help|--help|-h) cmd_help ;;

View file

@ -14,6 +14,7 @@ import signal
import subprocess import subprocess
import urllib.request import urllib.request
import json import json
from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
@ -45,10 +46,22 @@ RUNNER_BIN = SCRIPT_DIR / "bin" / "llm-runner"
TOTAL_USABLE_VRAM_GB = 30.0 TOTAL_USABLE_VRAM_GB = 30.0
@dataclass
class ModelProfile:
name: str
path: Path
size_gb: float
directory: Path
paired_mmproj: Optional[Path]
paired_template: Optional[Path]
preset_key: str
PRESETS = { PRESETS = {
"cluster": { "cluster": {
"name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)", "name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)",
"desc": "Planner on 11434, Worker 1 on 11435, Worker 2 on 11436. Optimized for Pi & Hermes agents.", "desc": "Planner on 11434 (SuperQwen 27B + mmproj), Worker 1 on 11435, Worker 2 on 11436 (Qwen 4B + mmproj). Optimized for Pi & Hermes agents.",
"port": 11434, "port": 11434,
"ctx": 65536, "ctx": 65536,
"temp": 0.6, "temp": 0.6,
@ -59,10 +72,12 @@ PRESETS = {
"flash_attn": True, "flash_attn": True,
"kv_quant": "q8_0", "kv_quant": "q8_0",
"command": "cluster", "command": "cluster",
"target_model_match": "SuperQwen",
"is_cluster": True,
}, },
"muse": { "muse": {
"name": "🔮 Muse-Glimmer 30B Solo (128K Context)", "name": "🔮 Muse-Glimmer 30B Solo (128K Context)",
"desc": "Dedicated deep-reasoning agent with max 128K context on port 11434.", "desc": "Dedicated deep-reasoning agent with max 128K context on port 11434. Auto-pairs Muse-Glimmer Q8_0 vision projector.",
"port": 11434, "port": 11434,
"ctx": 131072, "ctx": 131072,
"temp": 0.6, "temp": 0.6,
@ -73,10 +88,12 @@ PRESETS = {
"flash_attn": True, "flash_attn": True,
"kv_quant": "q8_0", "kv_quant": "q8_0",
"command": "muse", "command": "muse",
"target_model_match": "Muse-Glimmer",
"is_cluster": False,
}, },
"superqwen": { "superqwen": {
"name": "🧠 SuperQwen 3.8 27B Solo (256K Context)", "name": "🧠 SuperQwen 3.8 27B Solo (256K Context)",
"desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434.", "desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434. Auto-pairs SuperQwen Q8_0 vision projector.",
"port": 11434, "port": 11434,
"ctx": 262144, "ctx": 262144,
"temp": 0.6, "temp": 0.6,
@ -87,10 +104,12 @@ PRESETS = {
"flash_attn": True, "flash_attn": True,
"kv_quant": "q8_0", "kv_quant": "q8_0",
"command": "superqwen", "command": "superqwen",
"target_model_match": "SuperQwen",
"is_cluster": False,
}, },
"qwen4b": { "qwen4b": {
"name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)", "name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)",
"desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint.", "desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint. Auto-pairs BF16 vision projector.",
"port": 11434, "port": 11434,
"ctx": 32768, "ctx": 32768,
"temp": 0.7, "temp": 0.7,
@ -101,6 +120,8 @@ PRESETS = {
"flash_attn": True, "flash_attn": True,
"kv_quant": "q8_0", "kv_quant": "q8_0",
"command": "qwen4b", "command": "qwen4b",
"target_model_match": "Qwen3.5-4B",
"is_cluster": False,
}, },
"custom": { "custom": {
"name": "🛠️ Custom Model & Parameters", "name": "🛠️ Custom Model & Parameters",
@ -115,28 +136,80 @@ PRESETS = {
"flash_attn": True, "flash_attn": True,
"kv_quant": "q8_0", "kv_quant": "q8_0",
"command": "custom", "command": "custom",
"target_model_match": None,
"is_cluster": False,
}, },
} }
def find_gguf_models() -> list[tuple[str, str]]: def scan_models() -> tuple[list[ModelProfile], list[tuple[str, str]], list[tuple[str, str]]]:
models: list[tuple[str, str]] = [] profiles: list[ModelProfile] = []
if MODELS_DIR.exists(): model_options: list[tuple[str, str]] = []
for path in sorted(MODELS_DIR.rglob("*.gguf")): mmproj_options: list[tuple[str, str]] = [("None (Text Only / Vision Disabled)", "none")]
if "mmproj" not in path.name.lower():
rel = path.relative_to(MODELS_DIR)
models.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path)))
if not models:
models.append(("No GGUF models found in models/", ""))
return models
if not MODELS_DIR.exists():
return profiles, [("No GGUF models found in models/", "")], mmproj_options
def find_mmproj_files() -> list[tuple[str, str]]: # Gather all mmproj projector files first
projectors = [("None (Text Only)", "")] all_mmproj = sorted(MODELS_DIR.rglob("mmproj*.gguf"))
if MODELS_DIR.exists(): for p in all_mmproj:
for path in sorted(MODELS_DIR.rglob("mmproj*.gguf")): try:
projectors.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path))) sz = p.stat().st_size / 1e9
return projectors except Exception:
sz = 0.0
mmproj_options.append((f"{p.name} ({sz:.1f} GB) [{p.parent.name}]", str(p)))
# Gather base model GGUF files
for p in sorted(MODELS_DIR.rglob("*.gguf")):
if "mmproj" in p.name.lower():
continue
try:
sz = p.stat().st_size / 1e9
except Exception:
sz = 0.0
# Find paired mmproj in same folder
paired_mmproj = None
for m in all_mmproj:
if m.parent == p.parent:
paired_mmproj = m
break
# Find paired chat template in directory
tmpl = p.parent / "chat_template.jinja"
paired_template = tmpl if tmpl.exists() else None
# Determine preset mapping
p_lower = (p.name + " " + p.parent.name).lower()
if "muse" in p_lower:
preset_key = "muse"
tag = "Solo Reasoning (128K)"
elif "superqwen" in p_lower or "qwen3.8" in p_lower:
preset_key = "superqwen"
tag = "Solo 256K / Planner"
elif "qwen3.5" in p_lower or "4b" in p_lower:
preset_key = "qwen4b"
tag = "Fast Instruct / Worker"
else:
preset_key = "custom"
tag = "Custom"
prof = ModelProfile(
name=p.name,
path=p,
size_gb=sz,
directory=p.parent,
paired_mmproj=paired_mmproj,
paired_template=paired_template,
preset_key=preset_key,
)
profiles.append(prof)
model_options.append((f"{p.name} ({sz:.1f} GB) — {tag}", str(p)))
if not model_options:
model_options.append(("No GGUF models found in models/", ""))
return profiles, model_options, mmproj_options
def check_port_status(port: int) -> tuple[bool, str]: def check_port_status(port: int) -> tuple[bool, str]:
@ -224,6 +297,12 @@ class LLMConfigApp(App):
text-style: bold; text-style: bold;
border: solid #d08770; border: solid #d08770;
} }
#binding-indicator {
background: #1e2531;
border-left: thick #88c0d0;
padding: 1 2;
margin-bottom: 1;
}
#vram-meter { #vram-meter {
background: #232834; background: #232834;
border: solid #4c566a; border: solid #4c566a;
@ -253,13 +332,35 @@ class LLMConfigApp(App):
] ]
selected_preset: str = "cluster" selected_preset: str = "cluster"
models_list: list[tuple[str, str]] = [] profiles: list[ModelProfile] = []
mmproj_list: list[tuple[str, str]] = [] profiles_by_path: dict[str, ModelProfile] = {}
models_options: list[tuple[str, str]] = []
mmproj_options: list[tuple[str, str]] = []
_sync_lock: bool = False
_last_selected_model: str = ""
_last_selected_mmproj: str = ""
def compose(self) -> ComposeResult: def compose(self) -> ComposeResult:
yield Header(show_clock=True) yield Header(show_clock=True)
self.models_list = find_gguf_models() self.profiles, self.models_options, self.mmproj_options = scan_models()
self.mmproj_list = find_mmproj_files() self.profiles_by_path = {str(p.path): p for p in self.profiles}
# Choose default model and mmproj based on initial cluster preset
default_model = ""
default_mmproj = "none"
for p in self.profiles:
if "superqwen" in p.name.lower() or "qwen3.8" in p.name.lower():
default_model = str(p.path)
if p.paired_mmproj:
default_mmproj = str(p.paired_mmproj)
break
if not default_model and self.profiles:
default_model = str(self.profiles[0].path)
if self.profiles[0].paired_mmproj:
default_mmproj = str(self.profiles[0].paired_mmproj)
self._last_selected_model = default_model
self._last_selected_mmproj = default_mmproj
with TabbedContent(): with TabbedContent():
with TabPane("🚀 Model Launcher & Form", id="tab-launch"): with TabPane("🚀 Model Launcher & Form", id="tab-launch"):
@ -280,23 +381,26 @@ class LLMConfigApp(App):
classes="text-muted", classes="text-muted",
) )
# 2. Parameters Form # 2. Active Tied Binding Indicator Card
yield Static(id="binding-indicator")
# 3. Parameters Form (GGUF, MMProj, and Tuning)
with Container(classes="card", id="form-container"): with Container(classes="card", id="form-container"):
yield Label("Hyperparameters & Hardware Tuning", classes="title") yield Label("Hyperparameters & Hardware Tuning", classes="title")
with Horizontal(classes="field-row"): with Horizontal(classes="field-row"):
yield Label("GGUF Model:", classes="field-label") yield Label("GGUF Model:", classes="field-label")
yield Select( yield Select(
options=[(lbl, val) for lbl, val in self.models_list], options=[(lbl, val) for lbl, val in self.models_options],
value=self.models_list[0][1] if self.models_list else None, value=default_model if default_model else None,
id="select-model", id="select-model",
) )
with Horizontal(classes="field-row"): with Horizontal(classes="field-row"):
yield Label("Vision MMProj:", classes="field-label") yield Label("Vision MMProj:", classes="field-label")
yield Select( yield Select(
options=[(lbl, val) for lbl, val in self.mmproj_list], options=[(lbl, val) for lbl, val in self.mmproj_options],
value=self.mmproj_list[0][1] if self.mmproj_list else None, value=default_mmproj,
id="select-mmproj", id="select-mmproj",
) )
@ -339,7 +443,7 @@ class LLMConfigApp(App):
# VRAM Estimation Banner # VRAM Estimation Banner
yield Static(id="vram-meter") yield Static(id="vram-meter")
# 3. Execution Action Bar # 4. Execution Action Bar
with Horizontal(classes="card"): with Horizontal(classes="card"):
yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch") yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch")
yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop") yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop")
@ -355,15 +459,25 @@ class LLMConfigApp(App):
yield Footer() yield Footer()
def on_mount(self) -> None: def on_mount(self) -> None:
self.update_vram_estimation() self.apply_preset(self.selected_preset)
self.action_refresh_status() self.action_refresh_status()
self.set_interval(3.0, self.action_refresh_status) self.set_interval(3.0, self.action_refresh_status)
self.set_interval(1.5, self.tail_logs) self.set_interval(1.5, self.tail_logs)
def on_radio_set_changed(self, event: RadioSet.Changed) -> None: def _set_active_radio(self, preset_key: str) -> None:
radio_id = event.pressed.id or "" rs = self.query_one("#preset-radios", RadioSet)
preset_key = radio_id.replace("preset-", "") target = self.query_one(f"#preset-{preset_key}", RadioButton)
if preset_key in PRESETS: with rs.prevent(RadioButton.Changed):
if rs._pressed_button and rs._pressed_button != target:
rs._pressed_button.value = False
target.value = True
rs._pressed_button = target
def apply_preset(self, preset_key: str) -> None:
if preset_key not in PRESETS:
return
self._sync_lock = True
try:
self.selected_preset = preset_key self.selected_preset = preset_key
preset = PRESETS[preset_key] preset = PRESETS[preset_key]
self.query_one("#preset-description", Label).update(preset["desc"]) self.query_one("#preset-description", Label).update(preset["desc"])
@ -375,13 +489,134 @@ class LLMConfigApp(App):
self.query_one("#input-ngl", Input).value = str(preset["ngl"]) self.query_one("#input-ngl", Input).value = str(preset["ngl"])
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"] self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
self.query_one("#select-kv", Select).value = preset["kv_quant"] self.query_one("#select-kv", Select).value = preset["kv_quant"]
# Automatically tie GGUF model and MMProj to preset
prof = self._find_profile_for_preset(preset_key)
if prof:
self._last_selected_model = str(prof.path)
self.query_one("#select-model", Select).value = str(prof.path)
target_mmproj = str(prof.paired_mmproj) if prof.paired_mmproj else "none"
self._last_selected_mmproj = target_mmproj
self.query_one("#select-mmproj", Select).value = target_mmproj
# Sync active RadioButton in RadioSet
self._set_active_radio(preset_key)
self.update_binding_indicator()
self.update_vram_estimation()
finally:
self._sync_lock = False
def _find_profile_for_preset(self, preset_key: str) -> Optional[ModelProfile]:
preset = PRESETS.get(preset_key)
if not preset or not preset.get("target_model_match"):
return None
target = preset["target_model_match"].lower()
for prof in self.profiles:
if target in prof.name.lower() or target in prof.directory.name.lower():
return prof
return None
def on_radio_set_changed(self, event: RadioSet.Changed) -> None:
if self._sync_lock:
return
radio_id = event.pressed.id or ""
preset_key = radio_id.replace("preset-", "")
self.apply_preset(preset_key)
def on_select_changed(self, event: Select.Changed) -> None:
if self._sync_lock:
return
if event.select.id == "select-model":
model_path_str = str(event.value or "")
if model_path_str == self._last_selected_model:
return
self._last_selected_model = model_path_str
self._sync_lock = True
try:
prof = self.profiles_by_path.get(model_path_str)
if prof:
# 1. Automatically tie matching MMProj
if prof.paired_mmproj:
self._last_selected_mmproj = str(prof.paired_mmproj)
self.query_one("#select-mmproj", Select).value = str(prof.paired_mmproj)
else:
self._last_selected_mmproj = "none"
self.query_one("#select-mmproj", Select).value = "none"
# 2. Automatically sync inference preset and sampling defaults
p_key = prof.preset_key
if self.selected_preset == "cluster" and p_key in ("superqwen", "qwen4b"):
# SuperQwen and Qwen 4B are the Planner and Workers of Cluster mode
pass
elif p_key != "custom" and p_key in PRESETS:
self.selected_preset = p_key
self._set_active_radio(p_key)
preset = PRESETS[p_key]
self.query_one("#preset-description", Label).update(preset["desc"])
self.query_one("#input-port", Input).value = str(preset["port"])
self.query_one("#input-ctx", Input).value = str(preset["ctx"])
self.query_one("#input-temp", Input).value = str(preset["temp"])
self.query_one("#input-top-p", Input).value = str(preset["top_p"])
self.query_one("#input-min-p", Input).value = str(preset["min_p"])
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
self.query_one("#select-kv", Select).value = preset["kv_quant"]
else:
self.selected_preset = "custom"
self._set_active_radio("custom")
self.query_one("#preset-description", Label).update(PRESETS["custom"]["desc"])
self.update_binding_indicator()
self.update_vram_estimation()
finally:
self._sync_lock = False
elif event.select.id == "select-mmproj":
mmproj_str = str(event.value or "")
if mmproj_str == self._last_selected_mmproj:
return
self._last_selected_mmproj = mmproj_str
self.update_binding_indicator()
self.update_vram_estimation()
elif event.select.id == "select-kv":
self.update_binding_indicator()
self.update_vram_estimation() self.update_vram_estimation()
def on_input_changed(self, event: Input.Changed) -> None: def on_input_changed(self, event: Input.Changed) -> None:
self.update_vram_estimation() self.update_vram_estimation()
def on_select_changed(self, event: Select.Changed) -> None: def update_binding_indicator(self) -> None:
self.update_vram_estimation() preset_key = self.selected_preset
preset = PRESETS.get(preset_key, PRESETS["custom"])
model_val = str(self.query_one("#select-model", Select).value or "")
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
model_name = Path(model_val).name if (model_val and Path(model_val).exists()) else "None"
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
mmproj_text = f"[green]✔ {Path(mmproj_val).name} (Vision Enabled)[/]"
else:
mmproj_text = "[yellow]○ None (Text Only / Vision Disabled)[/]"
if preset.get("is_cluster"):
text = (
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
f" • [bold]Planner (Port 11434):[/] SuperQwen 3.8 27B + mmproj-SuperQwen (64K Ctx)\n"
f" • [bold]Workers (Ports 11435/11436):[/] 2x Qwen 3.5 4B + mmproj-BF16 (16K Ctx)\n"
f" • [bold green]● GGUF weights, projectors, and inference presets are coordinated across all 3 nodes[/]"
)
else:
text = (
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
f" • [bold]GGUF Model:[/] [bold magenta]{model_name}[/]\n"
f" • [bold]Vision Projector:[/] {mmproj_text}\n"
f" • [bold]Chat Template:[/] Jinja template auto-paired from model directory"
)
self.query_one("#binding-indicator", Static).update(text)
def update_vram_estimation(self) -> None: def update_vram_estimation(self) -> None:
try: try:
@ -390,25 +625,31 @@ class LLMConfigApp(App):
ctx = 65536 ctx = 65536
preset = self.selected_preset preset = self.selected_preset
kv_choice = str(self.query_one("#select-kv", Select).value or "q8_0")
kv_mult = 1.0 if kv_choice == "q8_0" else (2.0 if kv_choice == "f16" else 0.5)
if preset == "cluster": if preset == "cluster":
weights_gb = 25.9 weights_gb = 25.9
kv_gb = 3.2 kv_gb = 3.2 * kv_mult
elif preset == "muse":
weights_gb = 17.5
kv_gb = (ctx / 131072.0) * 8.5
elif preset == "superqwen":
weights_gb = 17.5
kv_gb = (ctx / 262144.0) * 10.5
elif preset == "qwen4b":
weights_gb = 3.9
kv_gb = (ctx / 32768.0) * 1.5
else: else:
model_val = str(self.query_one("#select-model", Select).value or "") model_val = str(self.query_one("#select-model", Select).value or "")
if model_val and Path(model_val).exists(): if model_val and Path(model_val).exists():
weights_gb = Path(model_val).stat().st_size / 1e9 model_gb = Path(model_val).stat().st_size / 1e9
else: else:
weights_gb = 15.0 model_gb = 16.0
kv_gb = (ctx / 65536.0) * 4.0
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
mmproj_gb = Path(mmproj_val).stat().st_size / 1e9
else:
mmproj_gb = 0.0
weights_gb = model_gb + mmproj_gb
if weights_gb > 10.0:
kv_gb = (ctx / 131072.0) * 8.5 * kv_mult
else:
kv_gb = (ctx / 32768.0) * 1.5 * kv_mult
total_est = weights_gb + kv_gb total_est = weights_gb + kv_gb
remaining = TOTAL_USABLE_VRAM_GB - total_est remaining = TOTAL_USABLE_VRAM_GB - total_est
@ -454,7 +695,9 @@ class LLMConfigApp(App):
def action_launch(self) -> None: def action_launch(self) -> None:
preset = self.selected_preset preset = self.selected_preset
log_widget = self.query_one("#log-stream", Log) log_widget = self.query_one("#log-stream", Log)
log_widget.write(f"\n[bold cyan]>>> Launching preset: {preset}...[/]\n")
model_val = str(self.query_one("#select-model", Select).value or "")
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
env = os.environ.copy() env = os.environ.copy()
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" env["CTX"] = self.query_one("#input-ctx", Input).value or "65536"
@ -467,9 +710,29 @@ class LLMConfigApp(App):
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
env["CTV"] = env["CTK"] env["CTV"] = env["CTK"]
cmd = [str(RUNNER_BIN), preset] if model_val and Path(model_val).exists():
env["MODEL"] = model_val
tmpl = Path(model_val).parent / "chat_template.jinja"
if tmpl.exists():
env["TEMPLATE"] = str(tmpl)
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
env["MMPROJ"] = mmproj_val
else:
env["MMPROJ"] = "none"
runner_cmd = PRESETS.get(preset, {}).get("command", preset)
cmd = [str(RUNNER_BIN), runner_cmd]
log_widget.write(
f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n"
f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n"
f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n"
f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n"
)
subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True) subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True)
self.notify(f"Launching {preset} server in background. Checking ports...", severity="information") self.notify(f"Launching {preset} server. Check Status & Live Logs tab.", severity="information")
self.action_refresh_status() self.action_refresh_status()
def action_stop(self) -> None: def action_stop(self) -> None: