diff --git a/README.md b/README.md index a36af54..d2b78f2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ uv run bin/llm-tui ``` Inside the TUI: -- `F5` / **Launch**: Starts the server(s) in the background. +- **Tied Configurations**: GGUF model weights, vision multimodal projectors (`mmproj`), chat templates, and inference presets are automatically bound bi-directionally. Selecting an inference preset auto-selects its matching GGUF model and mmproj; selecting a GGUF model auto-selects its paired mmproj and appropriate preset tuning. +- `F5` / **Launch**: Starts the server(s) in the background with selected weights, projector, and parameters. - `F8` / **Stop**: Stops server on the selected port. - `F9` / **Stop All**: Kills all running `llama-server` instances. - `F2` / **Refresh**: Refreshes health status. @@ -50,6 +51,12 @@ Inside the TUI: # SuperQwen 3.8 27B Solo (256K Context on port 11434) ./bin/llm-runner superqwen +# Qwen 3.5 4B Solo (port 11434) +./bin/llm-runner qwen4b + +# Custom Model Instance (with custom GGUF and vision projector) +MODEL=models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf MMPROJ=models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf ./bin/llm-runner custom + # Check status of running servers ./bin/llm-runner status diff --git a/bin/llm-runner b/bin/llm-runner index 83fd6fc..a830f80 100755 --- a/bin/llm-runner +++ b/bin/llm-runner @@ -199,16 +199,18 @@ cmd_cluster() { trap cleanup_cluster SIGINT SIGTERM EXIT # SuperQwen 3.8 27B (Planner) - local planner_model="${PLANNER_MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}" + local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}" if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" fi local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" # Qwen 3.5 4B (Workers) local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" # 1. Planner on 11434 @@ -244,6 +246,7 @@ cmd_muse() { log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..." local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}" local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}" + if [ "${mmproj}" = "none" ]; then mmproj=""; fi local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ @@ -264,6 +267,7 @@ cmd_superqwen() { model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" fi local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + if [ "${mmproj}" = "none" ]; then mmproj=""; fi local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ @@ -281,6 +285,7 @@ cmd_qwen4b() { log_info "Starting Qwen 3.5 4B Solo Server..." local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + if [ "${mmproj}" = "none" ]; then mmproj=""; fi local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ @@ -294,6 +299,29 @@ cmd_qwen4b() { wait } +cmd_custom() { + log_info "Starting Custom Server Instance..." + local model="${MODEL:-}" + if [ -z "${model}" ] || [ ! -f "${model}" ]; then + log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf" + exit 1 + fi + local mmproj="${MMPROJ:-}" + if [ "${mmproj}" = "none" ]; then mmproj=""; fi + local tmpl="${TEMPLATE:-}" + local name="custom_solo" + + launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \ + "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ + "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}" + + echo "" + echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log" + echo "Press Ctrl+C to stop." + wait +} + # ------------------------------------------------------------------------------ # Help / Usage # ------------------------------------------------------------------------------ @@ -306,11 +334,15 @@ Commands: muse Launch Muse-Glimmer 30B Solo (128K context, port 11434) superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434) qwen4b Launch Qwen 3.5 4B Solo (port 11434) + custom Launch custom model instance (requires MODEL=/path/to/model.gguf) status Show current server health and active ports stop [target] Stop servers (target: "all" or port number like 11434) help Display this help message Environment Variables: + MODEL Path to GGUF model file + MMPROJ Path to mmproj file (or "none" for text-only) + TEMPLATE Path to chat template file HOST Bind address (default: 0.0.0.0) PORT Main port override (default: 11434) CTX Context size override @@ -331,6 +363,7 @@ case "${ACTION}" in muse) cmd_muse "$@" ;; superqwen) cmd_superqwen "$@" ;; qwen4b) cmd_qwen4b "$@" ;; + custom) cmd_custom "$@" ;; status) cmd_status "$@" ;; stop) cmd_stop "$@" ;; help|--help|-h) cmd_help ;; diff --git a/bin/llm-tui b/bin/llm-tui index 0c8f3d0..ce2bd11 100755 --- a/bin/llm-tui +++ b/bin/llm-tui @@ -14,6 +14,7 @@ import signal import subprocess import urllib.request import json +from dataclasses import dataclass from pathlib import Path from typing import Optional @@ -45,10 +46,22 @@ RUNNER_BIN = SCRIPT_DIR / "bin" / "llm-runner" TOTAL_USABLE_VRAM_GB = 30.0 + +@dataclass +class ModelProfile: + name: str + path: Path + size_gb: float + directory: Path + paired_mmproj: Optional[Path] + paired_template: Optional[Path] + preset_key: str + + PRESETS = { "cluster": { "name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)", - "desc": "Planner on 11434, Worker 1 on 11435, Worker 2 on 11436. Optimized for Pi & Hermes agents.", + "desc": "Planner on 11434 (SuperQwen 27B + mmproj), Worker 1 on 11435, Worker 2 on 11436 (Qwen 4B + mmproj). Optimized for Pi & Hermes agents.", "port": 11434, "ctx": 65536, "temp": 0.6, @@ -59,10 +72,12 @@ PRESETS = { "flash_attn": True, "kv_quant": "q8_0", "command": "cluster", + "target_model_match": "SuperQwen", + "is_cluster": True, }, "muse": { "name": "🔮 Muse-Glimmer 30B Solo (128K Context)", - "desc": "Dedicated deep-reasoning agent with max 128K context on port 11434.", + "desc": "Dedicated deep-reasoning agent with max 128K context on port 11434. Auto-pairs Muse-Glimmer Q8_0 vision projector.", "port": 11434, "ctx": 131072, "temp": 0.6, @@ -73,10 +88,12 @@ PRESETS = { "flash_attn": True, "kv_quant": "q8_0", "command": "muse", + "target_model_match": "Muse-Glimmer", + "is_cluster": False, }, "superqwen": { "name": "🧠 SuperQwen 3.8 27B Solo (256K Context)", - "desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434.", + "desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434. Auto-pairs SuperQwen Q8_0 vision projector.", "port": 11434, "ctx": 262144, "temp": 0.6, @@ -87,10 +104,12 @@ PRESETS = { "flash_attn": True, "kv_quant": "q8_0", "command": "superqwen", + "target_model_match": "SuperQwen", + "is_cluster": False, }, "qwen4b": { "name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)", - "desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint.", + "desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint. Auto-pairs BF16 vision projector.", "port": 11434, "ctx": 32768, "temp": 0.7, @@ -101,6 +120,8 @@ PRESETS = { "flash_attn": True, "kv_quant": "q8_0", "command": "qwen4b", + "target_model_match": "Qwen3.5-4B", + "is_cluster": False, }, "custom": { "name": "🛠️ Custom Model & Parameters", @@ -115,28 +136,80 @@ PRESETS = { "flash_attn": True, "kv_quant": "q8_0", "command": "custom", + "target_model_match": None, + "is_cluster": False, }, } -def find_gguf_models() -> list[tuple[str, str]]: - models: list[tuple[str, str]] = [] - if MODELS_DIR.exists(): - for path in sorted(MODELS_DIR.rglob("*.gguf")): - if "mmproj" not in path.name.lower(): - rel = path.relative_to(MODELS_DIR) - models.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path))) - if not models: - models.append(("No GGUF models found in models/", "")) - return models +def scan_models() -> tuple[list[ModelProfile], list[tuple[str, str]], list[tuple[str, str]]]: + profiles: list[ModelProfile] = [] + model_options: list[tuple[str, str]] = [] + mmproj_options: list[tuple[str, str]] = [("None (Text Only / Vision Disabled)", "none")] + if not MODELS_DIR.exists(): + return profiles, [("No GGUF models found in models/", "")], mmproj_options -def find_mmproj_files() -> list[tuple[str, str]]: - projectors = [("None (Text Only)", "")] - if MODELS_DIR.exists(): - for path in sorted(MODELS_DIR.rglob("mmproj*.gguf")): - projectors.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path))) - return projectors + # Gather all mmproj projector files first + all_mmproj = sorted(MODELS_DIR.rglob("mmproj*.gguf")) + for p in all_mmproj: + try: + sz = p.stat().st_size / 1e9 + except Exception: + sz = 0.0 + mmproj_options.append((f"{p.name} ({sz:.1f} GB) [{p.parent.name}]", str(p))) + + # Gather base model GGUF files + for p in sorted(MODELS_DIR.rglob("*.gguf")): + if "mmproj" in p.name.lower(): + continue + try: + sz = p.stat().st_size / 1e9 + except Exception: + sz = 0.0 + + # Find paired mmproj in same folder + paired_mmproj = None + for m in all_mmproj: + if m.parent == p.parent: + paired_mmproj = m + break + + # Find paired chat template in directory + tmpl = p.parent / "chat_template.jinja" + paired_template = tmpl if tmpl.exists() else None + + # Determine preset mapping + p_lower = (p.name + " " + p.parent.name).lower() + if "muse" in p_lower: + preset_key = "muse" + tag = "Solo Reasoning (128K)" + elif "superqwen" in p_lower or "qwen3.8" in p_lower: + preset_key = "superqwen" + tag = "Solo 256K / Planner" + elif "qwen3.5" in p_lower or "4b" in p_lower: + preset_key = "qwen4b" + tag = "Fast Instruct / Worker" + else: + preset_key = "custom" + tag = "Custom" + + prof = ModelProfile( + name=p.name, + path=p, + size_gb=sz, + directory=p.parent, + paired_mmproj=paired_mmproj, + paired_template=paired_template, + preset_key=preset_key, + ) + profiles.append(prof) + model_options.append((f"{p.name} ({sz:.1f} GB) — {tag}", str(p))) + + if not model_options: + model_options.append(("No GGUF models found in models/", "")) + + return profiles, model_options, mmproj_options def check_port_status(port: int) -> tuple[bool, str]: @@ -224,6 +297,12 @@ class LLMConfigApp(App): text-style: bold; border: solid #d08770; } + #binding-indicator { + background: #1e2531; + border-left: thick #88c0d0; + padding: 1 2; + margin-bottom: 1; + } #vram-meter { background: #232834; border: solid #4c566a; @@ -253,13 +332,35 @@ class LLMConfigApp(App): ] selected_preset: str = "cluster" - models_list: list[tuple[str, str]] = [] - mmproj_list: list[tuple[str, str]] = [] + profiles: list[ModelProfile] = [] + profiles_by_path: dict[str, ModelProfile] = {} + models_options: list[tuple[str, str]] = [] + mmproj_options: list[tuple[str, str]] = [] + _sync_lock: bool = False + _last_selected_model: str = "" + _last_selected_mmproj: str = "" def compose(self) -> ComposeResult: yield Header(show_clock=True) - self.models_list = find_gguf_models() - self.mmproj_list = find_mmproj_files() + self.profiles, self.models_options, self.mmproj_options = scan_models() + self.profiles_by_path = {str(p.path): p for p in self.profiles} + + # Choose default model and mmproj based on initial cluster preset + default_model = "" + default_mmproj = "none" + for p in self.profiles: + if "superqwen" in p.name.lower() or "qwen3.8" in p.name.lower(): + default_model = str(p.path) + if p.paired_mmproj: + default_mmproj = str(p.paired_mmproj) + break + if not default_model and self.profiles: + default_model = str(self.profiles[0].path) + if self.profiles[0].paired_mmproj: + default_mmproj = str(self.profiles[0].paired_mmproj) + + self._last_selected_model = default_model + self._last_selected_mmproj = default_mmproj with TabbedContent(): with TabPane("🚀 Model Launcher & Form", id="tab-launch"): @@ -280,23 +381,26 @@ class LLMConfigApp(App): classes="text-muted", ) - # 2. Parameters Form + # 2. Active Tied Binding Indicator Card + yield Static(id="binding-indicator") + + # 3. Parameters Form (GGUF, MMProj, and Tuning) with Container(classes="card", id="form-container"): yield Label("Hyperparameters & Hardware Tuning", classes="title") with Horizontal(classes="field-row"): yield Label("GGUF Model:", classes="field-label") yield Select( - options=[(lbl, val) for lbl, val in self.models_list], - value=self.models_list[0][1] if self.models_list else None, + options=[(lbl, val) for lbl, val in self.models_options], + value=default_model if default_model else None, id="select-model", ) with Horizontal(classes="field-row"): yield Label("Vision MMProj:", classes="field-label") yield Select( - options=[(lbl, val) for lbl, val in self.mmproj_list], - value=self.mmproj_list[0][1] if self.mmproj_list else None, + options=[(lbl, val) for lbl, val in self.mmproj_options], + value=default_mmproj, id="select-mmproj", ) @@ -339,7 +443,7 @@ class LLMConfigApp(App): # VRAM Estimation Banner yield Static(id="vram-meter") - # 3. Execution Action Bar + # 4. Execution Action Bar with Horizontal(classes="card"): yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch") yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop") @@ -355,15 +459,25 @@ class LLMConfigApp(App): yield Footer() def on_mount(self) -> None: - self.update_vram_estimation() + self.apply_preset(self.selected_preset) self.action_refresh_status() self.set_interval(3.0, self.action_refresh_status) self.set_interval(1.5, self.tail_logs) - def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - radio_id = event.pressed.id or "" - preset_key = radio_id.replace("preset-", "") - if preset_key in PRESETS: + def _set_active_radio(self, preset_key: str) -> None: + rs = self.query_one("#preset-radios", RadioSet) + target = self.query_one(f"#preset-{preset_key}", RadioButton) + with rs.prevent(RadioButton.Changed): + if rs._pressed_button and rs._pressed_button != target: + rs._pressed_button.value = False + target.value = True + rs._pressed_button = target + + def apply_preset(self, preset_key: str) -> None: + if preset_key not in PRESETS: + return + self._sync_lock = True + try: self.selected_preset = preset_key preset = PRESETS[preset_key] self.query_one("#preset-description", Label).update(preset["desc"]) @@ -375,13 +489,134 @@ class LLMConfigApp(App): self.query_one("#input-ngl", Input).value = str(preset["ngl"]) self.query_one("#check-fa", Checkbox).value = preset["flash_attn"] self.query_one("#select-kv", Select).value = preset["kv_quant"] + + # Automatically tie GGUF model and MMProj to preset + prof = self._find_profile_for_preset(preset_key) + if prof: + self._last_selected_model = str(prof.path) + self.query_one("#select-model", Select).value = str(prof.path) + target_mmproj = str(prof.paired_mmproj) if prof.paired_mmproj else "none" + self._last_selected_mmproj = target_mmproj + self.query_one("#select-mmproj", Select).value = target_mmproj + + # Sync active RadioButton in RadioSet + self._set_active_radio(preset_key) + + self.update_binding_indicator() + self.update_vram_estimation() + finally: + self._sync_lock = False + + def _find_profile_for_preset(self, preset_key: str) -> Optional[ModelProfile]: + preset = PRESETS.get(preset_key) + if not preset or not preset.get("target_model_match"): + return None + target = preset["target_model_match"].lower() + for prof in self.profiles: + if target in prof.name.lower() or target in prof.directory.name.lower(): + return prof + return None + + def on_radio_set_changed(self, event: RadioSet.Changed) -> None: + if self._sync_lock: + return + radio_id = event.pressed.id or "" + preset_key = radio_id.replace("preset-", "") + self.apply_preset(preset_key) + + def on_select_changed(self, event: Select.Changed) -> None: + if self._sync_lock: + return + + if event.select.id == "select-model": + model_path_str = str(event.value or "") + if model_path_str == self._last_selected_model: + return + self._last_selected_model = model_path_str + + self._sync_lock = True + try: + prof = self.profiles_by_path.get(model_path_str) + if prof: + # 1. Automatically tie matching MMProj + if prof.paired_mmproj: + self._last_selected_mmproj = str(prof.paired_mmproj) + self.query_one("#select-mmproj", Select).value = str(prof.paired_mmproj) + else: + self._last_selected_mmproj = "none" + self.query_one("#select-mmproj", Select).value = "none" + + # 2. Automatically sync inference preset and sampling defaults + p_key = prof.preset_key + if self.selected_preset == "cluster" and p_key in ("superqwen", "qwen4b"): + # SuperQwen and Qwen 4B are the Planner and Workers of Cluster mode + pass + elif p_key != "custom" and p_key in PRESETS: + self.selected_preset = p_key + self._set_active_radio(p_key) + preset = PRESETS[p_key] + self.query_one("#preset-description", Label).update(preset["desc"]) + self.query_one("#input-port", Input).value = str(preset["port"]) + self.query_one("#input-ctx", Input).value = str(preset["ctx"]) + self.query_one("#input-temp", Input).value = str(preset["temp"]) + self.query_one("#input-top-p", Input).value = str(preset["top_p"]) + self.query_one("#input-min-p", Input).value = str(preset["min_p"]) + self.query_one("#input-ngl", Input).value = str(preset["ngl"]) + self.query_one("#check-fa", Checkbox).value = preset["flash_attn"] + self.query_one("#select-kv", Select).value = preset["kv_quant"] + else: + self.selected_preset = "custom" + self._set_active_radio("custom") + self.query_one("#preset-description", Label).update(PRESETS["custom"]["desc"]) + + self.update_binding_indicator() + self.update_vram_estimation() + finally: + self._sync_lock = False + + elif event.select.id == "select-mmproj": + mmproj_str = str(event.value or "") + if mmproj_str == self._last_selected_mmproj: + return + self._last_selected_mmproj = mmproj_str + self.update_binding_indicator() + self.update_vram_estimation() + + elif event.select.id == "select-kv": + self.update_binding_indicator() self.update_vram_estimation() def on_input_changed(self, event: Input.Changed) -> None: self.update_vram_estimation() - def on_select_changed(self, event: Select.Changed) -> None: - self.update_vram_estimation() + def update_binding_indicator(self) -> None: + preset_key = self.selected_preset + preset = PRESETS.get(preset_key, PRESETS["custom"]) + model_val = str(self.query_one("#select-model", Select).value or "") + mmproj_val = str(self.query_one("#select-mmproj", Select).value or "") + + model_name = Path(model_val).name if (model_val and Path(model_val).exists()) else "None" + if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists(): + mmproj_text = f"[green]✔ {Path(mmproj_val).name} (Vision Enabled)[/]" + else: + mmproj_text = "[yellow]○ None (Text Only / Vision Disabled)[/]" + + if preset.get("is_cluster"): + text = ( + f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n" + f" • [bold]Planner (Port 11434):[/] SuperQwen 3.8 27B + mmproj-SuperQwen (64K Ctx)\n" + f" • [bold]Workers (Ports 11435/11436):[/] 2x Qwen 3.5 4B + mmproj-BF16 (16K Ctx)\n" + f" • [bold green]● GGUF weights, projectors, and inference presets are coordinated across all 3 nodes[/]" + ) + else: + text = ( + f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n" + f" • [bold]GGUF Model:[/] [bold magenta]{model_name}[/]\n" + f" • [bold]Vision Projector:[/] {mmproj_text}\n" + f" • [bold]Chat Template:[/] Jinja template auto-paired from model directory" + ) + + self.query_one("#binding-indicator", Static).update(text) def update_vram_estimation(self) -> None: try: @@ -390,25 +625,31 @@ class LLMConfigApp(App): ctx = 65536 preset = self.selected_preset + kv_choice = str(self.query_one("#select-kv", Select).value or "q8_0") + kv_mult = 1.0 if kv_choice == "q8_0" else (2.0 if kv_choice == "f16" else 0.5) + if preset == "cluster": weights_gb = 25.9 - kv_gb = 3.2 - elif preset == "muse": - weights_gb = 17.5 - kv_gb = (ctx / 131072.0) * 8.5 - elif preset == "superqwen": - weights_gb = 17.5 - kv_gb = (ctx / 262144.0) * 10.5 - elif preset == "qwen4b": - weights_gb = 3.9 - kv_gb = (ctx / 32768.0) * 1.5 + kv_gb = 3.2 * kv_mult else: model_val = str(self.query_one("#select-model", Select).value or "") if model_val and Path(model_val).exists(): - weights_gb = Path(model_val).stat().st_size / 1e9 + model_gb = Path(model_val).stat().st_size / 1e9 else: - weights_gb = 15.0 - kv_gb = (ctx / 65536.0) * 4.0 + model_gb = 16.0 + + mmproj_val = str(self.query_one("#select-mmproj", Select).value or "") + if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists(): + mmproj_gb = Path(mmproj_val).stat().st_size / 1e9 + else: + mmproj_gb = 0.0 + + weights_gb = model_gb + mmproj_gb + + if weights_gb > 10.0: + kv_gb = (ctx / 131072.0) * 8.5 * kv_mult + else: + kv_gb = (ctx / 32768.0) * 1.5 * kv_mult total_est = weights_gb + kv_gb remaining = TOTAL_USABLE_VRAM_GB - total_est @@ -454,7 +695,9 @@ class LLMConfigApp(App): def action_launch(self) -> None: preset = self.selected_preset log_widget = self.query_one("#log-stream", Log) - log_widget.write(f"\n[bold cyan]>>> Launching preset: {preset}...[/]\n") + + model_val = str(self.query_one("#select-model", Select).value or "") + mmproj_val = str(self.query_one("#select-mmproj", Select).value or "") env = os.environ.copy() env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" @@ -467,9 +710,29 @@ class LLMConfigApp(App): env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") env["CTV"] = env["CTK"] - cmd = [str(RUNNER_BIN), preset] + if model_val and Path(model_val).exists(): + env["MODEL"] = model_val + tmpl = Path(model_val).parent / "chat_template.jinja" + if tmpl.exists(): + env["TEMPLATE"] = str(tmpl) + + if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists(): + env["MMPROJ"] = mmproj_val + else: + env["MMPROJ"] = "none" + + runner_cmd = PRESETS.get(preset, {}).get("command", preset) + cmd = [str(RUNNER_BIN), runner_cmd] + + log_widget.write( + f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n" + f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n" + f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n" + f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n" + ) + subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True) - self.notify(f"Launching {preset} server in background. Checking ports...", severity="information") + self.notify(f"Launching {preset} server. Check Status & Live Logs tab.", severity="information") self.action_refresh_status() def action_stop(self) -> None: