Compare commits

..

6 commits

3 changed files with 495 additions and 133 deletions

View file

@ -33,7 +33,8 @@ uv run bin/llm-tui
```
Inside the TUI:
- `F5` / **Launch**: Starts the server(s) in the background.
- **Tied Configurations**: GGUF model weights, vision multimodal projectors (`mmproj`), chat templates, and inference presets are automatically bound bi-directionally. Selecting an inference preset auto-selects its matching GGUF model and mmproj; selecting a GGUF model auto-selects its paired mmproj and appropriate preset tuning.
- `F5` / **Launch**: Starts the server(s) in the background with selected weights, projector, and parameters.
- `F8` / **Stop**: Stops server on the selected port.
- `F9` / **Stop All**: Kills all running `llama-server` instances.
- `F2` / **Refresh**: Refreshes health status.
@ -50,6 +51,12 @@ Inside the TUI:
# SuperQwen 3.8 27B Solo (256K Context on port 11434)
./bin/llm-runner superqwen
# Qwen 3.5 4B Solo (port 11434)
./bin/llm-runner qwen4b
# Custom Model Instance (with custom GGUF and vision projector)
MODEL=models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf MMPROJ=models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf ./bin/llm-runner custom
# Check status of running servers
./bin/llm-runner status

View file

@ -6,8 +6,19 @@
set -euo pipefail
# Sanitize environment:
# TEMP/TMP are standard POSIX variables for temporary file directories.
# If set to a sampling temperature (float/number), ROCm/libstdc++ crashes during initialization!
if [ -n "${TEMP:-}" ] && [ ! -d "${TEMP}" ]; then
TEMPERATURE="${TEMPERATURE:-${TEMP}}"
unset TEMP
fi
if [ -n "${TMP:-}" ] && [ ! -d "${TMP}" ]; then
unset TMP
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}"
LOG_DIR="${LLM_LOG_DIR:-${LOG_DIR:-${SCRIPT_DIR}/logs}}"
mkdir -p "${LOG_DIR}"
# ------------------------------------------------------------------------------
@ -45,7 +56,7 @@ LLAMA_SERVER="$(find_llama_server)"
check_health() {
local port="${1:-11434}"
local host="${2:-127.0.0.1}"
if curl -s "http://${host}:${port}/health" 2>/dev/null | grep -q '"status":\s*"ok"\|"status":\s*"loading model"\|"ok"'; then
if curl -s "http://${host}:${port}/health" 2>/dev/null | grep -q '"status":\s*"ok"\|"ok"'; then
return 0
fi
return 1
@ -84,14 +95,8 @@ cmd_stop() {
pkill -f "llama-server" 2>/dev/null || true
log_success "All llama-server processes terminated."
else
local pid
pid=$(lsof -ti :"$target" 2>/dev/null || fuser "$target/tcp" 2>/dev/null || true)
if [ -n "$pid" ]; then
kill "$pid" 2>/dev/null || true
log_success "Killed server on port ${target} (PID: ${pid})."
else
log_warn "No active server found on port ${target}."
fi
fuser -k -9 "$target/tcp" 2>/dev/null || true
log_success "Killed server on port ${target}."
fi
}
@ -122,20 +127,19 @@ launch_instance() {
exit 1
fi
# Check if port is already in use
local existing_pid
existing_pid=$(lsof -ti :"$port" 2>/dev/null || fuser "$port/tcp" 2>/dev/null || true)
if [ -n "$existing_pid" ]; then
log_warn "Port ${port} is already in use by PID ${existing_pid}. Terminating existing process..."
kill -9 "$existing_pid" 2>/dev/null || true
# Ensure target port is free
if fuser "$port/tcp" >/dev/null 2>&1; then
log_warn "Port ${port} is in use. Stopping existing process on port ${port}..."
fuser -k -9 "$port/tcp" 2>/dev/null || true
sleep 1
fi
local flags=(
-m "${model}"
--host "${HOST:-0.0.0.0}"
--host "${LLM_HOST:-${HOST:-0.0.0.0}}"
--port "${port}"
--ctx-size "${ctx}"
-np "${LLM_PARALLEL:-${PARALLEL:-1}}"
--n-gpu-layers "${ngl}"
--load-mode "${load_mode}"
--temp "${temp}"
@ -145,12 +149,12 @@ launch_instance() {
-ctk "${ctk}"
-ctv "${ctv}"
-fa "${flash_attn}"
-b "${BATCH_SIZE:-2048}"
-ub "${UBATCH_SIZE:-512}"
-b "${LLM_BATCH_SIZE:-${BATCH_SIZE:-2048}}"
-ub "${LLM_UBATCH_SIZE:-${UBATCH_SIZE:-512}}"
)
if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then
flags+=(--mmproj "${mmproj}" --image-min-tokens "${IMAGE_MIN_TOKENS:-1024}")
flags+=(--mmproj "${mmproj}" --image-min-tokens "${LLM_IMAGE_MIN_TOKENS:-${IMAGE_MIN_TOKENS:-1024}}")
fi
if [ -n "${template}" ] && [ -f "${template}" ]; then
@ -188,6 +192,8 @@ launch_instance() {
# ------------------------------------------------------------------------------
cmd_cluster() {
log_info "Starting Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)..."
pkill -f "llama-server" 2>/dev/null || true
sleep 1
# Trap to gracefully stop all background instances on Ctrl+C
cleanup_cluster() {
@ -199,41 +205,43 @@ cmd_cluster() {
trap cleanup_cluster SIGINT SIGTERM EXIT
# SuperQwen 3.8 27B (Planner)
local planner_model="${PLANNER_MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
local planner_model="${LLM_PLANNER_MODEL:-${PLANNER_MODEL:-${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}}}"
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
fi
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
local planner_mmproj="${LLM_PLANNER_MMPROJ:-${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi
local planner_tmpl="${LLM_PLANNER_TEMPLATE:-${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
# Qwen 3.5 4B (Workers)
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
local worker_model="${LLM_WORKER_MODEL:-${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
local worker_mmproj="${LLM_WORKER_MMPROJ:-${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi
local worker_tmpl="${LLM_WORKER_TEMPLATE:-${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
# 1. Planner on 11434
launch_instance "planner_27b" "$planner_model" "${PLANNER_PORT:-11434}" "${PLANNER_CTX:-65536}" \
"$planner_mmproj" "$planner_tmpl" "${PLANNER_TEMP:-0.6}" "${PLANNER_TOP_P:-0.95}" "${PLANNER_TOP_K:-40}" \
"${PLANNER_MIN_P:-0.05}" "${PLANNER_NGL:-99}" "${PLANNER_LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
"${PLANNER_CTK:-q8_0}" "${PLANNER_CTV:-q8_0}" "false"
launch_instance "planner_27b" "$planner_model" "${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}" "${LLM_PLANNER_CTX:-${PLANNER_CTX:-65536}}" \
"$planner_mmproj" "$planner_tmpl" "${LLM_PLANNER_TEMP:-${PLANNER_TEMP:-0.6}}" "${LLM_PLANNER_TOP_P:-${PLANNER_TOP_P:-0.95}}" "${LLM_PLANNER_TOP_K:-${PLANNER_TOP_K:-40}}" \
"${LLM_PLANNER_MIN_P:-${PLANNER_MIN_P:-0.05}}" "${LLM_PLANNER_NGL:-${PLANNER_NGL:-99}}" "${LLM_PLANNER_LOAD_MODE:-${PLANNER_LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_PLANNER_CTK:-${PLANNER_CTK:-q8_0}}" "${LLM_PLANNER_CTV:-${PLANNER_CTV:-q8_0}}" "false"
# 2. Worker 1 on 11435
launch_instance "worker_4b_1" "$worker_model" "${WORKER1_PORT:-11435}" "${WORKER1_CTX:-16384}" \
"$worker_mmproj" "$worker_tmpl" "${WORKER1_TEMP:-0.7}" "${WORKER1_TOP_P:-0.8}" "${WORKER1_TOP_K:-20}" \
"${WORKER1_MIN_P:-0.05}" "${WORKER1_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
"${WORKER1_CTK:-q8_0}" "${WORKER1_CTV:-q8_0}" "false"
launch_instance "worker_4b_1" "$worker_model" "${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}" "${LLM_WORKER1_CTX:-${WORKER1_CTX:-16384}}" \
"$worker_mmproj" "$worker_tmpl" "${LLM_WORKER1_TEMP:-${WORKER1_TEMP:-0.7}}" "${LLM_WORKER1_TOP_P:-${WORKER1_TOP_P:-0.8}}" "${LLM_WORKER1_TOP_K:-${WORKER1_TOP_K:-20}}" \
"${LLM_WORKER1_MIN_P:-${WORKER1_MIN_P:-0.05}}" "${LLM_WORKER1_NGL:-${WORKER1_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_WORKER1_CTK:-${WORKER1_CTK:-q8_0}}" "${LLM_WORKER1_CTV:-${WORKER1_CTV:-q8_0}}" "false"
# 3. Worker 2 on 11436
launch_instance "worker_4b_2" "$worker_model" "${WORKER2_PORT:-11436}" "${WORKER2_CTX:-16384}" \
"$worker_mmproj" "$worker_tmpl" "${WORKER2_TEMP:-0.7}" "${WORKER2_TOP_P:-0.8}" "${WORKER2_TOP_K:-20}" \
"${WORKER2_MIN_P:-0.05}" "${WORKER2_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
"${WORKER2_CTK:-q8_0}" "${WORKER2_CTV:-q8_0}" "false"
launch_instance "worker_4b_2" "$worker_model" "${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}" "${LLM_WORKER2_CTX:-${WORKER2_CTX:-16384}}" \
"$worker_mmproj" "$worker_tmpl" "${LLM_WORKER2_TEMP:-${WORKER2_TEMP:-0.7}}" "${LLM_WORKER2_TOP_P:-${WORKER2_TOP_P:-0.8}}" "${LLM_WORKER2_TOP_K:-${WORKER2_TOP_K:-20}}" \
"${LLM_WORKER2_MIN_P:-${WORKER2_MIN_P:-0.05}}" "${LLM_WORKER2_NGL:-${WORKER2_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_WORKER2_CTK:-${WORKER2_CTK:-q8_0}}" "${LLM_WORKER2_CTV:-${WORKER2_CTV:-q8_0}}" "false"
echo ""
log_success "All 3 cluster servers are operational:"
echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${PLANNER_PORT:-11434}"
echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${WORKER1_PORT:-11435}"
echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${WORKER2_PORT:-11436}"
echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}"
echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}"
echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}"
echo ""
echo "Logs directory: ${LOG_DIR}/"
echo "Press Ctrl+C to stop all instances gracefully."
@ -242,14 +250,20 @@ cmd_cluster() {
cmd_muse() {
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}"
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}"
pkill -f "llama-server" 2>/dev/null || true
sleep 1
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}}"
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}}"
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \
"$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}"
launch_instance "muse_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-131072}}" \
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log"
@ -259,17 +273,23 @@ cmd_muse() {
cmd_superqwen() {
log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..."
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}"
pkill -f "llama-server" 2>/dev/null || true
sleep 1
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}}"
if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
fi
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \
"$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}"
launch_instance "superqwen_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-262144}}" \
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log"
@ -279,14 +299,19 @@ cmd_superqwen() {
cmd_qwen4b() {
log_info "Starting Qwen 3.5 4B Solo Server..."
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
pkill -f "llama-server" 2>/dev/null || true
sleep 1
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \
"$mmproj" "$tmpl" "${TEMP:-0.7}" "${TOP_P:-0.8}" "${TOP_K:-20}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "false"
launch_instance "qwen4b_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-32768}}" \
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}}" \
"${LLM_TOP_P:-${TOP_P:-0.8}}" "${LLM_TOP_K:-${TOP_K:-20}}" \
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" "false"
echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log"
@ -294,6 +319,34 @@ cmd_qwen4b() {
wait
}
cmd_custom() {
log_info "Starting Custom Server Instance..."
pkill -f "llama-server" 2>/dev/null || true
sleep 1
local model="${LLM_MODEL:-${MODEL:-}}"
if [ -z "${model}" ] || [ ! -f "${model}" ]; then
log_error "No valid MODEL specified! Pass LLM_MODEL=/path/to/model.gguf"
exit 1
fi
local mmproj="${LLM_MMPROJ:-${MMPROJ:-}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-}}"
local name="custom_solo"
launch_instance "${name}" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-65536}}" \
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-false}}"
echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log"
echo "Press Ctrl+C to stop."
wait
}
# ------------------------------------------------------------------------------
# Help / Usage
# ------------------------------------------------------------------------------
@ -306,16 +359,22 @@ Commands:
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
qwen4b Launch Qwen 3.5 4B Solo (port 11434)
custom Launch custom model instance (requires LLM_MODEL=/path/to/model.gguf)
status Show current server health and active ports
stop [target] Stop servers (target: "all" or port number like 11434)
help Display this help message
Environment Variables:
HOST Bind address (default: 0.0.0.0)
PORT Main port override (default: 11434)
CTX Context size override
FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on)
LOG_DIR Path to write logs (default: ./logs)
LLM_MODEL Path to GGUF model file (fallback: MODEL)
LLM_MMPROJ Path to mmproj file, or "none" (fallback: MMPROJ)
LLM_TEMPLATE Path to chat template file (fallback: TEMPLATE)
LLM_HOST Bind address (default: 0.0.0.0)
LLM_PORT Main port override (default: 11434)
LLM_CTX Context size override (default: preset-specific)
LLM_TEMPERATURE Sampling temperature (default: preset-specific)
LLM_PARALLEL Parallel slot count (default: 1)
LLM_FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on)
LLM_LOG_DIR Path to write logs (default: ./logs)
EOF
}
@ -331,6 +390,7 @@ case "${ACTION}" in
muse) cmd_muse "$@" ;;
superqwen) cmd_superqwen "$@" ;;
qwen4b) cmd_qwen4b "$@" ;;
custom) cmd_custom "$@" ;;
status) cmd_status "$@" ;;
stop) cmd_stop "$@" ;;
help|--help|-h) cmd_help ;;

View file

@ -14,6 +14,7 @@ import signal
import subprocess
import urllib.request
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
@ -45,10 +46,22 @@ RUNNER_BIN = SCRIPT_DIR / "bin" / "llm-runner"
TOTAL_USABLE_VRAM_GB = 30.0
@dataclass
class ModelProfile:
name: str
path: Path
size_gb: float
directory: Path
paired_mmproj: Optional[Path]
paired_template: Optional[Path]
preset_key: str
PRESETS = {
"cluster": {
"name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)",
"desc": "Planner on 11434, Worker 1 on 11435, Worker 2 on 11436. Optimized for Pi & Hermes agents.",
"desc": "Planner on 11434 (SuperQwen 27B + mmproj), Worker 1 on 11435, Worker 2 on 11436 (Qwen 4B + mmproj). Optimized for Pi & Hermes agents.",
"port": 11434,
"ctx": 65536,
"temp": 0.6,
@ -59,10 +72,12 @@ PRESETS = {
"flash_attn": True,
"kv_quant": "q8_0",
"command": "cluster",
"target_model_match": "SuperQwen",
"is_cluster": True,
},
"muse": {
"name": "🔮 Muse-Glimmer 30B Solo (128K Context)",
"desc": "Dedicated deep-reasoning agent with max 128K context on port 11434.",
"desc": "Dedicated deep-reasoning agent with max 128K context on port 11434. Auto-pairs Muse-Glimmer Q8_0 vision projector.",
"port": 11434,
"ctx": 131072,
"temp": 0.6,
@ -73,10 +88,12 @@ PRESETS = {
"flash_attn": True,
"kv_quant": "q8_0",
"command": "muse",
"target_model_match": "Muse-Glimmer",
"is_cluster": False,
},
"superqwen": {
"name": "🧠 SuperQwen 3.8 27B Solo (256K Context)",
"desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434.",
"desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434. Auto-pairs SuperQwen Q8_0 vision projector.",
"port": 11434,
"ctx": 262144,
"temp": 0.6,
@ -87,10 +104,12 @@ PRESETS = {
"flash_attn": True,
"kv_quant": "q8_0",
"command": "superqwen",
"target_model_match": "SuperQwen",
"is_cluster": False,
},
"qwen4b": {
"name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)",
"desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint.",
"desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint. Auto-pairs BF16 vision projector.",
"port": 11434,
"ctx": 32768,
"temp": 0.7,
@ -101,6 +120,8 @@ PRESETS = {
"flash_attn": True,
"kv_quant": "q8_0",
"command": "qwen4b",
"target_model_match": "Qwen3.5-4B",
"is_cluster": False,
},
"custom": {
"name": "🛠️ Custom Model & Parameters",
@ -115,28 +136,80 @@ PRESETS = {
"flash_attn": True,
"kv_quant": "q8_0",
"command": "custom",
"target_model_match": None,
"is_cluster": False,
},
}
def find_gguf_models() -> list[tuple[str, str]]:
models: list[tuple[str, str]] = []
if MODELS_DIR.exists():
for path in sorted(MODELS_DIR.rglob("*.gguf")):
if "mmproj" not in path.name.lower():
rel = path.relative_to(MODELS_DIR)
models.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path)))
if not models:
models.append(("No GGUF models found in models/", ""))
return models
def scan_models() -> tuple[list[ModelProfile], list[tuple[str, str]], list[tuple[str, str]]]:
profiles: list[ModelProfile] = []
model_options: list[tuple[str, str]] = []
mmproj_options: list[tuple[str, str]] = [("None (Text Only / Vision Disabled)", "none")]
if not MODELS_DIR.exists():
return profiles, [("No GGUF models found in models/", "")], mmproj_options
def find_mmproj_files() -> list[tuple[str, str]]:
projectors = [("None (Text Only)", "")]
if MODELS_DIR.exists():
for path in sorted(MODELS_DIR.rglob("mmproj*.gguf")):
projectors.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path)))
return projectors
# Gather all mmproj projector files first
all_mmproj = sorted(MODELS_DIR.rglob("mmproj*.gguf"))
for p in all_mmproj:
try:
sz = p.stat().st_size / 1e9
except Exception:
sz = 0.0
mmproj_options.append((f"{p.name} ({sz:.1f} GB) [{p.parent.name}]", str(p)))
# Gather base model GGUF files
for p in sorted(MODELS_DIR.rglob("*.gguf")):
if "mmproj" in p.name.lower():
continue
try:
sz = p.stat().st_size / 1e9
except Exception:
sz = 0.0
# Find paired mmproj in same folder
paired_mmproj = None
for m in all_mmproj:
if m.parent == p.parent:
paired_mmproj = m
break
# Find paired chat template in directory
tmpl = p.parent / "chat_template.jinja"
paired_template = tmpl if tmpl.exists() else None
# Determine preset mapping
p_lower = (p.name + " " + p.parent.name).lower()
if "muse" in p_lower:
preset_key = "muse"
tag = "Solo Reasoning (128K)"
elif "superqwen" in p_lower or "qwen3.8" in p_lower:
preset_key = "superqwen"
tag = "Solo 256K / Planner"
elif "qwen3.5" in p_lower or "4b" in p_lower:
preset_key = "qwen4b"
tag = "Fast Instruct / Worker"
else:
preset_key = "custom"
tag = "Custom"
prof = ModelProfile(
name=p.name,
path=p,
size_gb=sz,
directory=p.parent,
paired_mmproj=paired_mmproj,
paired_template=paired_template,
preset_key=preset_key,
)
profiles.append(prof)
model_options.append((f"{p.name} ({sz:.1f} GB) — {tag}", str(p)))
if not model_options:
model_options.append(("No GGUF models found in models/", ""))
return profiles, model_options, mmproj_options
def check_port_status(port: int) -> tuple[bool, str]:
@ -147,6 +220,15 @@ def check_port_status(port: int) -> tuple[bool, str]:
data = json.loads(response.read().decode())
status = data.get("status", "unknown")
return True, f"Online ({status})"
except urllib.error.HTTPError as e:
if e.code == 503:
try:
data = json.loads(e.read().decode())
msg = data.get("error", {}).get("message", "Loading model")
return True, f"Loading ({msg})"
except Exception:
return True, "Loading model..."
return False, f"HTTP Error {e.code}"
except Exception:
return False, "Offline"
@ -173,6 +255,7 @@ class LLMConfigApp(App):
border: solid #3b4252;
padding: 1 2;
margin-bottom: 1;
height: auto;
}
.title {
color: #88c0d0;
@ -224,17 +307,26 @@ class LLMConfigApp(App):
text-style: bold;
border: solid #d08770;
}
#binding-indicator {
background: #1e2531;
border-left: thick #88c0d0;
padding: 1 2;
margin-bottom: 1;
height: auto;
}
#vram-meter {
background: #232834;
border: solid #4c566a;
padding: 1;
margin-top: 1;
height: auto;
}
#status-card {
background: #1e222b;
border: solid #4c566a;
padding: 1;
margin-bottom: 1;
height: auto;
}
Log {
background: #0f1117;
@ -253,13 +345,37 @@ class LLMConfigApp(App):
]
selected_preset: str = "cluster"
models_list: list[tuple[str, str]] = []
mmproj_list: list[tuple[str, str]] = []
profiles: list[ModelProfile] = []
profiles_by_path: dict[str, ModelProfile] = {}
models_options: list[tuple[str, str]] = []
mmproj_options: list[tuple[str, str]] = []
_sync_lock: bool = False
_last_selected_model: str = ""
_last_selected_mmproj: str = ""
_last_log_size: int = -1
_last_log_name: str = ""
def compose(self) -> ComposeResult:
yield Header(show_clock=True)
self.models_list = find_gguf_models()
self.mmproj_list = find_mmproj_files()
self.profiles, self.models_options, self.mmproj_options = scan_models()
self.profiles_by_path = {str(p.path): p for p in self.profiles}
# Choose default model and mmproj based on initial cluster preset
default_model = ""
default_mmproj = "none"
for p in self.profiles:
if "superqwen" in p.name.lower() or "qwen3.8" in p.name.lower():
default_model = str(p.path)
if p.paired_mmproj:
default_mmproj = str(p.paired_mmproj)
break
if not default_model and self.profiles:
default_model = str(self.profiles[0].path)
if self.profiles[0].paired_mmproj:
default_mmproj = str(self.profiles[0].paired_mmproj)
self._last_selected_model = default_model
self._last_selected_mmproj = default_mmproj
with TabbedContent():
with TabPane("🚀 Model Launcher & Form", id="tab-launch"):
@ -280,23 +396,26 @@ class LLMConfigApp(App):
classes="text-muted",
)
# 2. Parameters Form
# 2. Active Tied Binding Indicator Card
yield Static(id="binding-indicator")
# 3. Parameters Form (GGUF, MMProj, and Tuning)
with Container(classes="card", id="form-container"):
yield Label("Hyperparameters & Hardware Tuning", classes="title")
with Horizontal(classes="field-row"):
yield Label("GGUF Model:", classes="field-label")
yield Select(
options=[(lbl, val) for lbl, val in self.models_list],
value=self.models_list[0][1] if self.models_list else None,
options=[(lbl, val) for lbl, val in self.models_options],
value=default_model if default_model else None,
id="select-model",
)
with Horizontal(classes="field-row"):
yield Label("Vision MMProj:", classes="field-label")
yield Select(
options=[(lbl, val) for lbl, val in self.mmproj_list],
value=self.mmproj_list[0][1] if self.mmproj_list else None,
options=[(lbl, val) for lbl, val in self.mmproj_options],
value=default_mmproj,
id="select-mmproj",
)
@ -339,7 +458,7 @@ class LLMConfigApp(App):
# VRAM Estimation Banner
yield Static(id="vram-meter")
# 3. Execution Action Bar
# 4. Execution Action Bar
with Horizontal(classes="card"):
yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch")
yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop")
@ -355,15 +474,25 @@ class LLMConfigApp(App):
yield Footer()
def on_mount(self) -> None:
self.update_vram_estimation()
self.apply_preset(self.selected_preset)
self.action_refresh_status()
self.set_interval(3.0, self.action_refresh_status)
self.set_interval(1.5, self.tail_logs)
def on_radio_set_changed(self, event: RadioSet.Changed) -> None:
radio_id = event.pressed.id or ""
preset_key = radio_id.replace("preset-", "")
if preset_key in PRESETS:
def _set_active_radio(self, preset_key: str) -> None:
rs = self.query_one("#preset-radios", RadioSet)
target = self.query_one(f"#preset-{preset_key}", RadioButton)
with rs.prevent(RadioButton.Changed):
if rs._pressed_button and rs._pressed_button != target:
rs._pressed_button.value = False
target.value = True
rs._pressed_button = target
def apply_preset(self, preset_key: str) -> None:
if preset_key not in PRESETS:
return
self._sync_lock = True
try:
self.selected_preset = preset_key
preset = PRESETS[preset_key]
self.query_one("#preset-description", Label).update(preset["desc"])
@ -375,13 +504,134 @@ class LLMConfigApp(App):
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
self.query_one("#select-kv", Select).value = preset["kv_quant"]
# Automatically tie GGUF model and MMProj to preset
prof = self._find_profile_for_preset(preset_key)
if prof:
self._last_selected_model = str(prof.path)
self.query_one("#select-model", Select).value = str(prof.path)
target_mmproj = str(prof.paired_mmproj) if prof.paired_mmproj else "none"
self._last_selected_mmproj = target_mmproj
self.query_one("#select-mmproj", Select).value = target_mmproj
# Sync active RadioButton in RadioSet
self._set_active_radio(preset_key)
self.update_binding_indicator()
self.update_vram_estimation()
finally:
self._sync_lock = False
def _find_profile_for_preset(self, preset_key: str) -> Optional[ModelProfile]:
preset = PRESETS.get(preset_key)
if not preset or not preset.get("target_model_match"):
return None
target = preset["target_model_match"].lower()
for prof in self.profiles:
if target in prof.name.lower() or target in prof.directory.name.lower():
return prof
return None
def on_radio_set_changed(self, event: RadioSet.Changed) -> None:
if self._sync_lock:
return
radio_id = event.pressed.id or ""
preset_key = radio_id.replace("preset-", "")
self.apply_preset(preset_key)
def on_select_changed(self, event: Select.Changed) -> None:
if self._sync_lock:
return
if event.select.id == "select-model":
model_path_str = str(event.value or "")
if model_path_str == self._last_selected_model:
return
self._last_selected_model = model_path_str
self._sync_lock = True
try:
prof = self.profiles_by_path.get(model_path_str)
if prof:
# 1. Automatically tie matching MMProj
if prof.paired_mmproj:
self._last_selected_mmproj = str(prof.paired_mmproj)
self.query_one("#select-mmproj", Select).value = str(prof.paired_mmproj)
else:
self._last_selected_mmproj = "none"
self.query_one("#select-mmproj", Select).value = "none"
# 2. Automatically sync inference preset and sampling defaults
p_key = prof.preset_key
if self.selected_preset == "cluster" and p_key in ("superqwen", "qwen4b"):
# SuperQwen and Qwen 4B are the Planner and Workers of Cluster mode
pass
elif p_key != "custom" and p_key in PRESETS:
self.selected_preset = p_key
self._set_active_radio(p_key)
preset = PRESETS[p_key]
self.query_one("#preset-description", Label).update(preset["desc"])
self.query_one("#input-port", Input).value = str(preset["port"])
self.query_one("#input-ctx", Input).value = str(preset["ctx"])
self.query_one("#input-temp", Input).value = str(preset["temp"])
self.query_one("#input-top-p", Input).value = str(preset["top_p"])
self.query_one("#input-min-p", Input).value = str(preset["min_p"])
self.query_one("#input-ngl", Input).value = str(preset["ngl"])
self.query_one("#check-fa", Checkbox).value = preset["flash_attn"]
self.query_one("#select-kv", Select).value = preset["kv_quant"]
else:
self.selected_preset = "custom"
self._set_active_radio("custom")
self.query_one("#preset-description", Label).update(PRESETS["custom"]["desc"])
self.update_binding_indicator()
self.update_vram_estimation()
finally:
self._sync_lock = False
elif event.select.id == "select-mmproj":
mmproj_str = str(event.value or "")
if mmproj_str == self._last_selected_mmproj:
return
self._last_selected_mmproj = mmproj_str
self.update_binding_indicator()
self.update_vram_estimation()
elif event.select.id == "select-kv":
self.update_binding_indicator()
self.update_vram_estimation()
def on_input_changed(self, event: Input.Changed) -> None:
self.update_vram_estimation()
def on_select_changed(self, event: Select.Changed) -> None:
self.update_vram_estimation()
def update_binding_indicator(self) -> None:
preset_key = self.selected_preset
preset = PRESETS.get(preset_key, PRESETS["custom"])
model_val = str(self.query_one("#select-model", Select).value or "")
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
model_name = Path(model_val).name if (model_val and Path(model_val).exists()) else "None"
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
mmproj_text = f"[green]✔ {Path(mmproj_val).name} (Vision Enabled)[/]"
else:
mmproj_text = "[yellow]○ None (Text Only / Vision Disabled)[/]"
if preset.get("is_cluster"):
text = (
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
f" • [bold]Planner (Port 11434):[/] SuperQwen 3.8 27B + mmproj-SuperQwen (64K Ctx)\n"
f" • [bold]Workers (Ports 11435/11436):[/] 2x Qwen 3.5 4B + mmproj-BF16 (16K Ctx)\n"
f" • [bold green]● GGUF weights, projectors, and inference presets are coordinated across all 3 nodes[/]"
)
else:
text = (
f"[bold cyan]🔗 Tied Configuration:[/] [bold white]{preset['name']}[/]\n"
f" • [bold]GGUF Model:[/] [bold magenta]{model_name}[/]\n"
f" • [bold]Vision Projector:[/] {mmproj_text}\n"
f" • [bold]Chat Template:[/] Jinja template auto-paired from model directory"
)
self.query_one("#binding-indicator", Static).update(text)
def update_vram_estimation(self) -> None:
try:
@ -390,25 +640,31 @@ class LLMConfigApp(App):
ctx = 65536
preset = self.selected_preset
kv_choice = str(self.query_one("#select-kv", Select).value or "q8_0")
kv_mult = 1.0 if kv_choice == "q8_0" else (2.0 if kv_choice == "f16" else 0.5)
if preset == "cluster":
weights_gb = 25.9
kv_gb = 3.2
elif preset == "muse":
weights_gb = 17.5
kv_gb = (ctx / 131072.0) * 8.5
elif preset == "superqwen":
weights_gb = 17.5
kv_gb = (ctx / 262144.0) * 10.5
elif preset == "qwen4b":
weights_gb = 3.9
kv_gb = (ctx / 32768.0) * 1.5
kv_gb = 3.2 * kv_mult
else:
model_val = str(self.query_one("#select-model", Select).value or "")
if model_val and Path(model_val).exists():
weights_gb = Path(model_val).stat().st_size / 1e9
model_gb = Path(model_val).stat().st_size / 1e9
else:
weights_gb = 15.0
kv_gb = (ctx / 65536.0) * 4.0
model_gb = 16.0
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
mmproj_gb = Path(mmproj_val).stat().st_size / 1e9
else:
mmproj_gb = 0.0
weights_gb = model_gb + mmproj_gb
if weights_gb > 10.0:
kv_gb = (ctx / 131072.0) * 8.5 * kv_mult
else:
kv_gb = (ctx / 32768.0) * 1.5 * kv_mult
total_est = weights_gb + kv_gb
remaining = TOTAL_USABLE_VRAM_GB - total_est
@ -431,7 +687,12 @@ class LLMConfigApp(App):
lines = []
for port, label in [(11434, "Planner / Solo"), (11435, "Worker 1"), (11436, "Worker 2")]:
online, msg = check_port_status(port)
badge = "[green]● ONLINE[/]" if online else "[yellow]○ OFFLINE[/]"
if not online:
badge = "[yellow]○ OFFLINE[/]"
elif "loading" in msg.lower():
badge = "[cyan]⏳ LOADING[/]"
else:
badge = "[green]● ONLINE[/]"
lines.append(f"Port {port} ({label}): {badge} - {msg}")
card = self.query_one("#status-card", Static)
@ -443,8 +704,13 @@ class LLMConfigApp(App):
if logs:
latest = logs[0]
try:
cur_size = latest.stat().st_size
if cur_size == self._last_log_size and latest.name == self._last_log_name:
return
self._last_log_size = cur_size
self._last_log_name = latest.name
with open(latest, "r", errors="ignore") as f:
f.seek(max(0, latest.stat().st_size - 8192))
f.seek(max(0, cur_size - 16384))
tail = f.read()
log_widget.clear()
log_widget.write(f"--- Streaming {latest.name} ---\n" + tail)
@ -454,22 +720,51 @@ class LLMConfigApp(App):
def action_launch(self) -> None:
preset = self.selected_preset
log_widget = self.query_one("#log-stream", Log)
log_widget.write(f"\n[bold cyan]>>> Launching preset: {preset}...[/]\n")
model_val = str(self.query_one("#select-model", Select).value or "")
mmproj_val = str(self.query_one("#select-mmproj", Select).value or "")
env = os.environ.copy()
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536"
env["PORT"] = self.query_one("#input-port", Input).value or "11434"
env["TEMP"] = self.query_one("#input-temp", Input).value or "0.6"
env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95"
env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05"
env["NGL"] = self.query_one("#input-ngl", Input).value or "99"
env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
env["CTV"] = env["CTK"]
# Clean up POSIX temp directory variables that could collide with sampling parameters
env.pop("TEMP", None)
env.pop("TMP", None)
cmd = [str(RUNNER_BIN), preset]
subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True)
self.notify(f"Launching {preset} server in background. Checking ports...", severity="information")
env["LLM_TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6"
env["LLM_CTX"] = self.query_one("#input-ctx", Input).value or "65536"
env["LLM_PORT"] = self.query_one("#input-port", Input).value or "11434"
env["LLM_TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95"
env["LLM_MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05"
env["LLM_NGL"] = self.query_one("#input-ngl", Input).value or "99"
env["LLM_FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
env["LLM_CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
env["LLM_CTV"] = env["LLM_CTK"]
env["LLM_PARALLEL"] = "1"
if model_val and Path(model_val).exists():
env["LLM_MODEL"] = model_val
tmpl = Path(model_val).parent / "chat_template.jinja"
if tmpl.exists():
env["LLM_TEMPLATE"] = str(tmpl)
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
env["LLM_MMPROJ"] = mmproj_val
else:
env["LLM_MMPROJ"] = "none"
runner_cmd = PRESETS.get(preset, {}).get("command", preset)
cmd = [str(RUNNER_BIN), runner_cmd]
log_widget.write(
f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n"
f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n"
f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n"
f" Port: {env['LLM_PORT']} | Context: {env['LLM_CTX']} | KV Quant: {env['LLM_CTK']}\n"
)
runner_log = open(LOGS_DIR / "runner.log", "a")
subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), stdout=runner_log, stderr=runner_log, start_new_session=True)
self.notify(f"Launching {preset} server. Switching to Status & Logs...", severity="information")
self.query_one(TabbedContent).active = "tab-status"
self.action_refresh_status()
def action_stop(self) -> None: