diff --git a/bin/llm-runner b/bin/llm-runner index 8bf8be0..185020c 100755 --- a/bin/llm-runner +++ b/bin/llm-runner @@ -18,7 +18,7 @@ if [ -n "${TMP:-}" ] && [ ! -d "${TMP}" ]; then fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}" +LOG_DIR="${LLM_LOG_DIR:-${LOG_DIR:-${SCRIPT_DIR}/logs}}" mkdir -p "${LOG_DIR}" # ------------------------------------------------------------------------------ @@ -136,10 +136,10 @@ launch_instance() { local flags=( -m "${model}" - --host "${HOST:-0.0.0.0}" + --host "${LLM_HOST:-${HOST:-0.0.0.0}}" --port "${port}" --ctx-size "${ctx}" - -np "${PARALLEL:-1}" + -np "${LLM_PARALLEL:-${PARALLEL:-1}}" --n-gpu-layers "${ngl}" --load-mode "${load_mode}" --temp "${temp}" @@ -149,12 +149,12 @@ launch_instance() { -ctk "${ctk}" -ctv "${ctv}" -fa "${flash_attn}" - -b "${BATCH_SIZE:-2048}" - -ub "${UBATCH_SIZE:-512}" + -b "${LLM_BATCH_SIZE:-${BATCH_SIZE:-2048}}" + -ub "${LLM_UBATCH_SIZE:-${UBATCH_SIZE:-512}}" ) if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then - flags+=(--mmproj "${mmproj}" --image-min-tokens "${IMAGE_MIN_TOKENS:-1024}") + flags+=(--mmproj "${mmproj}" --image-min-tokens "${LLM_IMAGE_MIN_TOKENS:-${IMAGE_MIN_TOKENS:-1024}}") fi if [ -n "${template}" ] && [ -f "${template}" ]; then @@ -205,43 +205,43 @@ cmd_cluster() { trap cleanup_cluster SIGINT SIGTERM EXIT # SuperQwen 3.8 27B (Planner) - local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}" + local planner_model="${LLM_PLANNER_MODEL:-${PLANNER_MODEL:-${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}}}" if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" fi - local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + local planner_mmproj="${LLM_PLANNER_MMPROJ:-${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}" if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi - local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" + local planner_tmpl="${LLM_PLANNER_TEMPLATE:-${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}" # Qwen 3.5 4B (Workers) - local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" - local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + local worker_model="${LLM_WORKER_MODEL:-${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}" + local worker_mmproj="${LLM_WORKER_MMPROJ:-${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}" if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi - local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" + local worker_tmpl="${LLM_WORKER_TEMPLATE:-${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}" # 1. Planner on 11434 - launch_instance "planner_27b" "$planner_model" "${PLANNER_PORT:-11434}" "${PLANNER_CTX:-65536}" \ - "$planner_mmproj" "$planner_tmpl" "${PLANNER_TEMP:-0.6}" "${PLANNER_TOP_P:-0.95}" "${PLANNER_TOP_K:-40}" \ - "${PLANNER_MIN_P:-0.05}" "${PLANNER_NGL:-99}" "${PLANNER_LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ - "${PLANNER_CTK:-q8_0}" "${PLANNER_CTV:-q8_0}" "false" + launch_instance "planner_27b" "$planner_model" "${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}" "${LLM_PLANNER_CTX:-${PLANNER_CTX:-65536}}" \ + "$planner_mmproj" "$planner_tmpl" "${LLM_PLANNER_TEMP:-${PLANNER_TEMP:-0.6}}" "${LLM_PLANNER_TOP_P:-${PLANNER_TOP_P:-0.95}}" "${LLM_PLANNER_TOP_K:-${PLANNER_TOP_K:-40}}" \ + "${LLM_PLANNER_MIN_P:-${PLANNER_MIN_P:-0.05}}" "${LLM_PLANNER_NGL:-${PLANNER_NGL:-99}}" "${LLM_PLANNER_LOAD_MODE:-${PLANNER_LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_PLANNER_CTK:-${PLANNER_CTK:-q8_0}}" "${LLM_PLANNER_CTV:-${PLANNER_CTV:-q8_0}}" "false" # 2. Worker 1 on 11435 - launch_instance "worker_4b_1" "$worker_model" "${WORKER1_PORT:-11435}" "${WORKER1_CTX:-16384}" \ - "$worker_mmproj" "$worker_tmpl" "${WORKER1_TEMP:-0.7}" "${WORKER1_TOP_P:-0.8}" "${WORKER1_TOP_K:-20}" \ - "${WORKER1_MIN_P:-0.05}" "${WORKER1_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ - "${WORKER1_CTK:-q8_0}" "${WORKER1_CTV:-q8_0}" "false" + launch_instance "worker_4b_1" "$worker_model" "${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}" "${LLM_WORKER1_CTX:-${WORKER1_CTX:-16384}}" \ + "$worker_mmproj" "$worker_tmpl" "${LLM_WORKER1_TEMP:-${WORKER1_TEMP:-0.7}}" "${LLM_WORKER1_TOP_P:-${WORKER1_TOP_P:-0.8}}" "${LLM_WORKER1_TOP_K:-${WORKER1_TOP_K:-20}}" \ + "${LLM_WORKER1_MIN_P:-${WORKER1_MIN_P:-0.05}}" "${LLM_WORKER1_NGL:-${WORKER1_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_WORKER1_CTK:-${WORKER1_CTK:-q8_0}}" "${LLM_WORKER1_CTV:-${WORKER1_CTV:-q8_0}}" "false" # 3. Worker 2 on 11436 - launch_instance "worker_4b_2" "$worker_model" "${WORKER2_PORT:-11436}" "${WORKER2_CTX:-16384}" \ - "$worker_mmproj" "$worker_tmpl" "${WORKER2_TEMP:-0.7}" "${WORKER2_TOP_P:-0.8}" "${WORKER2_TOP_K:-20}" \ - "${WORKER2_MIN_P:-0.05}" "${WORKER2_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ - "${WORKER2_CTK:-q8_0}" "${WORKER2_CTV:-q8_0}" "false" + launch_instance "worker_4b_2" "$worker_model" "${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}" "${LLM_WORKER2_CTX:-${WORKER2_CTX:-16384}}" \ + "$worker_mmproj" "$worker_tmpl" "${LLM_WORKER2_TEMP:-${WORKER2_TEMP:-0.7}}" "${LLM_WORKER2_TOP_P:-${WORKER2_TOP_P:-0.8}}" "${LLM_WORKER2_TOP_K:-${WORKER2_TOP_K:-20}}" \ + "${LLM_WORKER2_MIN_P:-${WORKER2_MIN_P:-0.05}}" "${LLM_WORKER2_NGL:-${WORKER2_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_WORKER2_CTK:-${WORKER2_CTK:-q8_0}}" "${LLM_WORKER2_CTV:-${WORKER2_CTV:-q8_0}}" "false" echo "" log_success "All 3 cluster servers are operational:" - echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${PLANNER_PORT:-11434}" - echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${WORKER1_PORT:-11435}" - echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${WORKER2_PORT:-11436}" + echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}" + echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}" + echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}" echo "" echo "Logs directory: ${LOG_DIR}/" echo "Press Ctrl+C to stop all instances gracefully." @@ -252,15 +252,18 @@ cmd_muse() { log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..." pkill -f "llama-server" 2>/dev/null || true sleep 1 - local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}" - local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}" + local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}}" + local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}}" if [ "${mmproj}" = "none" ]; then mmproj=""; fi - local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" + local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}}" - launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ - "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ - "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ - "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" + launch_instance "muse_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-131072}}" \ + "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \ + "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \ + "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \ + "${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \ + "${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}" echo "" echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log" @@ -272,18 +275,21 @@ cmd_superqwen() { log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..." pkill -f "llama-server" 2>/dev/null || true sleep 1 - local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}" + local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}}" if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" fi - local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}" if [ "${mmproj}" = "none" ]; then mmproj=""; fi - local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" + local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}" - launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ - "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ - "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ - "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" + launch_instance "superqwen_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-262144}}" \ + "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \ + "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \ + "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \ + "${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \ + "${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}" echo "" echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log" @@ -295,15 +301,17 @@ cmd_qwen4b() { log_info "Starting Qwen 3.5 4B Solo Server..." pkill -f "llama-server" 2>/dev/null || true sleep 1 - local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" - local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}" + local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}" if [ "${mmproj}" = "none" ]; then mmproj=""; fi - local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" + local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}" - launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ - "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}" "${TOP_P:-0.8}" "${TOP_K:-20}" \ - "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ - "${CTK:-q8_0}" "${CTV:-q8_0}" "false" + launch_instance "qwen4b_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-32768}}" \ + "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}}" \ + "${LLM_TOP_P:-${TOP_P:-0.8}}" "${LLM_TOP_K:-${TOP_K:-20}}" \ + "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \ + "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" "false" echo "" echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log" @@ -315,20 +323,23 @@ cmd_custom() { log_info "Starting Custom Server Instance..." pkill -f "llama-server" 2>/dev/null || true sleep 1 - local model="${MODEL:-}" + local model="${LLM_MODEL:-${MODEL:-}}" if [ -z "${model}" ] || [ ! -f "${model}" ]; then - log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf" + log_error "No valid MODEL specified! Pass LLM_MODEL=/path/to/model.gguf" exit 1 fi - local mmproj="${MMPROJ:-}" + local mmproj="${LLM_MMPROJ:-${MMPROJ:-}}" if [ "${mmproj}" = "none" ]; then mmproj=""; fi - local tmpl="${TEMPLATE:-}" + local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-}}" local name="custom_solo" - launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \ - "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ - "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ - "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}" + launch_instance "${name}" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-65536}}" \ + "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \ + "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \ + "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \ + "${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \ + "${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \ + "${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-false}}" echo "" echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log" @@ -348,20 +359,22 @@ Commands: muse Launch Muse-Glimmer 30B Solo (128K context, port 11434) superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434) qwen4b Launch Qwen 3.5 4B Solo (port 11434) - custom Launch custom model instance (requires MODEL=/path/to/model.gguf) + custom Launch custom model instance (requires LLM_MODEL=/path/to/model.gguf) status Show current server health and active ports stop [target] Stop servers (target: "all" or port number like 11434) help Display this help message Environment Variables: - MODEL Path to GGUF model file - MMPROJ Path to mmproj file (or "none" for text-only) - TEMPLATE Path to chat template file - HOST Bind address (default: 0.0.0.0) - PORT Main port override (default: 11434) - CTX Context size override - FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on) - LOG_DIR Path to write logs (default: ./logs) + LLM_MODEL Path to GGUF model file (fallback: MODEL) + LLM_MMPROJ Path to mmproj file, or "none" (fallback: MMPROJ) + LLM_TEMPLATE Path to chat template file (fallback: TEMPLATE) + LLM_HOST Bind address (default: 0.0.0.0) + LLM_PORT Main port override (default: 11434) + LLM_CTX Context size override (default: preset-specific) + LLM_TEMPERATURE Sampling temperature (default: preset-specific) + LLM_PARALLEL Parallel slot count (default: 1) + LLM_FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on) + LLM_LOG_DIR Path to write logs (default: ./logs) EOF } diff --git a/bin/llm-tui b/bin/llm-tui index 169f048..18c97be 100755 --- a/bin/llm-tui +++ b/bin/llm-tui @@ -729,27 +729,27 @@ class LLMConfigApp(App): env.pop("TEMP", None) env.pop("TMP", None) - env["TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6" - env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" - env["PORT"] = self.query_one("#input-port", Input).value or "11434" - env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95" - env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05" - env["NGL"] = self.query_one("#input-ngl", Input).value or "99" - env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off" - env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") - env["CTV"] = env["CTK"] - env["PARALLEL"] = "1" + env["LLM_TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6" + env["LLM_CTX"] = self.query_one("#input-ctx", Input).value or "65536" + env["LLM_PORT"] = self.query_one("#input-port", Input).value or "11434" + env["LLM_TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95" + env["LLM_MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05" + env["LLM_NGL"] = self.query_one("#input-ngl", Input).value or "99" + env["LLM_FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off" + env["LLM_CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") + env["LLM_CTV"] = env["LLM_CTK"] + env["LLM_PARALLEL"] = "1" if model_val and Path(model_val).exists(): - env["MODEL"] = model_val + env["LLM_MODEL"] = model_val tmpl = Path(model_val).parent / "chat_template.jinja" if tmpl.exists(): - env["TEMPLATE"] = str(tmpl) + env["LLM_TEMPLATE"] = str(tmpl) if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists(): - env["MMPROJ"] = mmproj_val + env["LLM_MMPROJ"] = mmproj_val else: - env["MMPROJ"] = "none" + env["LLM_MMPROJ"] = "none" runner_cmd = PRESETS.get(preset, {}).get("command", preset) cmd = [str(RUNNER_BIN), runner_cmd] @@ -758,7 +758,7 @@ class LLMConfigApp(App): f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n" f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n" f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n" - f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n" + f" Port: {env['LLM_PORT']} | Context: {env['LLM_CTX']} | KV Quant: {env['LLM_CTK']}\n" ) runner_log = open(LOGS_DIR / "runner.log", "a")