noelbautista91/llms
1
0
Fork 0
Code Activity Actions

feat(orchestrator): prefix all LLM runner env vars with LLM_

Browse source
This commit is contained in:
Noel Bautista 2026-10-21 14:28:33 -07:00
parent b753f48c2b
commit fe6ea64128
No known key found for this signature in database
2 changed files with 93 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

143
bin/llm-runner
View file

@ -18,7 +18,7 @@ if [ -n "${TMP:-}" ] && [ ! -d "${TMP}" ]; then
fi fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}" LOG_DIR="${LLM_LOG_DIR:-${LOG_DIR:-${SCRIPT_DIR}/logs}}"
mkdir -p "${LOG_DIR}" mkdir -p "${LOG_DIR}"
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -136,10 +136,10 @@ launch_instance() {
local flags=( local flags=(
-m "${model}" -m "${model}"
--host "${HOST:-0.0.0.0}" --host "${LLM_HOST:-${HOST:-0.0.0.0}}"
--port "${port}" --port "${port}"
--ctx-size "${ctx}" --ctx-size "${ctx}"
-np "${PARALLEL:-1}" -np "${LLM_PARALLEL:-${PARALLEL:-1}}"
--n-gpu-layers "${ngl}" --n-gpu-layers "${ngl}"
--load-mode "${load_mode}" --load-mode "${load_mode}"
--temp "${temp}" --temp "${temp}"
@ -149,12 +149,12 @@ launch_instance() {
-ctk "${ctk}" -ctk "${ctk}"
-ctv "${ctv}" -ctv "${ctv}"
-fa "${flash_attn}" -fa "${flash_attn}"
-b "${BATCH_SIZE:-2048}" -b "${LLM_BATCH_SIZE:-${BATCH_SIZE:-2048}}"
-ub "${UBATCH_SIZE:-512}" -ub "${LLM_UBATCH_SIZE:-${UBATCH_SIZE:-512}}"
) )
if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then
flags+=(--mmproj "${mmproj}" --image-min-tokens "${IMAGE_MIN_TOKENS:-1024}") flags+=(--mmproj "${mmproj}" --image-min-tokens "${LLM_IMAGE_MIN_TOKENS:-${IMAGE_MIN_TOKENS:-1024}}")
fi fi
if [ -n "${template}" ] && [ -f "${template}" ]; then if [ -n "${template}" ] && [ -f "${template}" ]; then
@ -205,43 +205,43 @@ cmd_cluster() {
trap cleanup_cluster SIGINT SIGTERM EXIT trap cleanup_cluster SIGINT SIGTERM EXIT
# SuperQwen 3.8 27B (Planner) # SuperQwen 3.8 27B (Planner)
local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}" local planner_model="${LLM_PLANNER_MODEL:-${PLANNER_MODEL:-${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}}}"
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
fi fi
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" local planner_mmproj="${LLM_PLANNER_MMPROJ:-${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" local planner_tmpl="${LLM_PLANNER_TEMPLATE:-${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
# Qwen 3.5 4B (Workers) # Qwen 3.5 4B (Workers)
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local worker_model="${LLM_WORKER_MODEL:-${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" local worker_mmproj="${LLM_WORKER_MMPROJ:-${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" local worker_tmpl="${LLM_WORKER_TEMPLATE:-${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
# 1. Planner on 11434 # 1. Planner on 11434
launch_instance "planner_27b" "$planner_model" "${PLANNER_PORT:-11434}" "${PLANNER_CTX:-65536}" \ launch_instance "planner_27b" "$planner_model" "${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}" "${LLM_PLANNER_CTX:-${PLANNER_CTX:-65536}}" \
"$planner_mmproj" "$planner_tmpl" "${PLANNER_TEMP:-0.6}" "${PLANNER_TOP_P:-0.95}" "${PLANNER_TOP_K:-40}" \ "$planner_mmproj" "$planner_tmpl" "${LLM_PLANNER_TEMP:-${PLANNER_TEMP:-0.6}}" "${LLM_PLANNER_TOP_P:-${PLANNER_TOP_P:-0.95}}" "${LLM_PLANNER_TOP_K:-${PLANNER_TOP_K:-40}}" \
"${PLANNER_MIN_P:-0.05}" "${PLANNER_NGL:-99}" "${PLANNER_LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${LLM_PLANNER_MIN_P:-${PLANNER_MIN_P:-0.05}}" "${LLM_PLANNER_NGL:-${PLANNER_NGL:-99}}" "${LLM_PLANNER_LOAD_MODE:-${PLANNER_LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${PLANNER_CTK:-q8_0}" "${PLANNER_CTV:-q8_0}" "false" "${LLM_PLANNER_CTK:-${PLANNER_CTK:-q8_0}}" "${LLM_PLANNER_CTV:-${PLANNER_CTV:-q8_0}}" "false"
# 2. Worker 1 on 11435 # 2. Worker 1 on 11435
launch_instance "worker_4b_1" "$worker_model" "${WORKER1_PORT:-11435}" "${WORKER1_CTX:-16384}" \ launch_instance "worker_4b_1" "$worker_model" "${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}" "${LLM_WORKER1_CTX:-${WORKER1_CTX:-16384}}" \
"$worker_mmproj" "$worker_tmpl" "${WORKER1_TEMP:-0.7}" "${WORKER1_TOP_P:-0.8}" "${WORKER1_TOP_K:-20}" \ "$worker_mmproj" "$worker_tmpl" "${LLM_WORKER1_TEMP:-${WORKER1_TEMP:-0.7}}" "${LLM_WORKER1_TOP_P:-${WORKER1_TOP_P:-0.8}}" "${LLM_WORKER1_TOP_K:-${WORKER1_TOP_K:-20}}" \
"${WORKER1_MIN_P:-0.05}" "${WORKER1_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ "${LLM_WORKER1_MIN_P:-${WORKER1_MIN_P:-0.05}}" "${LLM_WORKER1_NGL:-${WORKER1_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${WORKER1_CTK:-q8_0}" "${WORKER1_CTV:-q8_0}" "false" "${LLM_WORKER1_CTK:-${WORKER1_CTK:-q8_0}}" "${LLM_WORKER1_CTV:-${WORKER1_CTV:-q8_0}}" "false"
# 3. Worker 2 on 11436 # 3. Worker 2 on 11436
launch_instance "worker_4b_2" "$worker_model" "${WORKER2_PORT:-11436}" "${WORKER2_CTX:-16384}" \ launch_instance "worker_4b_2" "$worker_model" "${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}" "${LLM_WORKER2_CTX:-${WORKER2_CTX:-16384}}" \
"$worker_mmproj" "$worker_tmpl" "${WORKER2_TEMP:-0.7}" "${WORKER2_TOP_P:-0.8}" "${WORKER2_TOP_K:-20}" \ "$worker_mmproj" "$worker_tmpl" "${LLM_WORKER2_TEMP:-${WORKER2_TEMP:-0.7}}" "${LLM_WORKER2_TOP_P:-${WORKER2_TOP_P:-0.8}}" "${LLM_WORKER2_TOP_K:-${WORKER2_TOP_K:-20}}" \
"${WORKER2_MIN_P:-0.05}" "${WORKER2_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ "${LLM_WORKER2_MIN_P:-${WORKER2_MIN_P:-0.05}}" "${LLM_WORKER2_NGL:-${WORKER2_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${WORKER2_CTK:-q8_0}" "${WORKER2_CTV:-q8_0}" "false" "${LLM_WORKER2_CTK:-${WORKER2_CTK:-q8_0}}" "${LLM_WORKER2_CTV:-${WORKER2_CTV:-q8_0}}" "false"
echo "" echo ""
log_success "All 3 cluster servers are operational:" log_success "All 3 cluster servers are operational:"
echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${PLANNER_PORT:-11434}" echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}"
echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${WORKER1_PORT:-11435}" echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}"
echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${WORKER2_PORT:-11436}" echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}"
echo "" echo ""
echo "Logs directory: ${LOG_DIR}/" echo "Logs directory: ${LOG_DIR}/"
echo "Press Ctrl+C to stop all instances gracefully." echo "Press Ctrl+C to stop all instances gracefully."
@ -252,15 +252,18 @@ cmd_muse() {
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..." log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
pkill -f "llama-server" 2>/dev/null || true pkill -f "llama-server" 2>/dev/null || true
sleep 1 sleep 1
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}" local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}" local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}}"
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ launch_instance "muse_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-131072}}" \
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
echo "" echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log" echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log"
@ -272,18 +275,21 @@ cmd_superqwen() {
log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..." log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..."
pkill -f "llama-server" 2>/dev/null || true pkill -f "llama-server" 2>/dev/null || true
sleep 1 sleep 1
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}" local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}}"
if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
fi fi
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ launch_instance "superqwen_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-262144}}" \
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
echo "" echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log" echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log"
@ -295,15 +301,17 @@ cmd_qwen4b() {
log_info "Starting Qwen 3.5 4B Solo Server..." log_info "Starting Qwen 3.5 4B Solo Server..."
pkill -f "llama-server" 2>/dev/null || true pkill -f "llama-server" 2>/dev/null || true
sleep 1 sleep 1
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ launch_instance "qwen4b_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-32768}}" \
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}" "${TOP_P:-0.8}" "${TOP_K:-20}" \ "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ "${LLM_TOP_P:-${TOP_P:-0.8}}" "${LLM_TOP_K:-${TOP_K:-20}}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "false" "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" "false"
echo "" echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log" echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log"
@ -315,20 +323,23 @@ cmd_custom() {
log_info "Starting Custom Server Instance..." log_info "Starting Custom Server Instance..."
pkill -f "llama-server" 2>/dev/null || true pkill -f "llama-server" 2>/dev/null || true
sleep 1 sleep 1
local model="${MODEL:-}" local model="${LLM_MODEL:-${MODEL:-}}"
if [ -z "${model}" ] || [ ! -f "${model}" ]; then if [ -z "${model}" ] || [ ! -f "${model}" ]; then
log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf" log_error "No valid MODEL specified! Pass LLM_MODEL=/path/to/model.gguf"
exit 1 exit 1
fi fi
local mmproj="${MMPROJ:-}" local mmproj="${LLM_MMPROJ:-${MMPROJ:-}}"
if [ "${mmproj}" = "none" ]; then mmproj=""; fi if [ "${mmproj}" = "none" ]; then mmproj=""; fi
local tmpl="${TEMPLATE:-}" local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-}}"
local name="custom_solo" local name="custom_solo"
launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \ launch_instance "${name}" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-65536}}" \
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}" "${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-false}}"
echo "" echo ""
echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log" echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log"
@ -348,20 +359,22 @@ Commands:
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434) muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434) superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
qwen4b Launch Qwen 3.5 4B Solo (port 11434) qwen4b Launch Qwen 3.5 4B Solo (port 11434)
custom Launch custom model instance (requires MODEL=/path/to/model.gguf) custom Launch custom model instance (requires LLM_MODEL=/path/to/model.gguf)
status Show current server health and active ports status Show current server health and active ports
stop [target] Stop servers (target: "all" or port number like 11434) stop [target] Stop servers (target: "all" or port number like 11434)
help Display this help message help Display this help message
Environment Variables: Environment Variables:
MODEL Path to GGUF model file LLM_MODEL Path to GGUF model file (fallback: MODEL)
MMPROJ Path to mmproj file (or "none" for text-only) LLM_MMPROJ Path to mmproj file, or "none" (fallback: MMPROJ)
TEMPLATE Path to chat template file LLM_TEMPLATE Path to chat template file (fallback: TEMPLATE)
HOST Bind address (default: 0.0.0.0) LLM_HOST Bind address (default: 0.0.0.0)
PORT Main port override (default: 11434) LLM_PORT Main port override (default: 11434)
CTX Context size override LLM_CTX Context size override (default: preset-specific)
FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on) LLM_TEMPERATURE Sampling temperature (default: preset-specific)
LOG_DIR Path to write logs (default: ./logs) LLM_PARALLEL Parallel slot count (default: 1)
LLM_FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on)
LLM_LOG_DIR Path to write logs (default: ./logs)
EOF EOF
} }

30
bin/llm-tui
View file

@ -729,27 +729,27 @@ class LLMConfigApp(App):
env.pop("TEMP", None) env.pop("TEMP", None)
env.pop("TMP", None) env.pop("TMP", None)
env["TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6" env["LLM_TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6"
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" env["LLM_CTX"] = self.query_one("#input-ctx", Input).value or "65536"
env["PORT"] = self.query_one("#input-port", Input).value or "11434" env["LLM_PORT"] = self.query_one("#input-port", Input).value or "11434"
env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95" env["LLM_TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95"
env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05" env["LLM_MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05"
env["NGL"] = self.query_one("#input-ngl", Input).value or "99" env["LLM_NGL"] = self.query_one("#input-ngl", Input).value or "99"
env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off" env["LLM_FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") env["LLM_CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
env["CTV"] = env["CTK"] env["LLM_CTV"] = env["LLM_CTK"]
env["PARALLEL"] = "1" env["LLM_PARALLEL"] = "1"
if model_val and Path(model_val).exists(): if model_val and Path(model_val).exists():
env["MODEL"] = model_val env["LLM_MODEL"] = model_val
tmpl = Path(model_val).parent / "chat_template.jinja" tmpl = Path(model_val).parent / "chat_template.jinja"
if tmpl.exists(): if tmpl.exists():
env["TEMPLATE"] = str(tmpl) env["LLM_TEMPLATE"] = str(tmpl)
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists(): if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
env["MMPROJ"] = mmproj_val env["LLM_MMPROJ"] = mmproj_val
else: else:
env["MMPROJ"] = "none" env["LLM_MMPROJ"] = "none"
runner_cmd = PRESETS.get(preset, {}).get("command", preset) runner_cmd = PRESETS.get(preset, {}).get("command", preset)
cmd = [str(RUNNER_BIN), runner_cmd] cmd = [str(RUNNER_BIN), runner_cmd]
@ -758,7 +758,7 @@ class LLMConfigApp(App):
f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n" f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n"
f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n" f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n"
f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n" f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n"
f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n" f" Port: {env['LLM_PORT']} | Context: {env['LLM_CTX']} | KV Quant: {env['LLM_CTK']}\n"
) )
runner_log = open(LOGS_DIR / "runner.log", "a") runner_log = open(LOGS_DIR / "runner.log", "a")