feat(orchestrator): prefix all LLM runner env vars with LLM_
This commit is contained in:
parent
b753f48c2b
commit
fe6ea64128
2 changed files with 93 additions and 80 deletions
143
bin/llm-runner
143
bin/llm-runner
|
|
@ -18,7 +18,7 @@ if [ -n "${TMP:-}" ] && [ ! -d "${TMP}" ]; then
|
|||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}"
|
||||
LOG_DIR="${LLM_LOG_DIR:-${LOG_DIR:-${SCRIPT_DIR}/logs}}"
|
||||
mkdir -p "${LOG_DIR}"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -136,10 +136,10 @@ launch_instance() {
|
|||
|
||||
local flags=(
|
||||
-m "${model}"
|
||||
--host "${HOST:-0.0.0.0}"
|
||||
--host "${LLM_HOST:-${HOST:-0.0.0.0}}"
|
||||
--port "${port}"
|
||||
--ctx-size "${ctx}"
|
||||
-np "${PARALLEL:-1}"
|
||||
-np "${LLM_PARALLEL:-${PARALLEL:-1}}"
|
||||
--n-gpu-layers "${ngl}"
|
||||
--load-mode "${load_mode}"
|
||||
--temp "${temp}"
|
||||
|
|
@ -149,12 +149,12 @@ launch_instance() {
|
|||
-ctk "${ctk}"
|
||||
-ctv "${ctv}"
|
||||
-fa "${flash_attn}"
|
||||
-b "${BATCH_SIZE:-2048}"
|
||||
-ub "${UBATCH_SIZE:-512}"
|
||||
-b "${LLM_BATCH_SIZE:-${BATCH_SIZE:-2048}}"
|
||||
-ub "${LLM_UBATCH_SIZE:-${UBATCH_SIZE:-512}}"
|
||||
)
|
||||
|
||||
if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then
|
||||
flags+=(--mmproj "${mmproj}" --image-min-tokens "${IMAGE_MIN_TOKENS:-1024}")
|
||||
flags+=(--mmproj "${mmproj}" --image-min-tokens "${LLM_IMAGE_MIN_TOKENS:-${IMAGE_MIN_TOKENS:-1024}}")
|
||||
fi
|
||||
|
||||
if [ -n "${template}" ] && [ -f "${template}" ]; then
|
||||
|
|
@ -205,43 +205,43 @@ cmd_cluster() {
|
|||
trap cleanup_cluster SIGINT SIGTERM EXIT
|
||||
|
||||
# SuperQwen 3.8 27B (Planner)
|
||||
local planner_model="${PLANNER_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}"
|
||||
local planner_model="${LLM_PLANNER_MODEL:-${PLANNER_MODEL:-${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}}}}"
|
||||
if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then
|
||||
planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf"
|
||||
fi
|
||||
local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
||||
local planner_mmproj="${LLM_PLANNER_MMPROJ:-${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
|
||||
if [ "${planner_mmproj}" = "none" ]; then planner_mmproj=""; fi
|
||||
local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
||||
local planner_tmpl="${LLM_PLANNER_TEMPLATE:-${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
|
||||
|
||||
# Qwen 3.5 4B (Workers)
|
||||
local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
||||
local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
||||
local worker_model="${LLM_WORKER_MODEL:-${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
|
||||
local worker_mmproj="${LLM_WORKER_MMPROJ:-${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
|
||||
if [ "${worker_mmproj}" = "none" ]; then worker_mmproj=""; fi
|
||||
local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
||||
local worker_tmpl="${LLM_WORKER_TEMPLATE:-${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
|
||||
|
||||
# 1. Planner on 11434
|
||||
launch_instance "planner_27b" "$planner_model" "${PLANNER_PORT:-11434}" "${PLANNER_CTX:-65536}" \
|
||||
"$planner_mmproj" "$planner_tmpl" "${PLANNER_TEMP:-0.6}" "${PLANNER_TOP_P:-0.95}" "${PLANNER_TOP_K:-40}" \
|
||||
"${PLANNER_MIN_P:-0.05}" "${PLANNER_NGL:-99}" "${PLANNER_LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
|
||||
"${PLANNER_CTK:-q8_0}" "${PLANNER_CTV:-q8_0}" "false"
|
||||
launch_instance "planner_27b" "$planner_model" "${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}" "${LLM_PLANNER_CTX:-${PLANNER_CTX:-65536}}" \
|
||||
"$planner_mmproj" "$planner_tmpl" "${LLM_PLANNER_TEMP:-${PLANNER_TEMP:-0.6}}" "${LLM_PLANNER_TOP_P:-${PLANNER_TOP_P:-0.95}}" "${LLM_PLANNER_TOP_K:-${PLANNER_TOP_K:-40}}" \
|
||||
"${LLM_PLANNER_MIN_P:-${PLANNER_MIN_P:-0.05}}" "${LLM_PLANNER_NGL:-${PLANNER_NGL:-99}}" "${LLM_PLANNER_LOAD_MODE:-${PLANNER_LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_PLANNER_CTK:-${PLANNER_CTK:-q8_0}}" "${LLM_PLANNER_CTV:-${PLANNER_CTV:-q8_0}}" "false"
|
||||
|
||||
# 2. Worker 1 on 11435
|
||||
launch_instance "worker_4b_1" "$worker_model" "${WORKER1_PORT:-11435}" "${WORKER1_CTX:-16384}" \
|
||||
"$worker_mmproj" "$worker_tmpl" "${WORKER1_TEMP:-0.7}" "${WORKER1_TOP_P:-0.8}" "${WORKER1_TOP_K:-20}" \
|
||||
"${WORKER1_MIN_P:-0.05}" "${WORKER1_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
|
||||
"${WORKER1_CTK:-q8_0}" "${WORKER1_CTV:-q8_0}" "false"
|
||||
launch_instance "worker_4b_1" "$worker_model" "${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}" "${LLM_WORKER1_CTX:-${WORKER1_CTX:-16384}}" \
|
||||
"$worker_mmproj" "$worker_tmpl" "${LLM_WORKER1_TEMP:-${WORKER1_TEMP:-0.7}}" "${LLM_WORKER1_TOP_P:-${WORKER1_TOP_P:-0.8}}" "${LLM_WORKER1_TOP_K:-${WORKER1_TOP_K:-20}}" \
|
||||
"${LLM_WORKER1_MIN_P:-${WORKER1_MIN_P:-0.05}}" "${LLM_WORKER1_NGL:-${WORKER1_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_WORKER1_CTK:-${WORKER1_CTK:-q8_0}}" "${LLM_WORKER1_CTV:-${WORKER1_CTV:-q8_0}}" "false"
|
||||
|
||||
# 3. Worker 2 on 11436
|
||||
launch_instance "worker_4b_2" "$worker_model" "${WORKER2_PORT:-11436}" "${WORKER2_CTX:-16384}" \
|
||||
"$worker_mmproj" "$worker_tmpl" "${WORKER2_TEMP:-0.7}" "${WORKER2_TOP_P:-0.8}" "${WORKER2_TOP_K:-20}" \
|
||||
"${WORKER2_MIN_P:-0.05}" "${WORKER2_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
|
||||
"${WORKER2_CTK:-q8_0}" "${WORKER2_CTV:-q8_0}" "false"
|
||||
launch_instance "worker_4b_2" "$worker_model" "${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}" "${LLM_WORKER2_CTX:-${WORKER2_CTX:-16384}}" \
|
||||
"$worker_mmproj" "$worker_tmpl" "${LLM_WORKER2_TEMP:-${WORKER2_TEMP:-0.7}}" "${LLM_WORKER2_TOP_P:-${WORKER2_TOP_P:-0.8}}" "${LLM_WORKER2_TOP_K:-${WORKER2_TOP_K:-20}}" \
|
||||
"${LLM_WORKER2_MIN_P:-${WORKER2_MIN_P:-0.05}}" "${LLM_WORKER2_NGL:-${WORKER2_NGL:-99}}" "${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_WORKER2_CTK:-${WORKER2_CTK:-q8_0}}" "${LLM_WORKER2_CTV:-${WORKER2_CTV:-q8_0}}" "false"
|
||||
|
||||
echo ""
|
||||
log_success "All 3 cluster servers are operational:"
|
||||
echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${PLANNER_PORT:-11434}"
|
||||
echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${WORKER1_PORT:-11435}"
|
||||
echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${WORKER2_PORT:-11436}"
|
||||
echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${LLM_PLANNER_PORT:-${PLANNER_PORT:-11434}}"
|
||||
echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${LLM_WORKER1_PORT:-${WORKER1_PORT:-11435}}"
|
||||
echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${LLM_WORKER2_PORT:-${WORKER2_PORT:-11436}}"
|
||||
echo ""
|
||||
echo "Logs directory: ${LOG_DIR}/"
|
||||
echo "Press Ctrl+C to stop all instances gracefully."
|
||||
|
|
@ -252,15 +252,18 @@ cmd_muse() {
|
|||
log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..."
|
||||
pkill -f "llama-server" 2>/dev/null || true
|
||||
sleep 1
|
||||
local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}"
|
||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}"
|
||||
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}}"
|
||||
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}}"
|
||||
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}"
|
||||
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}}"
|
||||
|
||||
launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \
|
||||
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
|
||||
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
|
||||
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}"
|
||||
launch_instance "muse_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-131072}}" \
|
||||
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
|
||||
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
|
||||
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
|
||||
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
|
||||
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
|
||||
|
||||
echo ""
|
||||
echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log"
|
||||
|
|
@ -272,18 +275,21 @@ cmd_superqwen() {
|
|||
log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..."
|
||||
pkill -f "llama-server" 2>/dev/null || true
|
||||
sleep 1
|
||||
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}"
|
||||
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}}"
|
||||
if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then
|
||||
model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf"
|
||||
fi
|
||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}"
|
||||
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}}"
|
||||
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}"
|
||||
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}}"
|
||||
|
||||
launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \
|
||||
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
|
||||
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
|
||||
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}"
|
||||
launch_instance "superqwen_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-262144}}" \
|
||||
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
|
||||
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
|
||||
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
|
||||
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
|
||||
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-true}}"
|
||||
|
||||
echo ""
|
||||
echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log"
|
||||
|
|
@ -295,15 +301,17 @@ cmd_qwen4b() {
|
|||
log_info "Starting Qwen 3.5 4B Solo Server..."
|
||||
pkill -f "llama-server" 2>/dev/null || true
|
||||
sleep 1
|
||||
local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}"
|
||||
local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}"
|
||||
local model="${LLM_MODEL:-${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}}"
|
||||
local mmproj="${LLM_MMPROJ:-${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}}"
|
||||
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||
local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}"
|
||||
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}}"
|
||||
|
||||
launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \
|
||||
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}" "${TOP_P:-0.8}" "${TOP_K:-20}" \
|
||||
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \
|
||||
"${CTK:-q8_0}" "${CTV:-q8_0}" "false"
|
||||
launch_instance "qwen4b_solo" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-32768}}" \
|
||||
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}}" \
|
||||
"${LLM_TOP_P:-${TOP_P:-0.8}}" "${LLM_TOP_K:-${TOP_K:-20}}" \
|
||||
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
|
||||
"${LLM_LOAD_MODE:-${LOAD_MODE:-mmap}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" "false"
|
||||
|
||||
echo ""
|
||||
echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log"
|
||||
|
|
@ -315,20 +323,23 @@ cmd_custom() {
|
|||
log_info "Starting Custom Server Instance..."
|
||||
pkill -f "llama-server" 2>/dev/null || true
|
||||
sleep 1
|
||||
local model="${MODEL:-}"
|
||||
local model="${LLM_MODEL:-${MODEL:-}}"
|
||||
if [ -z "${model}" ] || [ ! -f "${model}" ]; then
|
||||
log_error "No valid MODEL specified! Pass MODEL=/path/to/model.gguf"
|
||||
log_error "No valid MODEL specified! Pass LLM_MODEL=/path/to/model.gguf"
|
||||
exit 1
|
||||
fi
|
||||
local mmproj="${MMPROJ:-}"
|
||||
local mmproj="${LLM_MMPROJ:-${MMPROJ:-}}"
|
||||
if [ "${mmproj}" = "none" ]; then mmproj=""; fi
|
||||
local tmpl="${TEMPLATE:-}"
|
||||
local tmpl="${LLM_TEMPLATE:-${TEMPLATE:-}}"
|
||||
local name="custom_solo"
|
||||
|
||||
launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \
|
||||
"$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \
|
||||
"${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \
|
||||
"${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}"
|
||||
launch_instance "${name}" "$model" "${LLM_PORT:-${PORT:-11434}}" "${LLM_CTX:-${CTX:-65536}}" \
|
||||
"$mmproj" "$tmpl" "${LLM_TEMPERATURE:-${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}}" \
|
||||
"${LLM_TOP_P:-${TOP_P:-0.95}}" "${LLM_TOP_K:-${TOP_K:-40}}" \
|
||||
"${LLM_MIN_P:-${MIN_P:-0.05}}" "${LLM_NGL:-${NGL:-99}}" \
|
||||
"${LLM_LOAD_MODE:-${LOAD_MODE:-none}}" "${LLM_FLASH_ATTN:-${FLASH_ATTN:-on}}" \
|
||||
"${LLM_CTK:-${CTK:-q8_0}}" "${LLM_CTV:-${CTV:-q8_0}}" \
|
||||
"${LLM_REASONING_PRESERVE:-${REASONING_PRESERVE:-false}}"
|
||||
|
||||
echo ""
|
||||
echo "Streaming logs: tail -f ${LOG_DIR}/${name}.log"
|
||||
|
|
@ -348,20 +359,22 @@ Commands:
|
|||
muse Launch Muse-Glimmer 30B Solo (128K context, port 11434)
|
||||
superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434)
|
||||
qwen4b Launch Qwen 3.5 4B Solo (port 11434)
|
||||
custom Launch custom model instance (requires MODEL=/path/to/model.gguf)
|
||||
custom Launch custom model instance (requires LLM_MODEL=/path/to/model.gguf)
|
||||
status Show current server health and active ports
|
||||
stop [target] Stop servers (target: "all" or port number like 11434)
|
||||
help Display this help message
|
||||
|
||||
Environment Variables:
|
||||
MODEL Path to GGUF model file
|
||||
MMPROJ Path to mmproj file (or "none" for text-only)
|
||||
TEMPLATE Path to chat template file
|
||||
HOST Bind address (default: 0.0.0.0)
|
||||
PORT Main port override (default: 11434)
|
||||
CTX Context size override
|
||||
FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on)
|
||||
LOG_DIR Path to write logs (default: ./logs)
|
||||
LLM_MODEL Path to GGUF model file (fallback: MODEL)
|
||||
LLM_MMPROJ Path to mmproj file, or "none" (fallback: MMPROJ)
|
||||
LLM_TEMPLATE Path to chat template file (fallback: TEMPLATE)
|
||||
LLM_HOST Bind address (default: 0.0.0.0)
|
||||
LLM_PORT Main port override (default: 11434)
|
||||
LLM_CTX Context size override (default: preset-specific)
|
||||
LLM_TEMPERATURE Sampling temperature (default: preset-specific)
|
||||
LLM_PARALLEL Parallel slot count (default: 1)
|
||||
LLM_FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on)
|
||||
LLM_LOG_DIR Path to write logs (default: ./logs)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
|
|
|||
30
bin/llm-tui
30
bin/llm-tui
|
|
@ -729,27 +729,27 @@ class LLMConfigApp(App):
|
|||
env.pop("TEMP", None)
|
||||
env.pop("TMP", None)
|
||||
|
||||
env["TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6"
|
||||
env["CTX"] = self.query_one("#input-ctx", Input).value or "65536"
|
||||
env["PORT"] = self.query_one("#input-port", Input).value or "11434"
|
||||
env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95"
|
||||
env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05"
|
||||
env["NGL"] = self.query_one("#input-ngl", Input).value or "99"
|
||||
env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
|
||||
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
|
||||
env["CTV"] = env["CTK"]
|
||||
env["PARALLEL"] = "1"
|
||||
env["LLM_TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6"
|
||||
env["LLM_CTX"] = self.query_one("#input-ctx", Input).value or "65536"
|
||||
env["LLM_PORT"] = self.query_one("#input-port", Input).value or "11434"
|
||||
env["LLM_TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95"
|
||||
env["LLM_MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05"
|
||||
env["LLM_NGL"] = self.query_one("#input-ngl", Input).value or "99"
|
||||
env["LLM_FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
|
||||
env["LLM_CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
|
||||
env["LLM_CTV"] = env["LLM_CTK"]
|
||||
env["LLM_PARALLEL"] = "1"
|
||||
|
||||
if model_val and Path(model_val).exists():
|
||||
env["MODEL"] = model_val
|
||||
env["LLM_MODEL"] = model_val
|
||||
tmpl = Path(model_val).parent / "chat_template.jinja"
|
||||
if tmpl.exists():
|
||||
env["TEMPLATE"] = str(tmpl)
|
||||
env["LLM_TEMPLATE"] = str(tmpl)
|
||||
|
||||
if mmproj_val and mmproj_val != "none" and Path(mmproj_val).exists():
|
||||
env["MMPROJ"] = mmproj_val
|
||||
env["LLM_MMPROJ"] = mmproj_val
|
||||
else:
|
||||
env["MMPROJ"] = "none"
|
||||
env["LLM_MMPROJ"] = "none"
|
||||
|
||||
runner_cmd = PRESETS.get(preset, {}).get("command", preset)
|
||||
cmd = [str(RUNNER_BIN), runner_cmd]
|
||||
|
|
@ -758,7 +758,7 @@ class LLMConfigApp(App):
|
|||
f"\n[bold cyan]>>> Launching preset '{preset}' (runner command: {runner_cmd})...[/]\n"
|
||||
f" GGUF Model: {Path(model_val).name if model_val else 'Default'}\n"
|
||||
f" Vision MMProj: {Path(mmproj_val).name if (mmproj_val and mmproj_val != 'none') else 'None (Text Only)'}\n"
|
||||
f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n"
|
||||
f" Port: {env['LLM_PORT']} | Context: {env['LLM_CTX']} | KV Quant: {env['LLM_CTK']}\n"
|
||||
)
|
||||
|
||||
runner_log = open(LOGS_DIR / "runner.log", "a")
|
||||
|
|
|
|||
Loading…
Reference in a new issue