diff --git a/bin/llm-runner b/bin/llm-runner index d675e14..8bf8be0 100755 --- a/bin/llm-runner +++ b/bin/llm-runner @@ -6,6 +6,17 @@ set -euo pipefail +# Sanitize environment: +# TEMP/TMP are standard POSIX variables for temporary file directories. +# If set to a sampling temperature (float/number), ROCm/libstdc++ crashes during initialization! +if [ -n "${TEMP:-}" ] && [ ! -d "${TEMP}" ]; then + TEMPERATURE="${TEMPERATURE:-${TEMP}}" + unset TEMP +fi +if [ -n "${TMP:-}" ] && [ ! -d "${TMP}" ]; then + unset TMP +fi + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}" mkdir -p "${LOG_DIR}" @@ -45,7 +56,7 @@ LLAMA_SERVER="$(find_llama_server)" check_health() { local port="${1:-11434}" local host="${2:-127.0.0.1}" - if curl -s "http://${host}:${port}/health" 2>/dev/null | grep -q '"status":\s*"ok"\|"status":\s*"loading model"\|"ok"'; then + if curl -s "http://${host}:${port}/health" 2>/dev/null | grep -q '"status":\s*"ok"\|"ok"'; then return 0 fi return 1 @@ -247,7 +258,7 @@ cmd_muse() { local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ - "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" @@ -270,7 +281,7 @@ cmd_superqwen() { local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ - "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" @@ -290,7 +301,7 @@ cmd_qwen4b() { local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ - "$mmproj" "$tmpl" "${TEMP:-0.7}" "${TOP_P:-0.8}" "${TOP_K:-20}" \ + "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.7}}" "${TOP_P:-0.8}" "${TOP_K:-20}" \ "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ "${CTK:-q8_0}" "${CTV:-q8_0}" "false" @@ -315,7 +326,7 @@ cmd_custom() { local name="custom_solo" launch_instance "${name}" "$model" "${PORT:-11434}" "${CTX:-65536}" \ - "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "$mmproj" "$tmpl" "${TEMPERATURE:-${SAMPLING_TEMP:-0.6}}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-false}" diff --git a/bin/llm-tui b/bin/llm-tui index 8c5d63d..169f048 100755 --- a/bin/llm-tui +++ b/bin/llm-tui @@ -725,9 +725,13 @@ class LLMConfigApp(App): mmproj_val = str(self.query_one("#select-mmproj", Select).value or "") env = os.environ.copy() + # Clean up POSIX temp directory variables that could collide with sampling parameters + env.pop("TEMP", None) + env.pop("TMP", None) + + env["TEMPERATURE"] = self.query_one("#input-temp", Input).value or "0.6" env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" env["PORT"] = self.query_one("#input-port", Input).value or "11434" - env["TEMP"] = self.query_one("#input-temp", Input).value or "0.6" env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95" env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05" env["NGL"] = self.query_one("#input-ngl", Input).value or "99" @@ -757,7 +761,8 @@ class LLMConfigApp(App): f" Port: {env['PORT']} | Context: {env['CTX']} | KV Quant: {env['CTK']}\n" ) - subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True) + runner_log = open(LOGS_DIR / "runner.log", "a") + subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), stdout=runner_log, stderr=runner_log, start_new_session=True) self.notify(f"Launching {preset} server. Switching to Status & Logs...", severity="information") self.query_one(TabbedContent).active = "tab-status" self.action_refresh_status()