fix(llm-runner): set -np 1 to prevent OOM crash on large context windows

This commit is contained in:
Noel Bautista 2026-10-21 13:46:08 -07:00
parent d3b0f9ba7a
commit 954b6728e8
No known key found for this signature in database
2 changed files with 2 additions and 0 deletions

View file

@ -136,6 +136,7 @@ launch_instance() {
--host "${HOST:-0.0.0.0}"
--port "${port}"
--ctx-size "${ctx}"
-np "${PARALLEL:-1}"
--n-gpu-layers "${ngl}"
--load-mode "${load_mode}"
--temp "${temp}"

View file

@ -734,6 +734,7 @@ class LLMConfigApp(App):
env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
env["CTV"] = env["CTK"]
env["PARALLEL"] = "1"
if model_val and Path(model_val).exists():
env["MODEL"] = model_val