fix(llm-runner): set -np 1 to prevent OOM crash on large context windows
This commit is contained in:
parent
d3b0f9ba7a
commit
954b6728e8
2 changed files with 2 additions and 0 deletions
|
|
@ -136,6 +136,7 @@ launch_instance() {
|
|||
--host "${HOST:-0.0.0.0}"
|
||||
--port "${port}"
|
||||
--ctx-size "${ctx}"
|
||||
-np "${PARALLEL:-1}"
|
||||
--n-gpu-layers "${ngl}"
|
||||
--load-mode "${load_mode}"
|
||||
--temp "${temp}"
|
||||
|
|
|
|||
|
|
@ -734,6 +734,7 @@ class LLMConfigApp(App):
|
|||
env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off"
|
||||
env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0")
|
||||
env["CTV"] = env["CTK"]
|
||||
env["PARALLEL"] = "1"
|
||||
|
||||
if model_val and Path(model_val).exists():
|
||||
env["MODEL"] = model_val
|
||||
|
|
|
|||
Loading…
Reference in a new issue