diff --git a/bin/llm-runner b/bin/llm-runner index a830f80..ed7d61c 100755 --- a/bin/llm-runner +++ b/bin/llm-runner @@ -136,6 +136,7 @@ launch_instance() { --host "${HOST:-0.0.0.0}" --port "${port}" --ctx-size "${ctx}" + -np "${PARALLEL:-1}" --n-gpu-layers "${ngl}" --load-mode "${load_mode}" --temp "${temp}" diff --git a/bin/llm-tui b/bin/llm-tui index 78a0857..8c5d63d 100755 --- a/bin/llm-tui +++ b/bin/llm-tui @@ -734,6 +734,7 @@ class LLMConfigApp(App): env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off" env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") env["CTV"] = env["CTK"] + env["PARALLEL"] = "1" if model_val and Path(model_val).exists(): env["MODEL"] = model_val