From f78402d8bf4cdb52bd5022db9b334865c2546d2d Mon Sep 17 00:00:00 2001 From: Noel Bautista Date: Wed, 21 Oct 2026 12:17:34 -0700 Subject: [PATCH] Initial commit: LLM inference orchestrator with TUI form and ROCm configs --- .gitignore | 35 ++ README.md | 81 ++++ bin/llm-runner | 342 ++++++++++++++ bin/llm-tui | 499 +++++++++++++++++++++ docs/architecture.md | 75 ++++ docs/parameter-reference.md | 43 ++ docs/vram-budgeting.md | 49 ++ models/Qwen3.5/chat_template.jinja | 363 +++++++++++++++ models/Qwen3.8/chat_template.jinja | 363 +++++++++++++++ models/README.md | 33 ++ models/muse-glimmer/chat_template.jinja | 211 +++++++++ run.sh | 5 + run_muse.sh | 5 + run_superqwen.sh | 5 + shell.nix | 16 + templates/Qwen3.5/chat_template.jinja | 363 +++++++++++++++ templates/Qwen3.8/chat_template.jinja | 363 +++++++++++++++ templates/muse-glimmer/chat_template.jinja | 211 +++++++++ 18 files changed, 3062 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 bin/llm-runner create mode 100755 bin/llm-tui create mode 100644 docs/architecture.md create mode 100644 docs/parameter-reference.md create mode 100644 docs/vram-budgeting.md create mode 100644 models/Qwen3.5/chat_template.jinja create mode 100644 models/Qwen3.8/chat_template.jinja create mode 100644 models/README.md create mode 100644 models/muse-glimmer/chat_template.jinja create mode 100755 run.sh create mode 100755 run_muse.sh create mode 100755 run_superqwen.sh create mode 100644 shell.nix create mode 100644 templates/Qwen3.5/chat_template.jinja create mode 100644 templates/Qwen3.8/chat_template.jinja create mode 100644 templates/muse-glimmer/chat_template.jinja diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a6cba1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Model weights (Large binary files) +models/**/*.gguf +models/**/*.bin +models/**/*.safetensors +models/**/*.pt +*.gguf +*.bin +*.safetensors + +# Runtime logs +logs/ +*.log + +# Python virtual environments & bytecode caches +venv/ +.venv/ +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.ruff_cache/ + +# Databases & local state +*.db +*.sqlite +*.sqlite3 + +# Environment variables & secrets +.env +.env.local +.env.*.local + +# OS Artifacts +.DS_Store +Thumbs.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..a36af54 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# Local LLM Serving & Multi-Agent Inference + +High-performance inference orchestration using `llama-server` on an AMD ROCm GPU (~28–30GB usable VRAM). + +``` +llms/ +├── bin/ +│ ├── llm-runner # Unified CLI orchestrator (cluster, muse, superqwen, qwen4b, status, stop) +│ └── llm-tui # Interactive Textual TUI configurator form (run via uv) +├── templates/ # Jinja chat templates for Qwen and Muse-Glimmer +├── models/ # Quantized GGUF weights and vision encoders (ignored by git) +├── docs/ # Comprehensive documentation +│ ├── architecture.md +│ ├── vram-budgeting.md +│ └── parameter-reference.md +├── run.sh # Backwards-compatible wrapper -> bin/llm-runner cluster +├── run_muse.sh # Backwards-compatible wrapper -> bin/llm-runner muse +├── run_superqwen.sh # Backwards-compatible wrapper -> bin/llm-runner superqwen +└── shell.nix # Development environment shell +``` + +--- + +## Quickstart + +### 1. Launching via Interactive TUI Form +Launch the full interactive TUI form to visually pick presets or customize parameters with real-time VRAM estimates: + +```bash +uv run bin/llm-tui +# Or directly if executable: +./bin/llm-tui +``` + +Inside the TUI: +- `F5` / **Launch**: Starts the server(s) in the background. +- `F8` / **Stop**: Stops server on the selected port. +- `F9` / **Stop All**: Kills all running `llama-server` instances. +- `F2` / **Refresh**: Refreshes health status. +- Real-time VRAM budget calculation alerts you if selected context sizes risk VRAM overflow. + +### 2. Launching via CLI Runner +```bash +# Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B on ports 11434-11436) +./bin/llm-runner cluster + +# Muse-Glimmer 30B Solo (128K Context on port 11434) +./bin/llm-runner muse + +# SuperQwen 3.8 27B Solo (256K Context on port 11434) +./bin/llm-runner superqwen + +# Check status of running servers +./bin/llm-runner status + +# Stop all running instances +./bin/llm-runner stop all +``` + +--- + +## Server Endpoints & Ports + +| Port | Role | Default Model | Context Window | +| :--- | :--- | :--- | :--- | +| **`11434`** | Planner / Solo | SuperQwen 3.8 27B or Muse-Glimmer 30B | 64K (cluster) / 128K (muse) / 256K (superqwen) | +| **`11435`** | Worker #1 | Qwen 3.5 4B | 16K | +| **`11436`** | Worker #2 | Qwen 3.5 4B | 16K | + +Each instance exposes standard OpenAI-compatible API routes: +- **Chat Completions**: `http://localhost:/v1/chat/completions` +- **Models**: `http://localhost:/v1/models` +- **Health Check**: `http://localhost:/health` + +--- + +## Detailed Documentation +- [Architecture & Multi-Agent Orchestration](docs/architecture.md) +- [VRAM Budgeting & ROCm Memory Limits](docs/vram-budgeting.md) +- [Parameter Reference & Sampling Profiles](docs/parameter-reference.md) +- [Model Weights & Storage Guide](models/README.md) diff --git a/bin/llm-runner b/bin/llm-runner new file mode 100755 index 0000000..83fd6fc --- /dev/null +++ b/bin/llm-runner @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# ============================================================================== +# LLM Runner: Unified Inference Server Orchestrator for llama-server +# Target Hardware: AMD ROCm / Radeon AI PRO R9700 (~28-30GB VRAM) +# ============================================================================== + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="${LOG_DIR:-${SCRIPT_DIR}/logs}" +mkdir -p "${LOG_DIR}" + +# ------------------------------------------------------------------------------ +# Helpers & Color Output +# ------------------------------------------------------------------------------ +BOLD="\033[1m" +GREEN="\033[32m" +BLUE="\033[34m" +YELLOW="\033[33m" +RED="\033[31m" +CYAN="\033[36m" +RESET="\033[0m" + +log_info() { echo -e "${BLUE}[INFO]${RESET} $*"; } +log_success() { echo -e "${GREEN}[✓]${RESET} $*"; } +log_warn() { echo -e "${YELLOW}[!]${RESET} $*"; } +log_error() { echo -e "${RED}[✗]${RESET} $*" >&2; } + +# Find llama-server binary +find_llama_server() { + if command -v llama-server >/dev/null 2>&1; then + command -v llama-server + elif [ -x "/run/current-system/sw/bin/llama-server" ]; then + echo "/run/current-system/sw/bin/llama-server" + else + log_error "llama-server executable not found in PATH or system profile!" + exit 1 + fi +} +LLAMA_SERVER="$(find_llama_server)" + +# ------------------------------------------------------------------------------ +# Status & Inspection +# ------------------------------------------------------------------------------ +check_health() { + local port="${1:-11434}" + local host="${2:-127.0.0.1}" + if curl -s "http://${host}:${port}/health" 2>/dev/null | grep -q '"status":\s*"ok"\|"status":\s*"loading model"\|"ok"'; then + return 0 + fi + return 1 +} + +cmd_status() { + echo -e "${BOLD}=== Local LLM Server Status ===${RESET}" + local ports=(11434 11435 11436) + local any_running=0 + + for port in "${ports[@]}"; do + local pid + pid=$(lsof -ti :"$port" 2>/dev/null || fuser "$port/tcp" 2>/dev/null || true) + if [ -n "$pid" ]; then + any_running=1 + local model_info + model_info=$(curl -s "http://127.0.0.1:${port}/v1/models" 2>/dev/null | jq -r '.data[0].id // "unknown"' 2>/dev/null || echo "initializing...") + echo -e " Port ${CYAN}${port}${RESET}: ${GREEN}RUNNING${RESET} (PID: ${pid}) - Model: ${BOLD}${model_info}${RESET}" + else + echo -e " Port ${CYAN}${port}${RESET}: ${YELLOW}STOPPED${RESET}" + fi + done + + echo "" + return 0 +} + +# ------------------------------------------------------------------------------ +# Stop Servers +# ------------------------------------------------------------------------------ +cmd_stop() { + local target="${1:-all}" + log_info "Stopping server(s): ${target}..." + + if [ "$target" = "all" ]; then + pkill -f "llama-server" 2>/dev/null || true + log_success "All llama-server processes terminated." + else + local pid + pid=$(lsof -ti :"$target" 2>/dev/null || fuser "$target/tcp" 2>/dev/null || true) + if [ -n "$pid" ]; then + kill "$pid" 2>/dev/null || true + log_success "Killed server on port ${target} (PID: ${pid})." + else + log_warn "No active server found on port ${target}." + fi + fi +} + +# ------------------------------------------------------------------------------ +# Launch Single Server Instance +# ------------------------------------------------------------------------------ +launch_instance() { + local name="$1" + local model="$2" + local port="$3" + local ctx="$4" + local mmproj="${5:-}" + local template="${6:-}" + local temp="${7:-0.6}" + local top_p="${8:-0.95}" + local top_k="${9:-40}" + local min_p="${10:-0.05}" + local ngl="${11:-99}" + local load_mode="${12:-none}" + local flash_attn="${13:-on}" + local ctk="${14:-q8_0}" + local ctv="${15:-q8_0}" + local reasoning="${16:-false}" + local log_file="${LOG_DIR}/${name}.log" + + if [ ! -f "$model" ]; then + log_error "Model file not found: ${model}" + exit 1 + fi + + # Check if port is already in use + local existing_pid + existing_pid=$(lsof -ti :"$port" 2>/dev/null || fuser "$port/tcp" 2>/dev/null || true) + if [ -n "$existing_pid" ]; then + log_warn "Port ${port} is already in use by PID ${existing_pid}. Terminating existing process..." + kill -9 "$existing_pid" 2>/dev/null || true + sleep 1 + fi + + local flags=( + -m "${model}" + --host "${HOST:-0.0.0.0}" + --port "${port}" + --ctx-size "${ctx}" + --n-gpu-layers "${ngl}" + --load-mode "${load_mode}" + --temp "${temp}" + --top-p "${top_p}" + --top-k "${top_k}" + --min-p "${min_p}" + -ctk "${ctk}" + -ctv "${ctv}" + -fa "${flash_attn}" + -b "${BATCH_SIZE:-2048}" + -ub "${UBATCH_SIZE:-512}" + ) + + if [ -n "${mmproj}" ] && [ -f "${mmproj}" ]; then + flags+=(--mmproj "${mmproj}" --image-min-tokens "${IMAGE_MIN_TOKENS:-1024}") + fi + + if [ -n "${template}" ] && [ -f "${template}" ]; then + flags+=(--chat-template-file "${template}" --jinja) + fi + + if [ "${reasoning}" = "true" ]; then + flags+=(--reasoning-preserve) + fi + + log_info "Starting ${BOLD}${name}${RESET} on port ${CYAN}${port}${RESET} (CTX: ${ctx})..." + "${LLAMA_SERVER}" "${flags[@]}" > "${log_file}" 2>&1 & + local s_pid=$! + + # Wait for initialization + log_info "Waiting for ${name} to initialize (PID: ${s_pid})..." + while true; do + if ! kill -0 "${s_pid}" 2>/dev/null; then + log_error "${name} failed to start! Last 20 log lines from ${log_file}:" + tail -n 20 "${log_file}" >&2 + exit 1 + fi + if check_health "${port}"; then + break + fi + sleep 1 + done + + log_success "${name} is READY at http://localhost:${port} (PID: ${s_pid})" + echo "${s_pid}" +} + +# ------------------------------------------------------------------------------ +# Presets +# ------------------------------------------------------------------------------ +cmd_cluster() { + log_info "Starting Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)..." + + # Trap to gracefully stop all background instances on Ctrl+C + cleanup_cluster() { + log_warn "Stopping cluster instances..." + pkill -P $$ 2>/dev/null || true + pkill -f "llama-server" 2>/dev/null || true + exit 0 + } + trap cleanup_cluster SIGINT SIGTERM EXIT + + # SuperQwen 3.8 27B (Planner) + local planner_model="${PLANNER_MODEL:-${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf}" + if [ ! -f "${planner_model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" ]; then + planner_model="${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf" + fi + local planner_mmproj="${PLANNER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + local planner_tmpl="${PLANNER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" + + # Qwen 3.5 4B (Workers) + local worker_model="${WORKER_MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" + local worker_mmproj="${WORKER_MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + local worker_tmpl="${WORKER_TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" + + # 1. Planner on 11434 + launch_instance "planner_27b" "$planner_model" "${PLANNER_PORT:-11434}" "${PLANNER_CTX:-65536}" \ + "$planner_mmproj" "$planner_tmpl" "${PLANNER_TEMP:-0.6}" "${PLANNER_TOP_P:-0.95}" "${PLANNER_TOP_K:-40}" \ + "${PLANNER_MIN_P:-0.05}" "${PLANNER_NGL:-99}" "${PLANNER_LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ + "${PLANNER_CTK:-q8_0}" "${PLANNER_CTV:-q8_0}" "false" + + # 2. Worker 1 on 11435 + launch_instance "worker_4b_1" "$worker_model" "${WORKER1_PORT:-11435}" "${WORKER1_CTX:-16384}" \ + "$worker_mmproj" "$worker_tmpl" "${WORKER1_TEMP:-0.7}" "${WORKER1_TOP_P:-0.8}" "${WORKER1_TOP_K:-20}" \ + "${WORKER1_MIN_P:-0.05}" "${WORKER1_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ + "${WORKER1_CTK:-q8_0}" "${WORKER1_CTV:-q8_0}" "false" + + # 3. Worker 2 on 11436 + launch_instance "worker_4b_2" "$worker_model" "${WORKER2_PORT:-11436}" "${WORKER2_CTX:-16384}" \ + "$worker_mmproj" "$worker_tmpl" "${WORKER2_TEMP:-0.7}" "${WORKER2_TOP_P:-0.8}" "${WORKER2_TOP_K:-20}" \ + "${WORKER2_MIN_P:-0.05}" "${WORKER2_NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ + "${WORKER2_CTK:-q8_0}" "${WORKER2_CTV:-q8_0}" "false" + + echo "" + log_success "All 3 cluster servers are operational:" + echo " [Planner] SuperQwen 3.8 27B -> http://localhost:${PLANNER_PORT:-11434}" + echo " [Implementer] Qwen 3.5 4B #1 -> http://localhost:${WORKER1_PORT:-11435}" + echo " [Implementer] Qwen 3.5 4B #2 -> http://localhost:${WORKER2_PORT:-11436}" + echo "" + echo "Logs directory: ${LOG_DIR}/" + echo "Press Ctrl+C to stop all instances gracefully." + wait +} + +cmd_muse() { + log_info "Starting Muse-Glimmer 30B Solo Server (Max Context: 128K)..." + local model="${MODEL:-${SCRIPT_DIR}/models/muse-glimmer/Muse-Glimmer-30B-UD-Q4_K_XL.gguf}" + local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/muse-glimmer/mmproj-Muse-Glimmer-30B-Q8_0.gguf}" + local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/muse-glimmer/chat_template.jinja}" + + launch_instance "muse_solo" "$model" "${PORT:-11434}" "${CTX:-131072}" \ + "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ + "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" + + echo "" + echo "Streaming logs: tail -f ${LOG_DIR}/muse_solo.log" + echo "Press Ctrl+C to stop." + wait +} + +cmd_superqwen() { + log_info "Starting SuperQwen 3.8 27B Solo Server (Max Context: 256K)..." + local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.8/SuperQwen3.8-27b-abliterated-Q4_K_M.gguf}" + if [ ! -f "${model}" ] && [ -f "${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" ]; then + model="${SCRIPT_DIR}/models/Qwen3.8/Qwen3.8-27B-UD-Q4_K_XL.gguf" + fi + local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.8/mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf}" + local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.8/chat_template.jinja}" + + launch_instance "superqwen_solo" "$model" "${PORT:-11434}" "${CTX:-262144}" \ + "$mmproj" "$tmpl" "${TEMP:-0.6}" "${TOP_P:-0.95}" "${TOP_K:-40}" \ + "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-none}" "${FLASH_ATTN:-on}" \ + "${CTK:-q8_0}" "${CTV:-q8_0}" "${REASONING_PRESERVE:-true}" + + echo "" + echo "Streaming logs: tail -f ${LOG_DIR}/superqwen_solo.log" + echo "Press Ctrl+C to stop." + wait +} + +cmd_qwen4b() { + log_info "Starting Qwen 3.5 4B Solo Server..." + local model="${MODEL:-${SCRIPT_DIR}/models/Qwen3.5/Qwen3.5-4B-Q5_K_M.gguf}" + local mmproj="${MMPROJ:-${SCRIPT_DIR}/models/Qwen3.5/mmproj-BF16.gguf}" + local tmpl="${TEMPLATE:-${SCRIPT_DIR}/templates/Qwen3.5/chat_template.jinja}" + + launch_instance "qwen4b_solo" "$model" "${PORT:-11434}" "${CTX:-32768}" \ + "$mmproj" "$tmpl" "${TEMP:-0.7}" "${TOP_P:-0.8}" "${TOP_K:-20}" \ + "${MIN_P:-0.05}" "${NGL:-99}" "${LOAD_MODE:-mmap}" "${FLASH_ATTN:-on}" \ + "${CTK:-q8_0}" "${CTV:-q8_0}" "false" + + echo "" + echo "Streaming logs: tail -f ${LOG_DIR}/qwen4b_solo.log" + echo "Press Ctrl+C to stop." + wait +} + +# ------------------------------------------------------------------------------ +# Help / Usage +# ------------------------------------------------------------------------------ +cmd_help() { + cat < [options] + +Commands: + cluster Launch multi-agent cluster (SuperQwen 27B on 11434, 2x Qwen 4B on 11435/11436) + muse Launch Muse-Glimmer 30B Solo (128K context, port 11434) + superqwen Launch SuperQwen 3.8 27B Solo (256K context, port 11434) + qwen4b Launch Qwen 3.5 4B Solo (port 11434) + status Show current server health and active ports + stop [target] Stop servers (target: "all" or port number like 11434) + help Display this help message + +Environment Variables: + HOST Bind address (default: 0.0.0.0) + PORT Main port override (default: 11434) + CTX Context size override + FLASH_ATTN Enable Flash Attention: "on" or "off" (default: on) + LOG_DIR Path to write logs (default: ./logs) + +EOF +} + +# ------------------------------------------------------------------------------ +# Dispatch +# ------------------------------------------------------------------------------ +ACTION="${1:-help}" +shift || true + +case "${ACTION}" in + cluster) cmd_cluster "$@" ;; + muse) cmd_muse "$@" ;; + superqwen) cmd_superqwen "$@" ;; + qwen4b) cmd_qwen4b "$@" ;; + status) cmd_status "$@" ;; + stop) cmd_stop "$@" ;; + help|--help|-h) cmd_help ;; + *) + log_error "Unknown command: ${ACTION}" + cmd_help + exit 1 + ;; +esac diff --git a/bin/llm-tui b/bin/llm-tui new file mode 100755 index 0000000..0c8f3d0 --- /dev/null +++ b/bin/llm-tui @@ -0,0 +1,499 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "textual>=0.75.0", +# "rich>=13.0.0", +# ] +# /// + +from __future__ import annotations + +import os +import signal +import subprocess +import urllib.request +import json +from pathlib import Path +from typing import Optional + +from rich.text import Text +from textual.app import App, ComposeResult +from textual.containers import Container, Horizontal, Vertical, VerticalScroll +from textual.binding import Binding +from textual.widgets import ( + Button, + Checkbox, + Footer, + Header, + Input, + Label, + Log, + RadioSet, + RadioButton, + Select, + Static, + TabbedContent, + TabPane, +) + +SCRIPT_DIR = Path(__file__).resolve().parent.parent +MODELS_DIR = SCRIPT_DIR / "models" +TEMPLATES_DIR = SCRIPT_DIR / "templates" +LOGS_DIR = SCRIPT_DIR / "logs" +RUNNER_BIN = SCRIPT_DIR / "bin" / "llm-runner" + +TOTAL_USABLE_VRAM_GB = 30.0 + +PRESETS = { + "cluster": { + "name": "🌟 Multi-Agent Cluster (SuperQwen 27B + 2x Qwen 4B)", + "desc": "Planner on 11434, Worker 1 on 11435, Worker 2 on 11436. Optimized for Pi & Hermes agents.", + "port": 11434, + "ctx": 65536, + "temp": 0.6, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.05, + "ngl": 99, + "flash_attn": True, + "kv_quant": "q8_0", + "command": "cluster", + }, + "muse": { + "name": "🔮 Muse-Glimmer 30B Solo (128K Context)", + "desc": "Dedicated deep-reasoning agent with max 128K context on port 11434.", + "port": 11434, + "ctx": 131072, + "temp": 0.6, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.05, + "ngl": 99, + "flash_attn": True, + "kv_quant": "q8_0", + "command": "muse", + }, + "superqwen": { + "name": "🧠 SuperQwen 3.8 27B Solo (256K Context)", + "desc": "Dedicated ultra-long reasoning agent with max 256K context on port 11434.", + "port": 11434, + "ctx": 262144, + "temp": 0.6, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.05, + "ngl": 99, + "flash_attn": True, + "kv_quant": "q8_0", + "command": "superqwen", + }, + "qwen4b": { + "name": "⚡ Qwen 3.5 4B Solo (Fast Instruct)", + "desc": "Lightweight high-speed instruct model on port 11434 with ~3.9GB VRAM footprint.", + "port": 11434, + "ctx": 32768, + "temp": 0.7, + "top_p": 0.8, + "top_k": 20, + "min_p": 0.05, + "ngl": 99, + "flash_attn": True, + "kv_quant": "q8_0", + "command": "qwen4b", + }, + "custom": { + "name": "🛠️ Custom Model & Parameters", + "desc": "Specify exact GGUF weight files, context window, and sampling hyperparameters.", + "port": 11434, + "ctx": 65536, + "temp": 0.6, + "top_p": 0.95, + "top_k": 40, + "min_p": 0.05, + "ngl": 99, + "flash_attn": True, + "kv_quant": "q8_0", + "command": "custom", + }, +} + + +def find_gguf_models() -> list[tuple[str, str]]: + models: list[tuple[str, str]] = [] + if MODELS_DIR.exists(): + for path in sorted(MODELS_DIR.rglob("*.gguf")): + if "mmproj" not in path.name.lower(): + rel = path.relative_to(MODELS_DIR) + models.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path))) + if not models: + models.append(("No GGUF models found in models/", "")) + return models + + +def find_mmproj_files() -> list[tuple[str, str]]: + projectors = [("None (Text Only)", "")] + if MODELS_DIR.exists(): + for path in sorted(MODELS_DIR.rglob("mmproj*.gguf")): + projectors.append((f"{path.name} ({path.stat().st_size / 1e9:.1f} GB)", str(path))) + return projectors + + +def check_port_status(port: int) -> tuple[bool, str]: + url = f"http://127.0.0.1:{port}/health" + try: + req = urllib.request.Request(url, headers={"User-Agent": "LLM-TUI"}) + with urllib.request.urlopen(req, timeout=0.8) as response: + data = json.loads(response.read().decode()) + status = data.get("status", "unknown") + return True, f"Online ({status})" + except Exception: + return False, "Offline" + + +class LLMConfigApp(App): + CSS = """ + Screen { + background: #14171d; + color: #d8dee9; + } + Header { + background: #1e222b; + color: #88c0d0; + text-style: bold; + } + TabbedContent { + height: 1fr; + } + TabPane { + padding: 1 2; + } + .card { + background: #1a1e27; + border: solid #3b4252; + padding: 1 2; + margin-bottom: 1; + } + .title { + color: #88c0d0; + text-style: bold; + margin-bottom: 1; + } + .field-row { + height: auto; + margin-bottom: 1; + } + .field-label { + width: 18; + color: #81a1c1; + padding-top: 1; + } + Input { + background: #232834; + border: solid #434c5e; + color: #eceff4; + width: 1fr; + } + Input:focus { + border: solid #88c0d0; + } + Select { + background: #232834; + border: solid #434c5e; + color: #eceff4; + width: 1fr; + } + Button { + margin-right: 1; + background: #2e3440; + color: #eceff4; + border: solid #4c566a; + } + Button:hover { + background: #434c5e; + } + Button.-primary { + background: #5e81ac; + color: #eceff4; + text-style: bold; + border: solid #81a1c1; + } + Button.-error { + background: #bf616a; + color: #eceff4; + text-style: bold; + border: solid #d08770; + } + #vram-meter { + background: #232834; + border: solid #4c566a; + padding: 1; + margin-top: 1; + } + #status-card { + background: #1e222b; + border: solid #4c566a; + padding: 1; + margin-bottom: 1; + } + Log { + background: #0f1117; + border: solid #3b4252; + height: 1fr; + } + """ + + BINDINGS = [ + Binding("f5", "launch", "Launch Server"), + Binding("f8", "stop", "Stop Selected"), + Binding("f9", "stop_all", "Stop All"), + Binding("f2", "refresh_status", "Refresh Status"), + Binding("ctrl+c", "quit", "Quit"), + Binding("q", "quit", "Quit"), + ] + + selected_preset: str = "cluster" + models_list: list[tuple[str, str]] = [] + mmproj_list: list[tuple[str, str]] = [] + + def compose(self) -> ComposeResult: + yield Header(show_clock=True) + self.models_list = find_gguf_models() + self.mmproj_list = find_mmproj_files() + + with TabbedContent(): + with TabPane("🚀 Model Launcher & Form", id="tab-launch"): + with VerticalScroll(): + # 1. Preset Selector + with Container(classes="card"): + yield Label("Select Inference Preset", classes="title") + with RadioSet(id="preset-radios"): + for key, preset in PRESETS.items(): + yield RadioButton( + preset["name"], + id=f"preset-{key}", + value=(key == "cluster"), + ) + yield Label( + PRESETS["cluster"]["desc"], + id="preset-description", + classes="text-muted", + ) + + # 2. Parameters Form + with Container(classes="card", id="form-container"): + yield Label("Hyperparameters & Hardware Tuning", classes="title") + + with Horizontal(classes="field-row"): + yield Label("GGUF Model:", classes="field-label") + yield Select( + options=[(lbl, val) for lbl, val in self.models_list], + value=self.models_list[0][1] if self.models_list else None, + id="select-model", + ) + + with Horizontal(classes="field-row"): + yield Label("Vision MMProj:", classes="field-label") + yield Select( + options=[(lbl, val) for lbl, val in self.mmproj_list], + value=self.mmproj_list[0][1] if self.mmproj_list else None, + id="select-mmproj", + ) + + with Horizontal(classes="field-row"): + yield Label("Context Size:", classes="field-label") + yield Input(value="65536", id="input-ctx") + + with Horizontal(classes="field-row"): + yield Label("Port:", classes="field-label") + yield Input(value="11434", id="input-port") + + with Horizontal(classes="field-row"): + yield Label("Temperature:", classes="field-label") + yield Input(value="0.6", id="input-temp") + + with Horizontal(classes="field-row"): + yield Label("Top-P:", classes="field-label") + yield Input(value="0.95", id="input-top-p") + + with Horizontal(classes="field-row"): + yield Label("Min-P:", classes="field-label") + yield Input(value="0.05", id="input-min-p") + + with Horizontal(classes="field-row"): + yield Label("GPU Layers (-ngl):", classes="field-label") + yield Input(value="99", id="input-ngl") + + with Horizontal(classes="field-row"): + yield Label("KV Cache Quant:", classes="field-label") + yield Select( + options=[("q8_0 (High Quality, 50% savings)", "q8_0"), ("f16 (Full Precision)", "f16"), ("q4_0 (Max Memory Savings)", "q4_0")], + value="q8_0", + id="select-kv", + ) + + with Horizontal(classes="field-row"): + yield Label("Flash Attention:", classes="field-label") + yield Checkbox("Enabled (-fa on)", value=True, id="check-fa") + + # VRAM Estimation Banner + yield Static(id="vram-meter") + + # 3. Execution Action Bar + with Horizontal(classes="card"): + yield Button("🚀 Launch Server (F5)", variant="primary", id="btn-launch") + yield Button("🛑 Stop Server (F8)", variant="error", id="btn-stop") + yield Button("⚠️ Stop All Instances (F9)", variant="error", id="btn-stop-all") + yield Button("🔄 Refresh Health (F2)", id="btn-refresh") + + with TabPane("📊 Server Status & Live Logs", id="tab-status"): + with Vertical(): + yield Static(id="status-card") + yield Label("Live Logs Streaming:", classes="title") + yield Log(id="log-stream", highlight=True) + + yield Footer() + + def on_mount(self) -> None: + self.update_vram_estimation() + self.action_refresh_status() + self.set_interval(3.0, self.action_refresh_status) + self.set_interval(1.5, self.tail_logs) + + def on_radio_set_changed(self, event: RadioSet.Changed) -> None: + radio_id = event.pressed.id or "" + preset_key = radio_id.replace("preset-", "") + if preset_key in PRESETS: + self.selected_preset = preset_key + preset = PRESETS[preset_key] + self.query_one("#preset-description", Label).update(preset["desc"]) + self.query_one("#input-port", Input).value = str(preset["port"]) + self.query_one("#input-ctx", Input).value = str(preset["ctx"]) + self.query_one("#input-temp", Input).value = str(preset["temp"]) + self.query_one("#input-top-p", Input).value = str(preset["top_p"]) + self.query_one("#input-min-p", Input).value = str(preset["min_p"]) + self.query_one("#input-ngl", Input).value = str(preset["ngl"]) + self.query_one("#check-fa", Checkbox).value = preset["flash_attn"] + self.query_one("#select-kv", Select).value = preset["kv_quant"] + self.update_vram_estimation() + + def on_input_changed(self, event: Input.Changed) -> None: + self.update_vram_estimation() + + def on_select_changed(self, event: Select.Changed) -> None: + self.update_vram_estimation() + + def update_vram_estimation(self) -> None: + try: + ctx = int(self.query_one("#input-ctx", Input).value or "0") + except ValueError: + ctx = 65536 + + preset = self.selected_preset + if preset == "cluster": + weights_gb = 25.9 + kv_gb = 3.2 + elif preset == "muse": + weights_gb = 17.5 + kv_gb = (ctx / 131072.0) * 8.5 + elif preset == "superqwen": + weights_gb = 17.5 + kv_gb = (ctx / 262144.0) * 10.5 + elif preset == "qwen4b": + weights_gb = 3.9 + kv_gb = (ctx / 32768.0) * 1.5 + else: + model_val = str(self.query_one("#select-model", Select).value or "") + if model_val and Path(model_val).exists(): + weights_gb = Path(model_val).stat().st_size / 1e9 + else: + weights_gb = 15.0 + kv_gb = (ctx / 65536.0) * 4.0 + + total_est = weights_gb + kv_gb + remaining = TOTAL_USABLE_VRAM_GB - total_est + + if total_est <= TOTAL_USABLE_VRAM_GB: + status_color = "green" + verdict = "SAFE (Fits in VRAM)" + else: + status_color = "red" + verdict = "OVERFLOW WARNING (May spill to system RAM / ROCm crash)" + + meter_text = ( + f"VRAM Budget (~{TOTAL_USABLE_VRAM_GB:.0f}GB Usable): " + f"Weights ~{weights_gb:.1f}GB + KV Cache ~{kv_gb:.1f}GB = " + f"[{status_color}]{total_est:.1f}GB total[/] | Remaining Headroom: ~{remaining:.1f}GB [{verdict}]" + ) + self.query_one("#vram-meter", Static).update(meter_text) + + def action_refresh_status(self) -> None: + lines = [] + for port, label in [(11434, "Planner / Solo"), (11435, "Worker 1"), (11436, "Worker 2")]: + online, msg = check_port_status(port) + badge = "[green]● ONLINE[/]" if online else "[yellow]○ OFFLINE[/]" + lines.append(f"Port {port} ({label}): {badge} - {msg}") + + card = self.query_one("#status-card", Static) + card.update("\n".join(lines)) + + def tail_logs(self) -> None: + log_widget = self.query_one("#log-stream", Log) + logs = sorted(LOGS_DIR.glob("*.log"), key=os.path.getmtime, reverse=True) + if logs: + latest = logs[0] + try: + with open(latest, "r", errors="ignore") as f: + f.seek(max(0, latest.stat().st_size - 8192)) + tail = f.read() + log_widget.clear() + log_widget.write(f"--- Streaming {latest.name} ---\n" + tail) + except Exception: + pass + + def action_launch(self) -> None: + preset = self.selected_preset + log_widget = self.query_one("#log-stream", Log) + log_widget.write(f"\n[bold cyan]>>> Launching preset: {preset}...[/]\n") + + env = os.environ.copy() + env["CTX"] = self.query_one("#input-ctx", Input).value or "65536" + env["PORT"] = self.query_one("#input-port", Input).value or "11434" + env["TEMP"] = self.query_one("#input-temp", Input).value or "0.6" + env["TOP_P"] = self.query_one("#input-top-p", Input).value or "0.95" + env["MIN_P"] = self.query_one("#input-min-p", Input).value or "0.05" + env["NGL"] = self.query_one("#input-ngl", Input).value or "99" + env["FLASH_ATTN"] = "on" if self.query_one("#check-fa", Checkbox).value else "off" + env["CTK"] = str(self.query_one("#select-kv", Select).value or "q8_0") + env["CTV"] = env["CTK"] + + cmd = [str(RUNNER_BIN), preset] + subprocess.Popen(cmd, env=env, cwd=str(SCRIPT_DIR), start_new_session=True) + self.notify(f"Launching {preset} server in background. Checking ports...", severity="information") + self.action_refresh_status() + + def action_stop(self) -> None: + port = self.query_one("#input-port", Input).value or "11434" + subprocess.run([str(RUNNER_BIN), "stop", port], cwd=str(SCRIPT_DIR)) + self.notify(f"Stopped server on port {port}", severity="warning") + self.action_refresh_status() + + def action_stop_all(self) -> None: + subprocess.run([str(RUNNER_BIN), "stop", "all"], cwd=str(SCRIPT_DIR)) + self.notify("Stopped all running LLM servers", severity="warning") + self.action_refresh_status() + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-launch": + self.action_launch() + elif event.button.id == "btn-stop": + self.action_stop() + elif event.button.id == "btn-stop-all": + self.action_stop_all() + elif event.button.id == "btn-refresh": + self.action_refresh_status() + + +if __name__ == "__main__": + app = LLMConfigApp() + app.run() diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8dae9e8 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,75 @@ +# LLM Inference Architecture + +This repository orchestrates high-throughput local inference using `llama-server` on an AMD ROCm GPU (~28-30GB usable VRAM). + +```mermaid +flowchart TD + subgraph Clients["Agents & Applications"] + PiAgent["Pi Coding Agent"] + HermesAgent["Hermes Agent"] + RofiUI["Rofi / Ghostty TUI"] + Curl["HTTP / OpenAI API Clients"] + end + + subgraph Router["Ports & Endpoints"] + P11434["Port 11434 (Planner / Reasoning / Solo)"] + P11435["Port 11435 (Worker #1 / Instruct)"] + P11436["Port 11436 (Worker #2 / Instruct)"] + end + + subgraph Models["Running Models"] + SuperQwen["SuperQwen 3.8 27B / Muse-Glimmer 30B
(Vision mmproj + Reasoning Preservation)"] + Worker1["Qwen 3.5 4B #1
(Vision mmproj + Instruct)"] + Worker2["Qwen 3.5 4B #2
(Vision mmproj + Instruct)"] + end + + PiAgent -->|"Planner (models.json)"| P11434 + PiAgent -->|"Worker-1 (models.json)"| P11435 + PiAgent -->|"Worker-2 (models.json)"| P11436 + HermesAgent --> P11434 + Curl --> P11434 + RofiUI -->|"Spawns & Monitors"| Router + + P11434 --> SuperQwen + P11435 --> Worker1 + P11436 --> Worker2 +``` + +## Multi-Agent Cluster Mode + +When running in cluster mode (`bin/llm-runner cluster` or via `llm-tui`): +1. **Planner / Reasoning Agent (`11434`)**: + - Model: `SuperQwen3.8-27B` (or `Qwen3.8-27B-UD-Q4_K_XL.gguf`) + - Purpose: High-level architectural reasoning, step planning, code review. + - Context Window: 65,536 tokens. + - Temperature: `0.6`, Top-P: `0.95`, Top-K: `40`, Min-P: `0.05`. + - Multimodal projector enabled for screenshot/diagram analysis. +2. **Worker Agents #1 & #2 (`11435` and `11436`)**: + - Model: `Qwen 3.5 4B` (`Qwen3.5-4B-Q5_K_M.gguf`) + - Purpose: Fast parallel file edits, tool calls, unit test generation. + - Context Window: 16,384 tokens each. + - Temperature: `0.7`, Top-P: `0.8`, Top-K: `20`, Min-P: `0.05`. + - Multimodal projector enabled for visual grounding. + +## Solo Mode + +For single-model deep thinking tasks requiring maximal context windows: +- **Muse-Glimmer 30B Solo (`bin/llm-runner muse`)**: + - Bound to port `11434`. + - Max native training context: **131,072 tokens (128K)**. + - Multimodal vision encoder enabled. +- **SuperQwen 3.8 27B Solo (`bin/llm-runner superqwen`)**: + - Bound to port `11434`. + - Max native context: **262,144 tokens (256K)**. + +## Integration with Coding Agents + +### Pi Agent Configuration +`dotfiles/config/pi/models.json` maps directly to these endpoints: +- `planner`: `http://localhost:11434/v1` +- `worker-1`: `http://localhost:11435/v1` +- `worker-2`: `http://localhost:11436/v1` +All models expose standard OpenAI-compatible endpoints: +- Completions: `/v1/chat/completions` +- Models: `/v1/models` +- Health: `/health` diff --git a/docs/parameter-reference.md b/docs/parameter-reference.md new file mode 100644 index 0000000..485067b --- /dev/null +++ b/docs/parameter-reference.md @@ -0,0 +1,43 @@ +# Hyperparameter & llama-server Reference + +This document explains key runtime flags used across `llm-runner` and the `llm-tui` form. + +--- + +## 1. Engine & Acceleration Flags + +| Flag | Recommended Value | Purpose | +| :--- | :--- | :--- | +| `-ngl, --n-gpu-layers` | `99` | Offload all layers to GPU VRAM. | +| `-fa, --flash-attn` | `on` | Enables Flash Attention kernel for faster generation and lower memory pressure. | +| `--load-mode` | `none` or `mmap` | Use `none` for large models under high VRAM pressure on ROCm to prevent memory stalls; use `mmap` for lighter models. | +| `-ctk` / `-ctv` | `q8_0` | Quantize KV cache vectors to 8-bit. | +| `-b, --batch-size` | `2048` | Logical batch size for prompt processing. | +| `-ub, --ubatch-size` | `512` | Micro-batch size for physical memory chunking. | +| `--reasoning-preserve` | `true` | Preserves `...` tags without stripping them during chat template processing. | + +--- + +## 2. Sampling Hyperparameters + +### Reasoning & Planning Profile (SuperQwen / Muse-Glimmer) +Optimized for logical step planning, architecture design, and math: +- **Temperature (`--temp`)**: `0.6` (prevents hallucination while allowing creative problem exploration) +- **Top-P (`--top-p`)**: `0.95` +- **Top-K (`--top-k`)**: `40` +- **Min-P (`--min-p`)**: `0.05` (truncates low-probability noise tokens) + +### Instruct / Implementation Profile (Qwen 3.5 4B) +Optimized for precise code edits, syntax consistency, and fast tool execution: +- **Temperature (`--temp`)**: `0.7` +- **Top-P (`--top-p`)**: `0.80` +- **Top-K (`--top-k`)**: `20` +- **Min-P (`--min-p`)**: `0.05` + +--- + +## 3. Multimodal / Vision Projector + +When `--mmproj ` is passed: +- Enables visual image understanding (screenshots, architecture diagrams, charts). +- `--image-min-tokens 1024`: Ensures sufficient patch token density for grounding and OCR clarity. diff --git a/docs/vram-budgeting.md b/docs/vram-budgeting.md new file mode 100644 index 0000000..808e06b --- /dev/null +++ b/docs/vram-budgeting.md @@ -0,0 +1,49 @@ +# VRAM Budgeting & Hardware Tuning (AMD ROCm) + +## Target Platform: AMD ROCm (~28 – 30GB Usable VRAM) + +Serving large reasoning models with long context windows requires careful VRAM accounting to prevent ROCm memory allocation faults or context truncation. + +--- + +## 1. Multi-Agent Cluster Footprint + +Running all 3 models concurrently on a single ~30GB GPU: + +| Component | Model / Vision Weights | Weights in VRAM | Context Size | KV Cache (`q8_0`) | Total VRAM | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **Planner (SuperQwen 27B)** | `Qwen3.8-27B-UD-Q4_K_XL` + `mmproj-Q8_0` | ~18.1 GB | 65,536 (64K) | ~2.6 GB | ~20.7 GB | +| **Worker #1 (Qwen 4B)** | `Qwen3.5-4B-Q5_K_M` + `mmproj-BF16` | ~3.9 GB | 16,384 (16K) | ~0.4 GB | ~4.3 GB | +| **Worker #2 (Qwen 4B)** | `Qwen3.5-4B-Q5_K_M` + `mmproj-BF16` | ~3.9 GB | 16,384 (16K) | ~0.4 GB | ~4.3 GB | +| **Total** | | **~25.9 GB** | | **~3.4 GB** | **~29.3 GB** | + +> [!NOTE] +> Headroom is approximately ~0.7 – 1.0 GB. Keep system desktop composition (Niri / Wayland) overhead low. + +--- + +## 2. Solo Model Memory Footprints + +### Muse-Glimmer 30B Solo +- **Weights Footprint**: ~17.5 GB (`UD-Q4_K_XL` + `mmproj-Q8_0`). +- **Context Ceiling**: 131,072 tokens (128K). +- **KV Cache Footprint (`q8_0`)**: ~8.5 GB at full 128K context. +- **Total Footprint**: ~26.0 GB. Comfortably fits inside 30GB VRAM with ~4GB headroom. + +### SuperQwen 3.8 27B Solo +- **Weights Footprint**: ~17.5 GB (`SuperQwen3.8-27b-abliterated-Q4_K_M` + `mmproj-Q8_0`). +- **Context Ceiling**: 262,144 tokens (256K). +- **KV Cache Footprint (`q8_0`)**: ~10.5 GB at full 256K context. +- **Total Footprint**: ~28.0 GB. Fits inside 30GB VRAM with ~2GB headroom. + +--- + +## 3. KV Cache Quantization (`-ctk` and `-ctv`) + +By default, llama.cpp stores key and value cache vectors in 16-bit floating point (`f16`). +For 64K–256K contexts, full `f16` KV caches require 20–40+ GB alone! + +Using **`-ctk q8_0 -ctv q8_0`**: +- **50% Memory Reduction**: Halves KV cache memory requirements. +- **Near-Lossless Perplexity**: Q8_0 maintains >99.8% precision with no degradation in reasoning benchmarks. +- **Native ROCm Acceleration**: Supported with Flash Attention (`-fa on`). diff --git a/models/Qwen3.5/chat_template.jinja b/models/Qwen3.5/chat_template.jinja new file mode 100644 index 0000000..0aae6d5 --- /dev/null +++ b/models/Qwen3.5/chat_template.jinja @@ -0,0 +1,363 @@ +{%- set template_version = "qwen3.8-froggeric-v22.1" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else false %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else 'medium' %} +{%- set _initial_thinking = enable_thinking %} +{%- set _initial_effort = 'medium' %} +{%- if _effort_raw == 'none' %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw == 'minimal' or _effort_raw == 'low' %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw == 'high' or _effort_raw == 'xhigh' or _effort_raw == 'max' %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set _first_role = messages[0].role %} +{%- if _first_role == 'system' or _first_role == 'developer' %} + {%- set _sys_msg = messages[0] %} + {%- set _msgs = messages[1:] %} +{%- else %} + {%- set _sys_msg = none %} + {%- set _msgs = messages %} +{%- endif %} +{%- set _sc = '' %} +{%- if _sys_msg is not none %} + {%- set _sc = render_content(_sys_msg.content, false, true) | trim %} + {%- if '<|think_off|>' in _sc %}{%- set _sc = _sc.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _sc %}{%- set _sc = _sc.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _sc %}{%- set _sc = _sc.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _sc %}{%- set _sc = _sc.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _sc %}{%- set _sc = _sc.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _sc %}{%- set _sc = _sc.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _sc %}{%- set _sc = _sc.split('<|think_minimal|>') | join('') | trim %}{%- endif %} +{%- endif %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set reasoning_content = message.thinking %} + {%- else %} + {%- set reasoning_content = message.thinking | string %} + {%- endif %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) and reasoning_content %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if not loop.first or content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string and tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is mapping or (args_value is sequence and args_value is not string) %} + {%- set _av = args_value | tojson %} + {%- else %} + {%- set _av = args_value | string %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- elif tc.arguments is string and tc.arguments %} + {{- tc.arguments }} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:80] %} + {%- if content | length < 500 and '$ ' not in content and 'took ' not in _content_lower and ('"error":' in _content_head or 'error:' in _content_head or 'err!' in _content_head or 'fatal:' in _content_head or 'exception:' in _content_head or 'traceback' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'failed to' in _content_head) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- if max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/models/Qwen3.8/chat_template.jinja b/models/Qwen3.8/chat_template.jinja new file mode 100644 index 0000000..cdd286e --- /dev/null +++ b/models/Qwen3.8/chat_template.jinja @@ -0,0 +1,363 @@ +{%- set template_version = "qwen3.8-froggeric-v22.1" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else 'medium' %} +{%- set _initial_thinking = enable_thinking %} +{%- set _initial_effort = 'medium' %} +{%- if _effort_raw == 'none' %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw == 'minimal' or _effort_raw == 'low' %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw == 'high' or _effort_raw == 'xhigh' or _effort_raw == 'max' %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set _first_role = messages[0].role %} +{%- if _first_role == 'system' or _first_role == 'developer' %} + {%- set _sys_msg = messages[0] %} + {%- set _msgs = messages[1:] %} +{%- else %} + {%- set _sys_msg = none %} + {%- set _msgs = messages %} +{%- endif %} +{%- set _sc = '' %} +{%- if _sys_msg is not none %} + {%- set _sc = render_content(_sys_msg.content, false, true) | trim %} + {%- if '<|think_off|>' in _sc %}{%- set _sc = _sc.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _sc %}{%- set _sc = _sc.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _sc %}{%- set _sc = _sc.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _sc %}{%- set _sc = _sc.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _sc %}{%- set _sc = _sc.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _sc %}{%- set _sc = _sc.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _sc %}{%- set _sc = _sc.split('<|think_minimal|>') | join('') | trim %}{%- endif %} +{%- endif %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set reasoning_content = message.thinking %} + {%- else %} + {%- set reasoning_content = message.thinking | string %} + {%- endif %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) and reasoning_content %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if not loop.first or content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string and tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is mapping or (args_value is sequence and args_value is not string) %} + {%- set _av = args_value | tojson %} + {%- else %} + {%- set _av = args_value | string %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- elif tc.arguments is string and tc.arguments %} + {{- tc.arguments }} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:80] %} + {%- if content | length < 500 and '$ ' not in content and 'took ' not in _content_lower and ('"error":' in _content_head or 'error:' in _content_head or 'err!' in _content_head or 'fatal:' in _content_head or 'exception:' in _content_head or 'traceback' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'failed to' in _content_head) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- if max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..abb5ec5 --- /dev/null +++ b/models/README.md @@ -0,0 +1,33 @@ +# Model Storage Directory + +This directory stores GGUF quantized model weights and vision encoders (`mmproj`). +Model binary weights (`*.gguf`) are excluded from Git version control via `.gitignore`. + +## Expected Directory Layout + +``` +models/ +├── Qwen3.5/ +│ ├── Qwen3.5-4B-Q5_K_M.gguf # Instruct / Worker Agent +│ ├── mmproj-BF16.gguf # Multimodal Vision Projector +│ └── chat_template.jinja -> ../../templates/Qwen3.5/chat_template.jinja +├── Qwen3.8/ +│ ├── Qwen3.8-27B-UD-Q4_K_XL.gguf # Reasoning / Planning Agent +│ ├── SuperQwen3.8-27b-abliterated-Q4_K_M.gguf # Alternative 4-bit quant +│ ├── mmproj-SuperQwen3.8-27b-abliterated-Q8_0.gguf +│ └── chat_template.jinja -> ../../templates/Qwen3.8/chat_template.jinja +└── muse-glimmer/ + ├── Muse-Glimmer-30B-UD-Q4_K_XL.gguf # Deep Reasoning Solo Model + ├── mmproj-Muse-Glimmer-30B-Q8_0.gguf # Vision Projector + └── chat_template.jinja -> ../../templates/muse-glimmer/chat_template.jinja +``` + +## Recommended Quantizations & Sourcing + +| Model Name | Quantization | Size | Purpose | Recommended Port | +| :--- | :--- | :--- | :--- | :--- | +| **SuperQwen 3.8 27B** | `UD-Q4_K_XL` or `Q4_K_M` | ~17-18 GB | Planner / Architect agent | `11434` | +| **Qwen 3.5 4B** | `Q5_K_M` | ~3.9 GB | Instruct / Implementation agent | `11435`, `11436` | +| **Muse-Glimmer 30B** | `UD-Q4_K_XL` | ~17 GB | Deep reasoning solo agent | `11434` | + +All chat templates are tracked in `templates/` and referenced automatically by `bin/llm-runner`. diff --git a/models/muse-glimmer/chat_template.jinja b/models/muse-glimmer/chat_template.jinja new file mode 100644 index 0000000..7507f3c --- /dev/null +++ b/models/muse-glimmer/chat_template.jinja @@ -0,0 +1,211 @@ +{# + Template: Muse Glimmer ATEM Chat Template + Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool + channels (to=), and the user channel, plus tool definitions and the + valid-recipient list in the system block. + + Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so + the indentation below is purely for readability and contributes nothing to + the rendered output. +#} +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|patch|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {%- if args is not mapping -%} + {{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}} + {%- endif -%} + {{- '\n\n' -}} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%} + true + {%- else -%} + false + {%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {{- '\n' -}} +{%- endmacro -%} +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} +{%- endmacro -%} +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} +{{- bos_token -}} +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} +{%- if not ns.has_system -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} +{%- endif -%} +{%- for message in messages -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + {%- if role == 'system' -%} + {#- Callers sometimes write the directive into the system prompt themselves. + Normalise "Reasoning effort" to "Reasoning strength" (jinja has no + case-insensitive replace, hence the four realistic casings), then skip + the kwarg-driven line below if the prompt already carries one. -#} + {%- set sys_text = render_content(message['content']) + | replace('Reasoning effort', 'Reasoning strength') + | replace('Reasoning Effort', 'Reasoning Strength') + | replace('reasoning effort', 'reasoning strength') + | replace('REASONING EFFORT', 'REASONING STRENGTH') -%} + {{- '<|start|>system<|message|>' -}} + {{- sys_text -}} + {%- if 'reasoning strength' not in (sys_text | lower) -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- endif -%} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%} diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..ba51a82 --- /dev/null +++ b/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Backwards-compatible wrapper delegating to bin/llm-runner cluster +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/bin/llm-runner" cluster "$@" diff --git a/run_muse.sh b/run_muse.sh new file mode 100755 index 0000000..8d1d0f4 --- /dev/null +++ b/run_muse.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Backwards-compatible wrapper delegating to bin/llm-runner muse +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/bin/llm-runner" muse "$@" diff --git a/run_superqwen.sh b/run_superqwen.sh new file mode 100755 index 0000000..ab55ef9 --- /dev/null +++ b/run_superqwen.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Backwards-compatible wrapper delegating to bin/llm-runner superqwen +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/bin/llm-runner" superqwen "$@" diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..60f63bf --- /dev/null +++ b/shell.nix @@ -0,0 +1,16 @@ +{ pkgs ? import {} }: let + deps = with pkgs; [ + zsh + (python312.withPackages(p: with p; [ + pip + numpy + ninja + tokenizers + ])) + pkgs.uv + rustc + cargo + ]; +in pkgs.mkShell rec { + buildInputs = deps; +} diff --git a/templates/Qwen3.5/chat_template.jinja b/templates/Qwen3.5/chat_template.jinja new file mode 100644 index 0000000..0aae6d5 --- /dev/null +++ b/templates/Qwen3.5/chat_template.jinja @@ -0,0 +1,363 @@ +{%- set template_version = "qwen3.8-froggeric-v22.1" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else false %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else 'medium' %} +{%- set _initial_thinking = enable_thinking %} +{%- set _initial_effort = 'medium' %} +{%- if _effort_raw == 'none' %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw == 'minimal' or _effort_raw == 'low' %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw == 'high' or _effort_raw == 'xhigh' or _effort_raw == 'max' %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set _first_role = messages[0].role %} +{%- if _first_role == 'system' or _first_role == 'developer' %} + {%- set _sys_msg = messages[0] %} + {%- set _msgs = messages[1:] %} +{%- else %} + {%- set _sys_msg = none %} + {%- set _msgs = messages %} +{%- endif %} +{%- set _sc = '' %} +{%- if _sys_msg is not none %} + {%- set _sc = render_content(_sys_msg.content, false, true) | trim %} + {%- if '<|think_off|>' in _sc %}{%- set _sc = _sc.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _sc %}{%- set _sc = _sc.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _sc %}{%- set _sc = _sc.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _sc %}{%- set _sc = _sc.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _sc %}{%- set _sc = _sc.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _sc %}{%- set _sc = _sc.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _sc %}{%- set _sc = _sc.split('<|think_minimal|>') | join('') | trim %}{%- endif %} +{%- endif %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set reasoning_content = message.thinking %} + {%- else %} + {%- set reasoning_content = message.thinking | string %} + {%- endif %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) and reasoning_content %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if not loop.first or content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string and tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is mapping or (args_value is sequence and args_value is not string) %} + {%- set _av = args_value | tojson %} + {%- else %} + {%- set _av = args_value | string %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- elif tc.arguments is string and tc.arguments %} + {{- tc.arguments }} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:80] %} + {%- if content | length < 500 and '$ ' not in content and 'took ' not in _content_lower and ('"error":' in _content_head or 'error:' in _content_head or 'err!' in _content_head or 'fatal:' in _content_head or 'exception:' in _content_head or 'traceback' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'failed to' in _content_head) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- if max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/templates/Qwen3.8/chat_template.jinja b/templates/Qwen3.8/chat_template.jinja new file mode 100644 index 0000000..cdd286e --- /dev/null +++ b/templates/Qwen3.8/chat_template.jinja @@ -0,0 +1,363 @@ +{%- set template_version = "qwen3.8-froggeric-v22.1" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else 'medium' %} +{%- set _initial_thinking = enable_thinking %} +{%- set _initial_effort = 'medium' %} +{%- if _effort_raw == 'none' %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw == 'minimal' or _effort_raw == 'low' %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw == 'high' or _effort_raw == 'xhigh' or _effort_raw == 'max' %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set _first_role = messages[0].role %} +{%- if _first_role == 'system' or _first_role == 'developer' %} + {%- set _sys_msg = messages[0] %} + {%- set _msgs = messages[1:] %} +{%- else %} + {%- set _sys_msg = none %} + {%- set _msgs = messages %} +{%- endif %} +{%- set _sc = '' %} +{%- if _sys_msg is not none %} + {%- set _sc = render_content(_sys_msg.content, false, true) | trim %} + {%- if '<|think_off|>' in _sc %}{%- set _sc = _sc.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _sc %}{%- set _sc = _sc.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _sc %}{%- set _sc = _sc.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _sc %}{%- set _sc = _sc.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _sc %}{%- set _sc = _sc.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _sc %}{%- set _sc = _sc.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _sc %}{%- set _sc = _sc.split('<|think_minimal|>') | join('') | trim %}{%- endif %} +{%- endif %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- set reasoning_content = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set reasoning_content = message.thinking %} + {%- else %} + {%- set reasoning_content = message.thinking | string %} + {%- endif %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) and reasoning_content %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if not loop.first or content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string and tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is mapping or (args_value is sequence and args_value is not string) %} + {%- set _av = args_value | tojson %} + {%- else %} + {%- set _av = args_value | string %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- elif tc.arguments is string and tc.arguments %} + {{- tc.arguments }} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:80] %} + {%- if content | length < 500 and '$ ' not in content and 'took ' not in _content_lower and ('"error":' in _content_head or 'error:' in _content_head or 'err!' in _content_head or 'fatal:' in _content_head or 'exception:' in _content_head or 'traceback' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'failed to' in _content_head) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- if max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/templates/muse-glimmer/chat_template.jinja b/templates/muse-glimmer/chat_template.jinja new file mode 100644 index 0000000..7507f3c --- /dev/null +++ b/templates/muse-glimmer/chat_template.jinja @@ -0,0 +1,211 @@ +{# + Template: Muse Glimmer ATEM Chat Template + Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool + channels (to=), and the user channel, plus tool definitions and the + valid-recipient list in the system block. + + Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so + the indentation below is purely for readability and contributes nothing to + the rendered output. +#} +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content -}} + {%- elif content is not none -%} + {%- for part in content -%} + {%- if part['type'] == 'image' -%} + {{- '<|patch|>' -}} + {%- elif part['type'] == 'video' -%} + {{- '<|video|>' -}} + {%- elif part['type'] == 'text' -%} + {{- part['text'] -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} +{%- macro render_atem(tc) -%} + {%- set args = tc.function.arguments -%} + {%- if args is not mapping -%} + {{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}} + {%- endif -%} + {{- '\n\n' -}} + {%- for k, v in args.items() -%} + {{- '' -}} + {%- if v is boolean -%} + {%- if v -%} + true + {%- else -%} + false + {%- endif -%} + {%- elif v is none -%} + null + {%- elif v is mapping or (v is iterable and v is not string) -%} + {{- v | tojson -}} + {%- else -%} + {{- v -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {{- '\n' -}} +{%- endmacro -%} +{%- macro render_tool_defs(tools) -%} + {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}} + {{- 'You can invoke a function by writing a "" block like the following:\n' -}} + {{- '\n\n$PARAMETER_VALUE\n...\n\n\n\n' -}} + {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}} + {{- 'Here are the functions available in JSONSchema format:\n' -}} + {{- '// Tool metadata\n' -}} + {%- set nsns = namespace(seen=[]) -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in nsns.seen -%} + {%- set nsns.seen = nsns.seen + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%} + {%- for tns in nsns.seen -%} + {{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}} + {%- endfor -%} + {{- '// Function schemas' -}} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}} + {%- endfor -%} + {{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}} + {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}} + {{- 'to=example_tool_name.example_function_name\n\n' -}} + {{- '\n\n' -}} + {{- 'value_1\n' -}} + {{- 'This is the value for the second parameter\nthat can span\n"multiple" lines\n\n' -}} + {{- '\n' -}} +{%- endmacro -%} +{%- macro render_reasoning() -%} + {%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%} + {{- 'Reasoning strength: ' + rs + '.' -}} +{%- endmacro -%} +{%- macro render_system_meta(tools) -%} + {%- set rns = namespace(recipients=['"self"'], nslist=[]) -%} + {%- if tools -%} + {%- for tool in tools -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set tns = fn.name.split('.')[0] -%} + {%- if tns not in rns.nslist -%} + {%- set rns.nslist = rns.nslist + [tns] -%} + {%- endif -%} + {%- endfor -%} + {%- for tns in rns.nslist -%} + {%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%} + {%- endfor -%} + {%- endif -%} + {%- set rns.recipients = rns.recipients + ['"user"'] -%} + {{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}} +{%- endmacro -%} +{{- bos_token -}} +{%- set ns = namespace(has_system=false) -%} +{%- for m in messages -%} + {%- if m['role'] == 'system' -%} + {%- set ns.has_system = true -%} + {%- endif -%} +{%- endfor -%} +{%- if not ns.has_system -%} + {{- '<|start|>system<|message|>You are a helpful AI assistant.' -}} + {%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%} + {{- '\nKnowledge cutoff: ' + kc + '.' -}} + {%- if current_date is defined and current_date -%} + {{- '\nCurrent date: ' + current_date + '.' -}} + {%- elif strftime_now is defined -%} + {{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} +{%- endif -%} +{%- for message in messages -%} + {%- set role = message['role'] -%} + {%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%} + {%- if role == 'system' -%} + {#- Callers sometimes write the directive into the system prompt themselves. + Normalise "Reasoning effort" to "Reasoning strength" (jinja has no + case-insensitive replace, hence the four realistic casings), then skip + the kwarg-driven line below if the prompt already carries one. -#} + {%- set sys_text = render_content(message['content']) + | replace('Reasoning effort', 'Reasoning strength') + | replace('Reasoning Effort', 'Reasoning Strength') + | replace('reasoning effort', 'reasoning strength') + | replace('REASONING EFFORT', 'REASONING STRENGTH') -%} + {{- '<|start|>system<|message|>' -}} + {{- sys_text -}} + {%- if 'reasoning strength' not in (sys_text | lower) -%} + {{- '\n\n' -}} + {{- render_reasoning() -}} + {%- endif -%} + {%- if tools -%} + {{- '\n\n' -}} + {{- render_tool_defs(tools) -}} + {%- endif -%} + {{- '\n\n' -}} + {{- render_system_meta(tools) -}} + {{- '<|eot|>' -}} + {%- elif role == 'user' -%} + {{- '<|start|>user<|message|>' -}} + {{- render_content(message['content']) -}} + {{- '<|eot|>' -}} + {%- elif role == 'tool' -%} + {%- set tname = message.get('name') -%} + {%- if not tname -%} + {%- set tcid = message.get('tool_call_id') -%} + {%- set rns = namespace(name=tcid if tcid else '') -%} + {%- for m in messages -%} + {%- if m.get('tool_calls') -%} + {%- for tc in m['tool_calls'] -%} + {%- if tcid is not none and tc.id is defined and tc.id == tcid -%} + {%- set rns.name = tc.function.name -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- set tname = rns.name -%} + {%- endif -%} + {{- '<|start|>tool ' + tname + '<|message|>\n' -}} + {{- render_content(message['content']) -}} + {{- '\n<|eot|>' -}} + {%- elif role == 'assistant' -%} + {%- if message.get('reasoning_content') -%} + {{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {%- for tc in message['tool_calls'] -%} + {{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}} + {{- render_atem(tc) -}} + {%- if loop.last -%} + {{- end_token -}} + {%- else -%} + {{- '<|eom|>' -}} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set recipient = message.get('recipient') or 'user' -%} + {%- set end_turn = message.get('end_turn') -%} + {%- if end_turn is none -%} + {%- set end_turn = not (recipient and recipient != 'user') -%} + {%- endif -%} + {{- '<|start|>assistant' -}} + {%- if recipient -%} + {{- ' to=' + recipient -}} + {%- endif -%} + {{- '<|message|>' -}} + {{- render_content(message['content']) -}} + {{- ('<|eot|>' if end_turn else '<|eom|>') -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- '<|start|>assistant' -}} +{%- endif -%}