Bonsai+kvmem可以在4060 mobile上跑128K上下文跑出约40t/s的速度

作者：#NukaColaM 板块：#开发调优 参考kvmem官方发布的[Bonsai 2 prism.3 — Experimental]版本及其说明。 本机配置为7945HX + 4060 mobile，8G显存，32G内存。 具体模型效果不做详评，但在我的应用场景中，已经可用。 实际运行效果，提示词是为了长输出乱写的，然后输出限制10240，被截断了。 5.49.621.948 I slot prompt eval time = 6097.37 ms / 1543 tokens ( 253.06 tokens per second), cache = 0 5.49.621.952 I slot eval time = 252377.37 ms / 10240 tokens ( 40.57 tokens per second) 5.49.621.953 I slot total time = 258474.73 ms / 11783 tokens 参考官方给出的启动脚本、启动参数以及相关说明做了部分修改，加了个chat_template和调整了目录结构。分bat和ps1 bat @echo off setlocal cd /d "%~dp0" rem Launch the PowerShell startup script with fixed parameters powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0start-bonsai-cli.ps1" ^ -Model "Ternary-Bonsai-2-27B-PTQ1_0-MTP-r3-Q4_0.gguf" ^ -Mmproj "mmproj-Qwen3.8-27B-Q5_K-MIX.gguf" ^ -ChatTemplateFile "chat_template.jinja" ^ -Gpu 0 ^ -Context 131072 ^ -Budget 24576 ^ -Reserve 10240 ^ -KvType q8_0 ^ -ReasoningBudget 4096 ^ -Mtp ^ -DraftTokens 1 ^ -DraftKvType f16 endlocal ps1 #Requires -Version 5.1 [CmdletBinding()] param( [Parameter(Mandatory = $true)][string]$Model, [string]$Mmproj, [string]$ChatTemplateFile, [string]$Gpu = '0', [ValidateRange(1, 65535)][int]$Port = 18202, [ValidateRange(128, 262144)][int]$Context = 131072, [ValidateRange(128, 262144)][int]$Budget = 24576, [ValidateRange(128, 262144)][int]$Reserve = 10240, [ValidateRange(1, 4096)][int]$Batch = 128, [ValidateSet('q8_0', 'q5_0', 'q4_0')][string]$KvType = 'q8_0', [ValidateRange(-1, 262144)][int]$ReasoningBudget = 4096, [switch]$Mtp, [ValidateRange(1, 2)][int]$DraftTokens = 1, [ValidateSet('f16', 'q8_0', 'q5_0', 'q4_0')][string]$DraftKvType = 'f16', [switch]$MmprojGpu, [switch]$DryRun ) $ErrorActionPreference = 'Stop' # Script root directory (i.e. ..\) $root = (Resolve-Path -LiteralPath $PSScriptRoot).Path # kvmem and models are relative to the script root $kvmemRoot = Join-Path $root 'kvmem' $modelsRoot = Join-Path $root 'models' $server = Join-Path $kvmemRoot 'bin\llama-kvmem-server.exe' $ui = Join-Path $kvmemRoot 'share\kvmem\ui' # Resolve a relative path: try models dir first, then script root, then CWD function Resolve-AssetPath { param( [string]$Path, [string]$Label = 'Path' ) if ([string]::IsNullOrWhiteSpace($Path)) { throw "$Label is empty." } if ([System.IO.Path]::IsPathRooted($Path)) { return (Resolve-Path -LiteralPath $Path).Path } # Try relative to models directory $candidate = Join-Path $modelsRoot $Path if (Test-Path -LiteralPath $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } # Try relative to script root $candidate = Join-Path $root $Path if (Test-Path -LiteralPath $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } # Fall back to the current working directory return (Resolve-Path -LiteralPath $Path).Path } # Validate fixed paths foreach ($path in @($server, $ui)) { if (-not (Test-Path -LiteralPath $path)) { throw "Missing required path: $path" } } # Resolve model path $Model = Resolve-AssetPath -Path $Model -Label 'Model' if (-not (Test-Path -LiteralPath $Model -PathType Leaf)) { throw "Model not found: $Model" } # Resolve mmproj path (if provided) if ($Mmproj) { $Mmproj = Resolve-AssetPath -Path $Mmproj -Label 'Mmproj' if (-not (Test-Path -LiteralPath $Mmproj -PathType Leaf)) { throw "Vision projector not found: $Mmproj" } } # Resolve chat template path (if provided) if ($ChatTemplateFile) { $ChatTemplateFile = Resolve-AssetPath -Path $ChatTemplateFile -Label 'ChatTemplateFile' if (-not (Test-Path -LiteralPath $ChatTemplateFile -PathType Leaf)) { throw "Chat template file not found: $ChatTemplateFile" } } if (($Budget % 128) -ne 0 -or ($