Salastil/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-06-17 02:05:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: add vllm kv cache dtype option (#1185)

Browse Source
This commit is contained in:
spooky
2026-06-03 00:17:16 +10:00
committed by GitHub
parent 7b43fa9372
commit 5b87e69221
4 changed files with 16 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/js/cookbook.js
+2
View File
@@ -366,6 +366,8 @@ export function _buildServeCmd(f, modelName, backend) {
cmd += ` --gpu-memory-utilization ${f.gpu_mem || '0.90'}`;
if (f.swap && f.swap !== '0') cmd += ` --swap-space ${f.swap}`;
cmd += ` --dtype ${f.dtype || 'auto'}`;
const _kv = (f.vllm_kv_cache_dtype ?? '').toString().trim();
if (_kv === 'fp8') cmd += ' --kv-cache-dtype fp8';
if (f.max_seqs && f.max_seqs.toString().trim()) cmd += ` --max-num-seqs ${f.max_seqs.toString().trim()}`;
if (f.enforce_eager) cmd += ' --enforce-eager';
if (f.trust_remote) cmd += ' --trust-remote-code';