mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-06-17 10:15:27 -04:00
Merge remote-tracking branch 'origin/main' into visual-pr-playground
# Conflicts: # routes/cookbook_routes.py # routes/hwfit_routes.py # services/hwfit/fit.py # services/hwfit/models.py # static/js/cookbook-diagnosis.js # static/js/cookbook-hwfit.js # static/js/cookbook.js # static/js/cookbookRunning.js
This commit is contained in:
pewdiepie-archdaemon
+552
-102
@@ -7,6 +7,7 @@
|
||||
import uiModule from './ui.js';
|
||||
import { _diagnose, _showDiagnosis, _clearDiagnosis } from './cookbook-diagnosis.js';
|
||||
import { registerMenuDismiss } from './escMenuStack.js';
|
||||
import { computeProgressSignal } from './cookbookProgressSignal.js';
|
||||
|
||||
// Human-friendly badge label for a task's internal status. Avoids surfacing
|
||||
// the word "error" in the sidebar — a server the user stopped or one that
|
||||
@@ -34,12 +35,105 @@ function _taskBadge(task) {
|
||||
return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-' + task.status };
|
||||
}
|
||||
|
||||
function _canClearTask(task) {
|
||||
if (!task || task.status === 'running') return false;
|
||||
if (task.type === 'serve' && (task.status === 'ready' || task._serveReady)) return false;
|
||||
return ['done', 'stopped', 'error', 'crashed', 'failed'].includes(task.status);
|
||||
}
|
||||
|
||||
function _clearPillLabel(task) {
|
||||
return 'clear';
|
||||
}
|
||||
|
||||
function _shouldOfferCrashReport(task) {
|
||||
if (!task) return false;
|
||||
if (task._unreachable && task.type === 'serve') return true;
|
||||
return ['error', 'crashed', 'failed'].includes(task.status);
|
||||
}
|
||||
|
||||
function _serveTaskLooksAwqOnLocalBackend(task, outputText = '') {
|
||||
const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
|
||||
const cmd = `${task?.payload?._cmd || ''} ${outputText || ''}`.toLowerCase();
|
||||
return /\b(awq|gptq|fp8)\b/.test(repo) && /(llama-server|llama_cpp\.server|ollama|ggml_cuda_enable_unified_memory)/.test(cmd);
|
||||
}
|
||||
|
||||
function _serveTaskLooksAwqWithoutUsableAccelerator(task, outputText = '') {
|
||||
const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
|
||||
const out = String(outputText || '').toLowerCase();
|
||||
return /\b(awq|gptq|fp8)\b/.test(repo)
|
||||
&& /(no accelerator|no cuda runtime|failed to infer device type|triton is not supported|0 active driver)/i.test(out);
|
||||
}
|
||||
|
||||
async function _openDownloadForGgufTask(task) {
|
||||
const raw = task?.payload?.repo_id || task?.name || '';
|
||||
const modelName = String(raw)
|
||||
.split('/').pop()
|
||||
.replace(/[-_](?:AWQ|GPTQ|FP8|4bit|8bit|Int4|Int8).*$/i, '')
|
||||
.replace(/[-_]+$/g, '')
|
||||
|| String(raw).split('/').pop()
|
||||
|| raw;
|
||||
const cookbook = window.cookbookModule;
|
||||
if (cookbook && typeof cookbook.open === 'function') {
|
||||
cookbook.open({ tab: 'Search' });
|
||||
} else {
|
||||
document.getElementById('tool-cookbook-btn')?.click();
|
||||
}
|
||||
setTimeout(async () => {
|
||||
const modal = document.getElementById('cookbook-modal');
|
||||
const tab = modal?.querySelector('.cookbook-tab[data-backend="Search"]');
|
||||
if (tab && !tab.classList.contains('active')) tab.click();
|
||||
const search = document.getElementById('hwfit-search');
|
||||
if (search) {
|
||||
search.value = modelName;
|
||||
search.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
search.focus();
|
||||
}
|
||||
const quant = document.getElementById('hwfit-quant');
|
||||
if (quant) {
|
||||
quant.value = 'Q4_K_M';
|
||||
quant.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
try {
|
||||
const hwfit = await import('./cookbook-hwfit.js');
|
||||
if (typeof hwfit._hwfitFetch === 'function') hwfit._hwfitFetch(true);
|
||||
} catch {}
|
||||
}, 80);
|
||||
}
|
||||
|
||||
function _terminalServeDiagnosis(task, outputText) {
|
||||
const out = String(outputText || task?.output || '');
|
||||
if (!task || task.type !== 'serve' || !['stopped', 'error', 'crashed', 'failed'].includes(task.status) || !out.trim()) return null;
|
||||
if (_serveTaskLooksAwqOnLocalBackend(task, out)) {
|
||||
return {
|
||||
message: 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.',
|
||||
suggestion: 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.',
|
||||
fixes: [
|
||||
{ label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
|
||||
{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
|
||||
],
|
||||
};
|
||||
}
|
||||
if (_serveTaskLooksAwqWithoutUsableAccelerator(task, out)) {
|
||||
return {
|
||||
message: 'AWQ/GPTQ/FP8 needs a working vLLM/SGLang accelerator path; this server did not expose one.',
|
||||
suggestion: 'Suggested action: choose a CUDA/ROCm server where vLLM/SGLang can see the GPU, or download a GGUF version and serve it with llama.cpp/Ollama.',
|
||||
fixes: [
|
||||
{ label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
|
||||
{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
|
||||
],
|
||||
};
|
||||
}
|
||||
return _diagnose(out) || {
|
||||
message: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
|
||||
? 'llama.cpp build stopped before the server became reachable.'
|
||||
: 'Serve stopped before the model became reachable.',
|
||||
suggestion: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
|
||||
? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
|
||||
: 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
|
||||
fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
|
||||
};
|
||||
}
|
||||
|
||||
function _redactCrashReportText(text) {
|
||||
if (!text) return '';
|
||||
return String(text)
|
||||
@@ -136,6 +230,7 @@ const SERVE_STATE_KEY = 'cookbook-serve-state';
|
||||
const TASK_POLL_INTERVAL_MS = 3000; // delay between reconnect-loop iterations
|
||||
const BG_MONITOR_INTERVAL_MS = 10000; // background task status poll
|
||||
const STALE_PROGRESS_MS = 5 * 60 * 1000; // download with no progress this long = stale
|
||||
const STARTUP_STALE_PROGRESS_MS = 45 * 1000; // 0%-forever startup stall: retry much sooner
|
||||
|
||||
// ── Phase detection (mirrors Python _parse_serve_phase in cookbook_routes.py) ──
|
||||
// Single source of truth for serve task status. KEEP IN SYNC with the Python version.
|
||||
@@ -172,6 +267,23 @@ export function _parseServePhase(snapshot) {
|
||||
if (/Ollama API ready on port\s+\d+/i.test(flat)) {
|
||||
return { phase: 'ready', status: 'ready' };
|
||||
}
|
||||
const llamaBuildMatches = [...flat.matchAll(/\[\s*(\d{1,3})%\]\s*(?:Building|Linking)/gi)];
|
||||
if (llamaBuildMatches.length) {
|
||||
const pct = Math.min(100, parseInt(llamaBuildMatches[llamaBuildMatches.length - 1][1], 10));
|
||||
return { phase: `building llama.cpp ${pct}%`, status: 'running', pct };
|
||||
}
|
||||
if (/Native llama-server not found|building from source/i.test(flat)) {
|
||||
if (/Cloning into ['"]?llama\.cpp/i.test(flat) && !/Receiving objects:\s*100%/i.test(flat)) {
|
||||
return { phase: 'cloning llama.cpp', status: 'running' };
|
||||
}
|
||||
if (/Configuring incomplete|CMake Error/i.test(flat)) {
|
||||
return {};
|
||||
}
|
||||
if (/CMAKE_BUILD_TYPE|Detecting CXX|Found Threads|Including CPU backend|CUDA nvcc found|building llama-server/i.test(flat)) {
|
||||
return { phase: 'configuring llama.cpp', status: 'running' };
|
||||
}
|
||||
return { phase: 'building llama.cpp', status: 'running' };
|
||||
}
|
||||
// HTTP access logs (e.g. GET /v1/models 200 OK) mean the server is up
|
||||
if (/(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*\d{3}/.test(flat)) {
|
||||
return { phase: 'idle', status: 'ready' };
|
||||
@@ -264,10 +376,40 @@ function _refreshModelsAfterEndpointChange() {
|
||||
}, 1500);
|
||||
}
|
||||
|
||||
function _appendCookbookEndpointScope(fd, remoteHost) {
|
||||
const host = String(remoteHost || '').trim();
|
||||
if (!host || host === 'local' || host === 'localhost' || host === '127.0.0.1') {
|
||||
fd.append('container_local', 'true');
|
||||
}
|
||||
}
|
||||
|
||||
function _connectHostFromRemote(remoteHost, fallback = 'localhost') {
|
||||
const host = String(remoteHost || '').trim();
|
||||
if (!host || host === 'local') return fallback;
|
||||
return host.includes('@') ? host.split('@').pop() : host;
|
||||
}
|
||||
|
||||
function _isAnyBindHost(host) {
|
||||
const h = String(host || '').trim().toLowerCase();
|
||||
return h === '0.0.0.0' || h === '::' || h === '[::]';
|
||||
}
|
||||
|
||||
function _endpointFromAdvertisedUrl(rawUrl, currentHost, fallbackPort = '11434') {
|
||||
try {
|
||||
const u = new URL(rawUrl);
|
||||
const host = _isAnyBindHost(u.hostname) ? currentHost : (u.hostname || currentHost);
|
||||
const port = u.port || fallbackPort;
|
||||
const bracketedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
|
||||
return { host, port, baseUrl: `${u.protocol}//${bracketedHost}${port ? `:${port}` : ''}/v1` };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Download queue — runs one at a time per server ──
|
||||
|
||||
function _processQueue() {
|
||||
const tasks = _loadTasks();
|
||||
const tasks = _loadPrunedTasks();
|
||||
const running = tasks.filter(t => t.type === 'download' && t.status === 'running');
|
||||
const queued = tasks.filter(t => t.type === 'download' && t.status === 'queued');
|
||||
if (!queued.length) return;
|
||||
@@ -321,14 +463,24 @@ async function _startQueuedDownload(task) {
|
||||
return;
|
||||
}
|
||||
const oldId = task.sessionId;
|
||||
const tasks = _loadTasks();
|
||||
const t = tasks.find(t => t.sessionId === oldId);
|
||||
if (t) {
|
||||
t.sessionId = data.session_id;
|
||||
t.id = data.session_id;
|
||||
t.status = 'running';
|
||||
_saveTasks(tasks);
|
||||
}
|
||||
const launchedTask = { ...task, sessionId: data.session_id, id: data.session_id, status: 'running' };
|
||||
const key = _downloadDedupeKey(launchedTask);
|
||||
let found = false;
|
||||
const tasks = _loadTasks().filter(t => {
|
||||
if (t.sessionId === oldId) {
|
||||
found = true;
|
||||
t.sessionId = data.session_id;
|
||||
t.id = data.session_id;
|
||||
t.status = 'running';
|
||||
t._startLaunched = true;
|
||||
return true;
|
||||
}
|
||||
if (t.sessionId === data.session_id) return false;
|
||||
return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
|
||||
});
|
||||
if (!found) tasks.push(_stripTaskSecrets(launchedTask));
|
||||
_saveTasks(tasks);
|
||||
_renderRunningTab();
|
||||
_startBackgroundMonitor();
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
_renderRunningTab();
|
||||
@@ -340,11 +492,74 @@ async function _startQueuedDownload(task) {
|
||||
|
||||
// ── Task CRUD ──
|
||||
|
||||
function _serveOutputLooksReady(task) {
|
||||
const out = String(task?.output || '');
|
||||
return !!task?._serveReady
|
||||
|| /Application startup complete/i.test(out)
|
||||
|| /Ollama API ready on port\s+\d+/i.test(out)
|
||||
|| /(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*2\d\d/i.test(out);
|
||||
}
|
||||
|
||||
function _normalizeTaskForDisplay(task) {
|
||||
if (!task || typeof task !== 'object') return task;
|
||||
if (task.type === 'serve' && task.status === 'done' && !_serveOutputLooksReady(task)) {
|
||||
return { ...task, status: 'error' };
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
export function _loadTasks() {
|
||||
try { return JSON.parse(localStorage.getItem(TASKS_KEY)) || []; }
|
||||
try { return (JSON.parse(localStorage.getItem(TASKS_KEY)) || []).map(_normalizeTaskForDisplay); }
|
||||
catch { return []; }
|
||||
}
|
||||
|
||||
function _downloadRepoKey(task) {
|
||||
return String(task?.payload?.repo_id || task?.repo_id || task?.repo || task?.name || '').trim();
|
||||
}
|
||||
|
||||
function _downloadHostKey(task) {
|
||||
return String(task?.remoteHost || task?.payload?.remote_host || 'local').trim() || 'local';
|
||||
}
|
||||
|
||||
function _downloadDedupeKey(task) {
|
||||
if (!task || task.type !== 'download') return '';
|
||||
const repo = _downloadRepoKey(task);
|
||||
if (!repo) return '';
|
||||
return `${_downloadHostKey(task)}\n${repo}`;
|
||||
}
|
||||
|
||||
function _pruneQueuedDownloadDuplicates(tasks) {
|
||||
if (!Array.isArray(tasks) || !tasks.length) return tasks || [];
|
||||
const launched = new Set();
|
||||
for (const task of tasks) {
|
||||
if (task?.type !== 'download' || task.status === 'queued') continue;
|
||||
const key = _downloadDedupeKey(task);
|
||||
if (key) launched.add(key);
|
||||
}
|
||||
|
||||
let changed = false;
|
||||
const seenQueued = new Set();
|
||||
const next = tasks.filter(task => {
|
||||
if (task?.type !== 'download' || task.status !== 'queued') return true;
|
||||
const key = _downloadDedupeKey(task);
|
||||
if (!key) return true;
|
||||
if (launched.has(key) || seenQueued.has(key)) {
|
||||
changed = true;
|
||||
return false;
|
||||
}
|
||||
seenQueued.add(key);
|
||||
return true;
|
||||
});
|
||||
return changed ? next : tasks;
|
||||
}
|
||||
|
||||
function _loadPrunedTasks() {
|
||||
const tasks = _loadTasks();
|
||||
const pruned = _pruneQueuedDownloadDuplicates(tasks);
|
||||
if (pruned !== tasks) _saveTasks(pruned);
|
||||
return pruned;
|
||||
}
|
||||
|
||||
// Tombstones for removed tasks. Without these, removing a task only deletes it
|
||||
// locally — but the server still has it (its own POST guard even re-preserves
|
||||
// recently-added ones), so the next sync/poll merges it right back ("I removed
|
||||
@@ -407,6 +622,13 @@ export function _addTask(sessionId, name, type, payload) {
|
||||
const _repoId = payload.repo_id;
|
||||
tasks = tasks.filter(t => !(t.type === 'download' && t.status === 'done' && t.payload && t.payload.repo_id === _repoId));
|
||||
}
|
||||
if (type === 'download' && payload && payload.repo_id) {
|
||||
const key = _downloadDedupeKey({ type: 'download', payload, remoteHost });
|
||||
tasks = tasks.filter(t => {
|
||||
if (t.sessionId === sessionId) return false;
|
||||
return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
|
||||
});
|
||||
}
|
||||
const task = _stripTaskSecrets({ id: sessionId, sessionId, name, type, status: 'running', output: '', ts: Date.now(), payload: payload || null, remoteHost, sshPort, platform });
|
||||
tasks.push(task);
|
||||
_saveTasks(tasks);
|
||||
@@ -523,6 +745,52 @@ function _tmuxGracefulKill(task) {
|
||||
return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
|
||||
}
|
||||
|
||||
function _shQuote(value) {
|
||||
return "'" + String(value ?? '').replace(/'/g, "'\\''") + "'";
|
||||
}
|
||||
|
||||
function _taskLooksOllama(task, outputText = '') {
|
||||
const haystack = `${task?.payload?.backend || ''} ${task?.payload?._cmd || ''} ${task?.payload?._fields?.backend || ''} ${outputText || ''}`;
|
||||
return /\bollama\b/i.test(haystack) || /Ollama API ready on port\s+\d+/i.test(haystack);
|
||||
}
|
||||
|
||||
function _ollamaBaseUrlForTask(task, outputText = '') {
|
||||
const out = String(outputText || '');
|
||||
const ready = out.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
|
||||
if (ready) return ready[1].replace(/\/+$/, '');
|
||||
const cmd = String(task?.payload?._cmd || '');
|
||||
const host = cmd.match(/OLLAMA_HOST=([^\s]+)/)?.[1] || '';
|
||||
const port = host.match(/:(\d+)$/)?.[1] || '11434';
|
||||
return `http://127.0.0.1:${port}`;
|
||||
}
|
||||
|
||||
function _ollamaModelForTask(task) {
|
||||
return String(task?.payload?.model || task?.payload?.repo_id || task?.name || '').trim();
|
||||
}
|
||||
|
||||
function _ollamaUnloadCommand(task, outputText = '') {
|
||||
if (!_taskLooksOllama(task, outputText)) return '';
|
||||
const model = _ollamaModelForTask(task);
|
||||
if (!model) return '';
|
||||
const base = _ollamaBaseUrlForTask(task, outputText);
|
||||
const body = JSON.stringify({ model, prompt: '', keep_alive: 0, stream: false });
|
||||
const inner = `curl -sf -X POST ${_shQuote(base + '/api/generate')} -H 'Content-Type: application/json' -d ${_shQuote(body)} >/dev/null 2>&1 || true`;
|
||||
if (task.remoteHost) {
|
||||
return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
|
||||
function _endpointUrlForTask(task, outputText = '') {
|
||||
if (_taskLooksOllama(task, outputText)) {
|
||||
return _ollamaBaseUrlForTask(task, outputText) + '/v1';
|
||||
}
|
||||
const host = _connectHostFromRemote(task.remoteHost);
|
||||
const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
|
||||
const port = portMatch ? portMatch[1] : '8000';
|
||||
return `http://${host}:${port}/v1`;
|
||||
}
|
||||
|
||||
// ── Wave animation ──
|
||||
|
||||
const _waveFrames = ['▁▂▃', '▂▃▄', '▃▄▅', '▄▅▆', '▅▆▅', '▆▅▄', '▅▄▃', '▄▃▂', '▃▂▁'];
|
||||
@@ -781,17 +1049,23 @@ async function _retryTask(el, task) {
|
||||
body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
|
||||
});
|
||||
} catch {}
|
||||
_removeTask(task.sessionId);
|
||||
if (task.payload) {
|
||||
if (task.type === 'serve' && task.payload._cmd) {
|
||||
_removeTask(task.sessionId);
|
||||
_launchServeTask(task.name, task.payload.repo_id, task.payload._cmd, task.payload._fields, task.remoteHost || '');
|
||||
} else {
|
||||
_retryDownload(task.name, task.payload);
|
||||
uiModule.showToast('Retrying download — progress may look reset while HuggingFace checks cached files, then it should resume.', 7000);
|
||||
_updateTask(task.sessionId, {
|
||||
status: 'running',
|
||||
output: `${task.output || ''}\n\n[odysseus] Retrying download. Progress may briefly look like a fresh download while HuggingFace checks cached/incomplete files; cached partial files will be reused when available.`.trim(),
|
||||
_retrying: true,
|
||||
});
|
||||
_retryDownload(task.name, task.payload, task.sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function _retryDownload(name, payload) {
|
||||
async function _retryDownload(name, payload, replaceSessionId = '') {
|
||||
try {
|
||||
// A retry means the fast hf_transfer path already failed once — fall back to
|
||||
// the plain, reliable downloader for this and any further attempt (it resumes
|
||||
@@ -804,17 +1078,40 @@ async function _retryDownload(name, payload) {
|
||||
});
|
||||
if (!res.ok) {
|
||||
uiModule.showToast('Download failed: HTTP ' + res.status);
|
||||
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
if (!data.ok) {
|
||||
uiModule.showToast('Download failed: ' + (data.error || ''));
|
||||
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
|
||||
return;
|
||||
}
|
||||
_addTask(data.session_id, name, 'download', _payload);
|
||||
if (replaceSessionId) {
|
||||
const tasks = _loadTasks();
|
||||
const task = tasks.find(t => t.sessionId === replaceSessionId);
|
||||
if (task) {
|
||||
task.id = data.session_id;
|
||||
task.sessionId = data.session_id;
|
||||
task.status = 'running';
|
||||
task.output = '';
|
||||
task.ts = Date.now();
|
||||
task.payload = _payload;
|
||||
task._retrying = false;
|
||||
_saveTasks(tasks);
|
||||
_soloExpandTaskId = data.session_id;
|
||||
_renderRunningTab();
|
||||
_startBackgroundMonitor();
|
||||
} else {
|
||||
_addTask(data.session_id, name, 'download', _payload);
|
||||
}
|
||||
} else {
|
||||
_addTask(data.session_id, name, 'download', _payload);
|
||||
}
|
||||
uiModule.showToast(`Downloading ${name}...`);
|
||||
} catch (e) {
|
||||
uiModule.showToast('Download failed: ' + e.message);
|
||||
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -875,7 +1172,7 @@ export async function _serveAutoFix(panel, envVar) {
|
||||
// Edit button, but optionally with a modified command (used by the diagnosis
|
||||
// "Retry with X" buttons so a retry lands in the editable Serve panel with the
|
||||
// adjusted setting, instead of blindly relaunching).
|
||||
async function _openServeEditForTask(task, cmdOverride) {
|
||||
async function _openServeEditForTask(task, cmdOverride, fieldOverrides = null) {
|
||||
const repo = task.payload?.repo_id;
|
||||
if (!repo) { uiModule.showToast('No model info on this task'); return; }
|
||||
const cmd = cmdOverride || task.payload?._cmd;
|
||||
@@ -883,6 +1180,9 @@ async function _openServeEditForTask(task, cmdOverride) {
|
||||
let fields = cmdOverride
|
||||
? _parseServeCmdToFields(cmd)
|
||||
: (task.payload?._fields || (cmd ? _parseServeCmdToFields(cmd) : null));
|
||||
if (fieldOverrides && typeof fieldOverrides === 'object') {
|
||||
fields = { ...(fields || {}), ...fieldOverrides };
|
||||
}
|
||||
// Switch the active server to the one this serve ran on (mirrors _openEdit).
|
||||
const _tHost = task.remoteHost || '';
|
||||
_envState.remoteHost = _tHost;
|
||||
@@ -1062,12 +1362,27 @@ function _parseServeCmdToFields(cmd) {
|
||||
gpu_mem: ex(/--gpu-memory-utilization\s+([\d.]+)/) || '0.90',
|
||||
swap: ex(/--swap-space\s+(\d+)/) || '',
|
||||
dtype: ex(/--dtype\s+(\w+)/) || 'auto',
|
||||
vllm_kv_cache_dtype: ex(/--kv-cache-dtype\s+([\w.-]+)/) || 'auto',
|
||||
max_seqs: ex(/--max-num-seqs\s+(\d+)/) || '',
|
||||
gpus: ex(/CUDA_VISIBLE_DEVICES=(\S+)/) || '',
|
||||
cache_type: ex(/(?:--cache-type-k|-ctk)\s+(\S+)/) || '',
|
||||
llama_fit: ex(/(?:--fit|-fit)\s+(on|off)/) || '',
|
||||
llama_split_mode: ex(/(?:--split-mode|-sm)\s+(none|layer|row|tensor)/) || '',
|
||||
llama_tensor_split: ex(/(?:--tensor-split|-ts)\s+([0-9.,]+)/) || '',
|
||||
llama_main_gpu: ex(/(?:--main-gpu|-mg)\s+(\d+)/) || '',
|
||||
llama_parallel: ex(/(?:--parallel|-np)\s+(\d+)/) || '',
|
||||
llama_batch_size: ex(/(?:--batch-size|-b)\s+(\d+)/) || '',
|
||||
llama_ubatch_size: ex(/(?:--ubatch-size|-ub)\s+(\d+)/) || '',
|
||||
llama_spec_tokens: ex(/--spec-draft-n-max\s+(\d+)/) || '3',
|
||||
enforce_eager: cmd.includes('--enforce-eager'),
|
||||
trust_remote: cmd.includes('--trust-remote-code'),
|
||||
prefix_cache: cmd.includes('--enable-prefix-caching'),
|
||||
auto_tool: cmd.includes('--enable-auto-tool-choice'),
|
||||
flash_attn: /--flash-attn\s+on\b/.test(cmd),
|
||||
unified_mem: /GGML_CUDA_ENABLE_UNIFIED_MEMORY=1/.test(cmd),
|
||||
llama_no_mmap: /--no-mmap\b/.test(cmd),
|
||||
llama_no_warmup: /--no-warmup\b/.test(cmd),
|
||||
llama_speculative_mtp: /--spec-type\s+\S*draft-mtp/.test(cmd),
|
||||
speculative: cmd.includes('--speculative-config'),
|
||||
};
|
||||
const spec = cmd.match(/--speculative-config\s+'?\{[^}]*"method"\s*:\s*"([^"]+)"[^}]*"num_speculative_tokens"\s*:\s*(\d+)/);
|
||||
@@ -1181,7 +1496,7 @@ export function _renderRunningTab() {
|
||||
// event but the matching clear only ran on modal-open, so the highlight
|
||||
// persisted indefinitely after tasks finished in the background.
|
||||
try {
|
||||
const _activeTasks = _loadTasks().filter(t => t.status === 'running' || t.status === 'queued' || t.status === 'error');
|
||||
const _activeTasks = _loadPrunedTasks().filter(t => t.status === 'running' || t.status === 'queued' || t.status === 'error');
|
||||
if (!_activeTasks.length) _clearCookbookNotif();
|
||||
} catch {}
|
||||
|
||||
@@ -1222,6 +1537,8 @@ export function _renderRunningTab() {
|
||||
|
||||
const tasks = _loadTasks();
|
||||
const hasContent = tasks.length > 0;
|
||||
const activeCount = tasks.filter(t => t.status === 'running' || t.status === 'queued').length;
|
||||
const activeCountHtml = activeCount ? ` <span class="cookbook-tab-count">${activeCount}</span>` : '';
|
||||
|
||||
let tabBar = body.querySelector('.cookbook-tabs');
|
||||
if (!tabBar) return;
|
||||
@@ -1231,7 +1548,7 @@ export function _renderRunningTab() {
|
||||
runTab.className = 'cookbook-tab';
|
||||
runTab.dataset.backend = 'Running';
|
||||
const _errCount = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
|
||||
runTab.innerHTML = `Running <span class="cookbook-tab-count">${tasks.length}</span>${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`;
|
||||
runTab.innerHTML = `Running${activeCountHtml}${_errCount ? `<span class="cookbook-tab-error-dot"></span>` : ''}`;
|
||||
tabBar.insertBefore(runTab, tabBar.firstChild);
|
||||
runTab.addEventListener('click', () => {
|
||||
tabBar.querySelectorAll('.cookbook-tab').forEach(t => t.classList.remove('active'));
|
||||
@@ -1242,7 +1559,7 @@ export function _renderRunningTab() {
|
||||
});
|
||||
} else if (runTab) {
|
||||
const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
|
||||
runTab.innerHTML = tasks.length ? `Running <span class="cookbook-tab-count">${tasks.length}</span>${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Running';
|
||||
runTab.innerHTML = tasks.length ? `Running${activeCountHtml}${_errCount2 ? '<span class="cookbook-tab-error-dot"></span>' : ''}` : 'Running';
|
||||
if (!hasContent) {
|
||||
if (runTab.classList.contains('active')) {
|
||||
const wfTab = tabBar.querySelector('.cookbook-tab[data-backend="Search"]');
|
||||
@@ -1259,7 +1576,7 @@ export function _renderRunningTab() {
|
||||
group.dataset.backendGroup = 'Running';
|
||||
group.innerHTML = '<div class="admin-card" style="flex:1;display:flex;flex-direction:column;overflow:hidden;">' +
|
||||
'<div style="display:flex;align-items:baseline;gap:8px;margin-bottom:2px;">' +
|
||||
'<h2 style="margin:0;padding:0;line-height:1;">Running <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">' + tasks.length + '</span></h2>' +
|
||||
'<h2 style="margin:0;padding:0;line-height:1;">Running <span id="running-count" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal">' + activeCount + '</span></h2>' +
|
||||
'</div>' +
|
||||
'<p class="memory-desc doclib-desc" style="margin-top:6px;">Active downloads and serving processes.</p>' +
|
||||
'</div>';
|
||||
@@ -1271,7 +1588,7 @@ export function _renderRunningTab() {
|
||||
if (!group) return;
|
||||
|
||||
const countEl = group.querySelector('#running-count');
|
||||
if (countEl) countEl.textContent = tasks.length;
|
||||
if (countEl) countEl.textContent = activeCount;
|
||||
|
||||
if (!hasContent) {
|
||||
group.remove();
|
||||
@@ -1351,8 +1668,8 @@ export function _renderRunningTab() {
|
||||
const host = btn.dataset.clearServer;
|
||||
if (!await window.styledConfirm(`Clear finished tasks on ${_serverName(host)}?`, { confirmText: 'Clear' })) return;
|
||||
const allTasks = _loadTasks();
|
||||
const toRemove = allTasks.filter(t => (t.remoteHost || '') === host && t.status !== 'running');
|
||||
const remaining = allTasks.filter(t => (t.remoteHost || '') !== host || t.status === 'running');
|
||||
const toRemove = allTasks.filter(t => (t.remoteHost || '') === host && _canClearTask(t));
|
||||
const remaining = allTasks.filter(t => (t.remoteHost || '') !== host || !_canClearTask(t));
|
||||
_saveTasks(remaining);
|
||||
// Fade/slide each finished card out (same exit as the per-card clear)
|
||||
// instead of yanking them instantly.
|
||||
@@ -1389,6 +1706,9 @@ export function _renderRunningTab() {
|
||||
const running = _loadTasks().filter(t => (t.remoteHost || '') === host && t.status === 'running');
|
||||
if (!running.length) { uiModule.showToast(`Nothing running on ${_serverName(host)}`); return; }
|
||||
if (!await window.styledConfirm(`Stop ${running.length} running task${running.length > 1 ? 's' : ''} on ${_serverName(host)}?`, { confirmText: 'Stop all' })) return;
|
||||
// Mark every task as user-stopped BEFORE firing the kills so that the
|
||||
// download auto-retry logic never restarts a task the user just stopped.
|
||||
running.forEach(t => _updateTask(t.sessionId, { _userStopped: true }));
|
||||
// Reuse each task's own Stop action so it does the full teardown
|
||||
// (send C-c, drop the endpoint, mark stopped) consistently.
|
||||
running.forEach(t => {
|
||||
@@ -1442,16 +1762,21 @@ export function _renderRunningTab() {
|
||||
const _bdg = _taskBadge(task);
|
||||
badge.textContent = _bdg.text;
|
||||
badge.className = 'cookbook-task-status' + (_bdg.cls ? ' ' + _bdg.cls : '');
|
||||
badge.style.display = isDone ? 'none' : ''; // hidden — type chip carries it
|
||||
badge.style.display = '';
|
||||
}
|
||||
// Indicator: spinning wave while running, green check when finished.
|
||||
const wave = el.querySelector('.cookbook-task-wave');
|
||||
if (wave) wave.style.display = task.status === 'running' ? '' : 'none';
|
||||
// Model downloads (which have a Serve → button) don't get a clear pill —
|
||||
// pressing Serve clears them. Dep installs / serve tasks keep it.
|
||||
const check = el.querySelector('.cookbook-task-check');
|
||||
const _showClear = isDone && !(task.type === 'download' && !task.payload?._dep);
|
||||
if (check) check.style.display = _showClear ? '' : 'none';
|
||||
if (check) {
|
||||
check.style.display = _canClearTask(task) ? '' : 'none';
|
||||
const label = check.querySelector('.cookbook-task-done-label');
|
||||
if (label) label.textContent = _clearPillLabel(task);
|
||||
}
|
||||
const startNow = el.querySelector('.cookbook-task-start-now');
|
||||
if (startNow) startNow.style.display = (task.type === 'download' && task.status === 'queued') ? '' : 'none';
|
||||
const terminalDiag = _terminalServeDiagnosis(task, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
|
||||
if (terminalDiag) _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
|
||||
}
|
||||
if (!task) {
|
||||
if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
|
||||
@@ -1475,20 +1800,21 @@ export function _renderRunningTab() {
|
||||
<div class="cookbook-task-header">
|
||||
<span class="cookbook-task-type${(task.status === 'done' && task.type === 'download') ? ' cookbook-task-type-done' : ''}" data-type="${esc(task.type)}">${esc((task.status === 'done' && task.type === 'download') ? 'finished' : task.type)}</span>
|
||||
<span class="cookbook-task-name">${modelLogo(task.name)}${esc(task.name)}</span>
|
||||
<span class="cookbook-task-status ${_bdg.cls}" style="display:${task.status === 'done' ? 'none' : ''}"${_bdgTitle}>${esc(_bdg.text)}</span>
|
||||
${task.type === 'serve' && task.payload?._cmd ? '<button class="cookbook-task-edit-btn" title="Edit settings & relaunch"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.121 2.121 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg></button>' : ''}
|
||||
${task.type === 'serve' && task.payload?._cmd ? '<button class="cookbook-task-save-btn" title="Save preset"><svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg></button>' : ''}
|
||||
<span class="cookbook-task-indicator"><span class="cookbook-task-wave" style="display:${task.status === 'running' ? '' : 'none'}"></span><span class="cookbook-task-check" title="Clear" style="display:${(task.status === 'done' && !(task.type === 'download' && !task.payload?._dep)) ? '' : 'none'}"><svg class="cookbook-task-check-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg><svg class="cookbook-task-clear-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg><span class="cookbook-task-done-label">done</span><span class="cookbook-task-clear-label">clear</span></span></span>
|
||||
${task.type === 'download' && !task.payload?._dep && task.status === 'done' ? `<span class="cookbook-task-status cookbook-task-done">finished</span>` : ''}
|
||||
<span class="cookbook-task-indicator"><span class="cookbook-task-wave" style="display:${task.status === 'running' ? '' : 'none'}"></span><span class="cookbook-task-check" title="Clear" style="display:${_canClearTask(task) ? '' : 'none'}"><svg class="cookbook-task-check-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg><svg class="cookbook-task-clear-ico" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg><span class="cookbook-task-done-label">${esc(_clearPillLabel(task))}</span><span class="cookbook-task-clear-label">clear</span></span></span>
|
||||
<button type="button" class="cookbook-task-start-now" title="Start this queued download now" style="display:${(task.type === 'download' && task.status === 'queued') ? '' : 'none'}"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><polygon points="8 5 19 12 8 19 8 5"/></svg><span>start now</span></button>
|
||||
<span class="cookbook-task-status ${_bdg.cls}"${_bdgTitle}>${esc(_bdg.text)}</span>
|
||||
<button class="cookbook-task-menu-btn" title="Actions">⋮</button>
|
||||
</div>
|
||||
<div class="cookbook-task-sub"><span class="cookbook-task-session">${esc(task.sessionId)}</span><span class="cookbook-task-uptime" style="display:${((task.type === 'serve' || task.type === 'download') && task.status === 'running') ? '' : 'none'}"></span></div>
|
||||
<div class="cookbook-task-sub"><span class="cookbook-task-session">${esc(task.sessionId)}</span><span class="cookbook-task-uptime" style="display:${((task.type === 'serve' || task.type === 'download') && task.status === 'running') ? '' : 'none'}"></span>${(task.type === 'download') ? `<span class="cookbook-task-dldir" title="Download destination" style="font-size:9px;color:var(--fg-muted);font-family:'Fira Code',monospace;opacity:0.4;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:40ch;">Dir: ${esc(task.payload?.local_dir || '~/.cache/huggingface/hub')}</span>` : ''}</div>
|
||||
<div class="cookbook-output-wrap cookbook-task-collapsible${_mobileCollapseDefault ? ' cookbook-task-collapsed' : ''}"><pre class="cookbook-output-pre">${esc(task.output || '')}</pre><button type="button" class="copy-code cookbook-output-copy"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button></div>
|
||||
`;
|
||||
|
||||
const _waveEl = el.querySelector('.cookbook-task-wave');
|
||||
if (_waveEl && task.status === 'running') _registerWaveEl(_waveEl);
|
||||
|
||||
const terminalDiag = _terminalServeDiagnosis(task, task.output || '');
|
||||
if (terminalDiag) _showDiagnosis(el, terminalDiag, task.output || '');
|
||||
|
||||
const _uptimeEl = el.querySelector('.cookbook-task-uptime');
|
||||
if (_uptimeEl && (task.type === 'serve' || task.type === 'download') && task.status === 'running') {
|
||||
const _startedAt = task.ts || Date.now();
|
||||
@@ -1505,35 +1831,12 @@ export function _renderRunningTab() {
|
||||
}
|
||||
|
||||
// Re-open the Serve panel for this model, pre-filled with the EXACT
|
||||
// settings this instance launched with, and on the SERVER it runs on —
|
||||
// shared by the edit icon button and the ⋮ "Edit settings" menu item.
|
||||
// settings this instance launched with, and on the SERVER it runs on.
|
||||
const _openEdit = () => _openServeEditForTask(task);
|
||||
const editBtn = el.querySelector('.cookbook-task-edit-btn');
|
||||
if (editBtn) {
|
||||
editBtn.addEventListener('click', (e) => { e.stopPropagation(); _openEdit(); });
|
||||
}
|
||||
|
||||
// Wire save icon button
|
||||
const saveBtn = el.querySelector('.cookbook-task-save-btn');
|
||||
if (saveBtn) {
|
||||
saveBtn.addEventListener('click', async (e) => {
|
||||
e.stopPropagation();
|
||||
// Tell them it's already saved up front (often true now that working
|
||||
// configs auto-save) instead of after they've typed a name.
|
||||
if (_loadPresets().some(p => p.cmd === task.payload?._cmd)) {
|
||||
uiModule.showToast('Already saved');
|
||||
return;
|
||||
}
|
||||
const label = (await uiModule.styledPrompt('Name this config so you can recall it later.', {
|
||||
title: 'Save Config', defaultValue: task.name, placeholder: 'e.g. 8-bit, fast', confirmText: 'Save',
|
||||
}) || '').trim();
|
||||
if (!label) return;
|
||||
if (!_saveTaskAsPreset(task, label)) { uiModule.showToast('Already saved'); return; }
|
||||
saveBtn.innerHTML = '<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="2.5" stroke-linecap="round"><polyline points="20 6 9 17 4 12"/></svg>';
|
||||
uiModule.showToast(`Saved "${label}"`);
|
||||
setTimeout(() => { saveBtn.style.display = 'none'; }, 1500);
|
||||
});
|
||||
}
|
||||
el.addEventListener('cookbook:edit-serve', (e) => {
|
||||
e.stopPropagation();
|
||||
_openServeEditForTask(task, null, e.detail?.fields || null);
|
||||
});
|
||||
|
||||
// Finished download → an explicit "Serve →" button jumps straight to the
|
||||
// Serve tab with this model pre-selected (on the server it downloaded to).
|
||||
@@ -1571,10 +1874,30 @@ export function _renderRunningTab() {
|
||||
if (_clearChk) {
|
||||
_clearChk.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
// Belt-and-suspenders: kill the tmux session too. For a real-finished
|
||||
// task the session is already gone and kill-session errors silently,
|
||||
// but for a task that was falsely flagged done (the strict-finish
|
||||
// bug), this guarantees the still-running download actually stops
|
||||
// rather than continuing to write to disk after the row is removed.
|
||||
try {
|
||||
fetch('/api/shell/exec', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${task.sessionId}`) }),
|
||||
}).catch(() => {});
|
||||
} catch {}
|
||||
_animateOutThenRemove(el, task.sessionId);
|
||||
});
|
||||
}
|
||||
|
||||
const _startNowBtn = el.querySelector('.cookbook-task-start-now');
|
||||
if (_startNowBtn) {
|
||||
_startNowBtn.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
_startQueuedDownload(task);
|
||||
});
|
||||
}
|
||||
|
||||
// Wire header click to collapse/expand output
|
||||
el.querySelector('.cookbook-task-header').addEventListener('click', (e) => {
|
||||
if (e.target.closest('button')) return;
|
||||
@@ -1675,8 +1998,7 @@ export function _renderRunningTab() {
|
||||
// serve to the model-endpoints list regardless of prior flag state.
|
||||
if (task.type === 'serve' && task.payload?._cmd) {
|
||||
items.push({ label: 'Register endpoint', action: 'register-endpoint', custom: async () => {
|
||||
const rawHost = task.remoteHost || 'localhost';
|
||||
const host = rawHost.includes('@') ? rawHost.split('@').pop() : rawHost;
|
||||
const host = _connectHostFromRemote(task.remoteHost);
|
||||
const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
|
||||
const port = portMatch ? portMatch[1] : '8000';
|
||||
const baseUrl = `http://${host}:${port}/v1`;
|
||||
@@ -1699,6 +2021,7 @@ export function _renderRunningTab() {
|
||||
fd.append('base_url', baseUrl);
|
||||
fd.append('name', task.name);
|
||||
fd.append('skip_probe', 'true');
|
||||
_appendCookbookEndpointScope(fd, task.remoteHost || '');
|
||||
if (task.payload?._cmd?.includes('diffusion_server')) fd.append('model_type', 'image');
|
||||
const res = await fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
|
||||
if (res.ok) {
|
||||
@@ -1859,13 +2182,21 @@ export function _renderRunningTab() {
|
||||
const badge = el.querySelector('.cookbook-task-status');
|
||||
if (badge) { badge.textContent = 'stopping...'; badge.className = 'cookbook-task-status cookbook-task-stopping'; }
|
||||
el.dataset.status = 'stopped';
|
||||
_updateTask(task.sessionId, { _userStopped: true });
|
||||
const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
|
||||
// Drop the model endpoint so the picker stops listing it.
|
||||
if (task.type === 'serve' && task.payload) {
|
||||
const rawHost = task.remoteHost || 'localhost';
|
||||
const host = rawHost.includes('@') ? rawHost.split('@').pop() : rawHost;
|
||||
const portMatch = task.payload._cmd?.match(/--port\s+(\d+)/);
|
||||
const port = portMatch ? portMatch[1] : '8000';
|
||||
_removeEndpointByUrl(`http://${host}:${port}/v1`);
|
||||
_removeEndpointByUrl(_endpointUrlForTask(task, outputText));
|
||||
}
|
||||
const ollamaUnload = _ollamaUnloadCommand(task, outputText);
|
||||
if (ollamaUnload) {
|
||||
try {
|
||||
await fetch('/api/shell/exec', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ command: ollamaUnload }),
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
// Gracefully stop (C-c, then kill the session) so it's fully down...
|
||||
try {
|
||||
@@ -1882,23 +2213,29 @@ export function _renderRunningTab() {
|
||||
|
||||
// Wire kill
|
||||
el.querySelector('.cookbook-task-action-kill').addEventListener('click', () => {
|
||||
const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || '';
|
||||
const ollamaUnload = _ollamaUnloadCommand(task, outputText);
|
||||
if (ollamaUnload) {
|
||||
fetch('/api/shell/exec', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ command: ollamaUnload }),
|
||||
}).catch(() => {});
|
||||
}
|
||||
fetch('/api/shell/exec', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
|
||||
}).catch(() => {});
|
||||
if (task.type === 'serve' && task.payload) {
|
||||
const rawHost = task.remoteHost || 'localhost';
|
||||
const host = rawHost.includes('@') ? rawHost.split('@').pop() : rawHost;
|
||||
const portMatch = task.payload._cmd?.match(/--port\s+(\d+)/);
|
||||
const port = portMatch ? portMatch[1] : '8000';
|
||||
_removeEndpointByUrl(`http://${host}:${port}/v1`);
|
||||
const endpointUrl = _endpointUrlForTask(task, outputText);
|
||||
_removeEndpointByUrl(endpointUrl);
|
||||
const modelName = task.payload.model || task.name || '';
|
||||
if (modelName) {
|
||||
fetch('/api/model-endpoints', { credentials: 'same-origin' })
|
||||
.then(r => r.json())
|
||||
.then(eps => {
|
||||
const ep = eps.find(e => e.name === modelName || (e.base_url && e.base_url.includes(':' + port)));
|
||||
const ep = eps.find(e => e.name === modelName || e.base_url === endpointUrl);
|
||||
if (ep) fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' }).then(() => _refreshModelsAfterEndpointChange());
|
||||
}).catch(() => {});
|
||||
}
|
||||
@@ -2017,19 +2354,65 @@ async function _reconnectTask(el, task) {
|
||||
if (badge) { badge.textContent = _statusLabel('error', task.type); badge.className = 'cookbook-task-status cookbook-task-error'; }
|
||||
_showCookbookNotif(true);
|
||||
} else {
|
||||
const looksSuccessful = !lastOutput.includes('DOWNLOAD_FAILED') && (lastOutput.includes('DONE') || lastOutput.includes('100%') || lastOutput.includes('Application startup complete') || lastOutput.includes('/snapshots/') || lastOutput.includes('Download complete') || lastOutput.includes('DOWNLOAD_OK'));
|
||||
if (!lastOutput.trim() || (task.type === 'download' && !looksSuccessful)) {
|
||||
const downloadLooksSuccessful = !lastOutput.includes('DOWNLOAD_FAILED')
|
||||
&& (lastOutput.includes('DONE') || lastOutput.includes('100%') || lastOutput.includes('/snapshots/') || lastOutput.includes('Download complete') || lastOutput.includes('DOWNLOAD_OK'));
|
||||
const serveLooksReady = task.type === 'serve' && _serveOutputLooksReady({ ...task, output: lastOutput });
|
||||
const looksSuccessful = task.type === 'download' ? downloadLooksSuccessful : serveLooksReady;
|
||||
if (!lastOutput.trim() || !looksSuccessful) {
|
||||
_updateTask(task.sessionId, { status: 'crashed' });
|
||||
el.dataset.status = 'crashed';
|
||||
const badge = el.querySelector('.cookbook-task-status');
|
||||
if (badge) { badge.textContent = _statusLabel('crashed', task.type); badge.className = 'cookbook-task-status cookbook-task-crashed'; }
|
||||
if (task.type === 'serve') {
|
||||
const diag = _diagnose(lastOutput) || {
|
||||
message: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
|
||||
? 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.'
|
||||
: /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
|
||||
? 'llama.cpp build stopped before the server became reachable.'
|
||||
: 'Serve stopped before the model became reachable.',
|
||||
suggestion: _serveTaskLooksAwqOnLocalBackend(task, lastOutput)
|
||||
? 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.'
|
||||
: /Native llama-server not found|building llama-server|llama\.cpp/i.test(lastOutput)
|
||||
? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
|
||||
: 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
|
||||
fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
|
||||
};
|
||||
_showDiagnosis(el, diag, lastOutput);
|
||||
} else if (task.type === 'download') {
|
||||
const isDisk = /no space left|disk quota|enospc/i.test(lastOutput);
|
||||
const isNetwork = /connection|timeout|timed out|incompleteread|chunkedencoding|reset by peer|protocolerror|all connection attempts failed/i.test(lastOutput);
|
||||
const progressMatch = String(lastOutput || '').match(/(\d+)%\|/);
|
||||
const nearDone = progressMatch && Number(progressMatch[1]) >= 80;
|
||||
const diag = {
|
||||
message: isDisk
|
||||
? 'Download stopped because this server ran out of disk space.'
|
||||
: isNetwork
|
||||
? 'Download stopped after the HuggingFace connection was interrupted.'
|
||||
: nearDone
|
||||
? 'Download stopped near the end before the final completion marker was captured.'
|
||||
: 'Download stopped before HuggingFace reported completion.',
|
||||
suggestion: isDisk
|
||||
? 'Suggested action: free disk space, then retry the download. HuggingFace resumes incomplete files when possible.'
|
||||
: nearDone
|
||||
? 'Suggested action: retry the download. It may briefly look like it restarted while cached files are checked, then it should reuse incomplete files.'
|
||||
: 'Suggested action: retry the download. HuggingFace resumes incomplete files when possible.',
|
||||
fixes: [
|
||||
{ label: 'Retry download', action: () => _retryTask(el, task) },
|
||||
{ label: 'Copy last 50 lines', action: () => {
|
||||
const last = String(lastOutput || '').split('\n').slice(-50).join('\n');
|
||||
_copyText(last || 'No download log available.');
|
||||
} },
|
||||
],
|
||||
};
|
||||
_showDiagnosis(el, diag, lastOutput);
|
||||
}
|
||||
_showCookbookNotif(true);
|
||||
} else {
|
||||
_updateTask(task.sessionId, { status: 'done' });
|
||||
el.dataset.status = 'done';
|
||||
const badge = el.querySelector('.cookbook-task-status');
|
||||
if (badge) { badge.textContent = _statusLabel('done', task.type); badge.className = 'cookbook-task-status cookbook-task-done'; }
|
||||
const _chk = el.querySelector('.cookbook-task-check'); if (_chk && task.type !== 'download') _chk.style.display = '';
|
||||
const _chk = el.querySelector('.cookbook-task-check'); if (_chk) _chk.style.display = '';
|
||||
const _sb = el.querySelector('.cookbook-task-serve-btn'); if (_sb) _sb.style.display = '';
|
||||
_showCookbookNotif();
|
||||
_refreshDepsAfterInstall(task);
|
||||
@@ -2071,10 +2454,17 @@ async function _reconnectTask(el, task) {
|
||||
// stale speed/ETA — so keying off speed masked real stalls (that's why a
|
||||
// 97%-stuck download went undetected). Bytes are the honest signal; fall
|
||||
// back to %/aggregate only when no byte counter is present.
|
||||
const _STALE_TIMEOUT = STALE_PROGRESS_MS;
|
||||
const _byteMatches = [...snapshot.matchAll(/([\d.]+\s?[KMGT])B?\s*\/\s*[\d.]+\s?[KMGT]B?/gi)];
|
||||
const _bytes = _byteMatches.length ? _byteMatches[_byteMatches.length - 1][1].replace(/\s/g, '') : null;
|
||||
const curProgress = _bytes || (_dlAgg != null ? String(_dlAgg) : (lastPct || '0'));
|
||||
// When there's no byte counter (pip resolve / native build phase of a
|
||||
// dependency install), key off the output tail so new build lines count
|
||||
// as progress — otherwise a long quiet build is falsely declared stale
|
||||
// and restarted mid-build, looping forever (#1568).
|
||||
const curProgress = computeProgressSignal(_bytes, _dlAgg, lastPct, snapshot);
|
||||
const _fetchPctMatches = [...snapshot.matchAll(/Fetching\s+\d+\s+files:\s*(\d+)%/g)];
|
||||
const _fetchPct = _fetchPctMatches.length ? parseInt(_fetchPctMatches[_fetchPctMatches.length - 1][1]) : null;
|
||||
const _startupStalled = !_bytes && ((_dlAgg === 0) || (_fetchPct === 0)) && curProgress === '0';
|
||||
const _STALE_TIMEOUT = _startupStalled ? STARTUP_STALE_PROGRESS_MS : STALE_PROGRESS_MS;
|
||||
if (!el._lastProgress) { el._lastProgress = curProgress; el._lastProgressTime = Date.now(); }
|
||||
if (curProgress !== el._lastProgress) {
|
||||
el._lastProgress = curProgress;
|
||||
@@ -2095,7 +2485,7 @@ async function _reconnectTask(el, task) {
|
||||
} else if (Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && !task._autoRestarted) {
|
||||
task._autoRestarted = true;
|
||||
_updateTask(task.sessionId, { _autoRestarted: true });
|
||||
badge.textContent = 'stale — restarting';
|
||||
badge.textContent = _startupStalled ? '0% stall — retrying' : 'stale — restarting';
|
||||
badge.className = 'cookbook-task-status cookbook-task-error';
|
||||
_showCookbookNotif(true);
|
||||
try {
|
||||
@@ -2139,14 +2529,37 @@ async function _reconnectTask(el, task) {
|
||||
break;
|
||||
}
|
||||
|
||||
// When the snapshot includes a shard-of-N marker (e.g.
|
||||
// "model-00006-of-00082.safetensors"), TRUE overall progress is
|
||||
// ((shard-1) + currentShardFraction) / totalShards. Before, _dlAgg
|
||||
// (hf_transfer's per-current-shard aggregate, e.g. 53% of shard 6)
|
||||
// was treated as overall and the row read "53%" while only 5 of
|
||||
// 82 shards were actually done.
|
||||
const _shardPat = [...snapshot.matchAll(/model-(\d+)-of-(\d+)\.(?:safetensors|bin)/g)];
|
||||
const _lastShard = _shardPat.length ? _shardPat[_shardPat.length - 1] : null;
|
||||
const _curShardNum = _lastShard ? parseInt(_lastShard[1], 10) : null;
|
||||
const _totalShards = _lastShard ? parseInt(_lastShard[2], 10) : null;
|
||||
const _useShardAgg = _curShardNum && _totalShards && _totalShards > 1;
|
||||
|
||||
// HF's own "Fetching N files: X%" aggregate counts ALL files,
|
||||
// including ones already finished in a previous session (resume) —
|
||||
// so on a resumed download it reflects the true overall progress,
|
||||
// whereas completed/totalFiles only see this session's files (→ 0%).
|
||||
// Take the higher of the two so resume doesn't read as 0%.
|
||||
const _fetchPctMatches = [...snapshot.matchAll(/Fetching\s+\d+\s+files:\s*(\d+)%/g)];
|
||||
const _fetchPct = _fetchPctMatches.length ? parseInt(_fetchPctMatches[_fetchPctMatches.length - 1][1]) : null;
|
||||
if (_dlAgg != null) {
|
||||
if (_useShardAgg) {
|
||||
// Multi-shard download: compute TRUE overall as completed shards
|
||||
// plus the current shard's fraction. _dlAgg / lastPct represent
|
||||
// *this shard's* progress, not the whole download.
|
||||
const curShardFrac = (_dlAgg != null)
|
||||
? _dlAgg / 100
|
||||
: (lastPct ? parseInt(lastPct, 10) / 100 : 0);
|
||||
let overallPct = Math.round((((_curShardNum - 1) + curShardFrac) / _totalShards) * 100);
|
||||
if (_fetchPct != null) overallPct = Math.max(overallPct, _fetchPct);
|
||||
let text = `${overallPct}%`;
|
||||
if (lastSpeed) text += ` · ${lastSpeed}`;
|
||||
badge.textContent = text;
|
||||
badge.className = 'cookbook-task-status cookbook-task-running';
|
||||
} else if (_dlAgg != null) {
|
||||
// Real aggregate byte progress — most accurate; take the max of all signals.
|
||||
let pct = _dlAgg;
|
||||
if (_fetchPct != null) pct = Math.max(pct, _fetchPct);
|
||||
@@ -2182,7 +2595,7 @@ async function _reconnectTask(el, task) {
|
||||
const _accessDenied = /Access to model.*is restricted|gated repo|GatedRepoError|401 Unauthorized|403 Forbidden|not in the authorized list|awaiting a review|must (?:be authenticated|have access)/i.test(snapshot);
|
||||
const _dlKey = task.payload?.repo_id || task.name;
|
||||
const _dlN = _dlRetryCount.get(_dlKey) || 0;
|
||||
if (!_accessDenied && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) {
|
||||
if (!_accessDenied && !task._userStopped && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) {
|
||||
// Auto-retry: kill the dead session and re-launch (resumes from
|
||||
// the cached .incomplete files) after a short delay.
|
||||
_dlRetryCount.set(_dlKey, _dlN + 1);
|
||||
@@ -2297,8 +2710,7 @@ async function _reconnectTask(el, task) {
|
||||
// first one's dedup check can observe the newly-added row.
|
||||
if (task.type === 'serve' && !task._endpointAdded && !task._endpointAddInFlight && task._serveReady) {
|
||||
task._endpointAddInFlight = true;
|
||||
const rawHost = task.remoteHost || 'localhost';
|
||||
let host = rawHost.includes('@') ? rawHost.split('@').pop() : rawHost;
|
||||
let host = _connectHostFromRemote(task.remoteHost);
|
||||
const portMatch = task.payload?._cmd?.match(/--port[=\s]+(\d+)/)
|
||||
|| task.payload?._cmd?.match(/(?:^|\s)-p[=\s]+(\d+)/)
|
||||
|| snapshot.match(/Uvicorn running on\D*?:(\d+)/i)
|
||||
@@ -2309,12 +2721,8 @@ async function _reconnectTask(el, task) {
|
||||
let baseUrl = `http://${host}:${port}/v1`;
|
||||
const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
|
||||
if (ollamaUrlMatch) {
|
||||
try {
|
||||
const u = new URL(ollamaUrlMatch[1]);
|
||||
host = u.hostname || host;
|
||||
port = u.port || '11434';
|
||||
baseUrl = `${u.origin}/v1`;
|
||||
} catch {}
|
||||
const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
|
||||
if (endpoint) ({ host, port, baseUrl } = endpoint);
|
||||
}
|
||||
fetch('/api/model-endpoints', { credentials: 'same-origin' })
|
||||
.then(r => r.json())
|
||||
@@ -2342,6 +2750,7 @@ async function _reconnectTask(el, task) {
|
||||
fd.append('base_url', baseUrl);
|
||||
fd.append('name', task.name);
|
||||
fd.append('skip_probe', 'true');
|
||||
_appendCookbookEndpointScope(fd, task.remoteHost || '');
|
||||
if (_isDiffusion) fd.append('model_type', 'image');
|
||||
return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
|
||||
})
|
||||
@@ -2445,8 +2854,7 @@ async function _checkServeReachability() {
|
||||
]);
|
||||
} catch { return; }
|
||||
for (const task of serveTasks) {
|
||||
const rawHost = task.remoteHost || 'localhost';
|
||||
const host = rawHost.includes('@') ? rawHost.split('@').pop() : rawHost;
|
||||
const host = _connectHostFromRemote(task.remoteHost);
|
||||
const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
|
||||
const port = portMatch ? portMatch[1] : '8000';
|
||||
const baseUrl = `http://${host}:${port}/v1`;
|
||||
@@ -2641,6 +3049,52 @@ async function _pollBackgroundStatus() {
|
||||
const data = await res.json();
|
||||
const tasks = data.tasks || [];
|
||||
|
||||
// Reconcile the authoritative tmux/process status back into the persisted
|
||||
// client task list. The Running-tab reconnect loop also does this, but it
|
||||
// only exists while cards are rendered; after a page refresh or closed modal
|
||||
// dependency installs could finish server-side while localStorage stayed
|
||||
// stuck at "running".
|
||||
try {
|
||||
const statusById = new Map(tasks.map(t => [t.session_id, t]));
|
||||
const localTasks = _loadTasks();
|
||||
let changed = false;
|
||||
const completedDeps = [];
|
||||
for (const task of localTasks) {
|
||||
const live = statusById.get(task.sessionId);
|
||||
if (!live) continue;
|
||||
const updates = {};
|
||||
const nextStatus = live.status === 'completed'
|
||||
? 'done'
|
||||
: (live.status === 'error'
|
||||
? 'error'
|
||||
: (live.status === 'stopped' ? (task.type === 'download' ? 'crashed' : 'stopped') : null));
|
||||
if (nextStatus && task.status !== nextStatus) {
|
||||
updates.status = nextStatus;
|
||||
if (nextStatus === 'done' && task.payload?._dep) completedDeps.push(task);
|
||||
}
|
||||
if ((live.status === 'running' || live.status === 'ready') && task.status !== live.status) {
|
||||
updates.status = live.status === 'ready' ? 'ready' : 'running';
|
||||
}
|
||||
if (live.progress && live.progress !== task.progress) updates.progress = live.progress;
|
||||
if (live.output_tail) {
|
||||
const previous = String(task.output || '');
|
||||
const tail = String(live.output_tail || '');
|
||||
if (tail && !previous.endsWith(tail)) {
|
||||
updates.output = `${previous ? `${previous}\n` : ''}${tail}`.slice(-5000);
|
||||
}
|
||||
}
|
||||
if (Object.keys(updates).length) {
|
||||
Object.assign(task, updates);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) {
|
||||
_saveTasks(localTasks);
|
||||
_renderRunningTab();
|
||||
completedDeps.forEach(t => _refreshDepsAfterInstall(t));
|
||||
}
|
||||
} catch (_) { /* non-fatal: background status should never break polling */ }
|
||||
|
||||
const statusEl = document.getElementById('cookbook-bg-status');
|
||||
const activeTasks = tasks.filter(t => t.status === 'running' || t.status === 'ready');
|
||||
const errorTasks = tasks.filter(t => t.status === 'error');
|
||||
@@ -2653,8 +3107,7 @@ async function _pollBackgroundStatus() {
|
||||
const localTask = localTasks.find(lt => lt.sessionId === t.session_id);
|
||||
if (localTask && localTask._endpointAdded) continue;
|
||||
|
||||
const rawHost = localTask?.remoteHost || t.remote || 'localhost';
|
||||
let host = rawHost.includes('@') ? rawHost.split('@').pop() : (rawHost === 'local' ? 'localhost' : rawHost);
|
||||
let host = _connectHostFromRemote(localTask?.remoteHost || t.remote);
|
||||
const portMatch = localTask?.payload?._cmd?.match(/--port\s+(\d+)/)
|
||||
|| localTask?.payload?._cmd?.match(/OLLAMA_HOST=[^\s:]+:(\d+)/);
|
||||
let port = portMatch ? portMatch[1] : '8000';
|
||||
@@ -2662,12 +3115,8 @@ async function _pollBackgroundStatus() {
|
||||
const snapshot = t.output || localTask?.output || '';
|
||||
const ollamaUrlMatch = snapshot.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
|
||||
if (ollamaUrlMatch) {
|
||||
try {
|
||||
const u = new URL(ollamaUrlMatch[1]);
|
||||
host = u.hostname || host;
|
||||
port = u.port || '11434';
|
||||
baseUrl = `${u.origin}/v1`;
|
||||
} catch {}
|
||||
const endpoint = _endpointFromAdvertisedUrl(ollamaUrlMatch[1], host, '11434');
|
||||
if (endpoint) ({ host, port, baseUrl } = endpoint);
|
||||
}
|
||||
const _isDiffusion = localTask?.payload?._cmd?.includes('diffusion_server');
|
||||
|
||||
@@ -2698,6 +3147,7 @@ async function _pollBackgroundStatus() {
|
||||
fd.append('base_url', baseUrl);
|
||||
fd.append('name', t.model);
|
||||
fd.append('skip_probe', 'true');
|
||||
_appendCookbookEndpointScope(fd, localTask?.remoteHost || t.remote || '');
|
||||
if (_isDiffusion) fd.append('model_type', 'image');
|
||||
if (_supportsTools) fd.append('supports_tools', 'true');
|
||||
return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
|
||||
|
||||
Reference in New Issue
Block a user