feat(ui): add real-time diagnostic logs console (#974)

* feat(diagnostics): add admin-gated real-time diagnostics logs terminal UI

* feat(ui): resolve diagnostics logs feedback and optimize client-side caching

* feat(ui): resolve diagnostics logs feedback
This commit is contained in:
Kfir Sadeh
2026-06-15 11:32:51 +03:00
committed by GitHub
parent f7e2d0c0b7
commit d8e7cc7053
6 changed files with 518 additions and 6 deletions
+31 -4
View File
@@ -69,10 +69,37 @@ from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_imag
from starlette.responses import RedirectResponse
# ========= LOGGING =========
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
)
import logging.handlers
from core.constants import DATA_DIR
_root_logger = logging.getLogger()
_root_logger.setLevel(logging.INFO)
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Clear existing handlers to avoid duplicates
for _h in list(_root_logger.handlers):
_root_logger.removeHandler(_h)
_console_h = logging.StreamHandler()
_console_h.setFormatter(_formatter)
_root_logger.addHandler(_console_h)
try:
_log_dir = os.path.join(DATA_DIR, "logs")
os.makedirs(_log_dir, exist_ok=True)
_log_file = os.path.join(_log_dir, "app.log")
# RotatingFileHandler is not multi-process safe (e.g. if uvicorn is run with --workers N).
# Odysseus is single-process by convention, so this is acceptable, but be aware that
# concurrent log rotation issues can arise if multiple workers are configured.
_file_h = logging.handlers.RotatingFileHandler(
_log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
)
_file_h.setFormatter(_formatter)
_root_logger.addHandler(_file_h)
except Exception as e:
_root_logger.warning(f"Failed to initialize file logging handler (falling back to console-only): {e}")
logger = logging.getLogger(__name__)
# ========= APP =========
+26 -1
View File
@@ -1,12 +1,13 @@
"""Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research."""
import logging
import os
from typing import Dict, Any
from fastapi import APIRouter, HTTPException, Form, Request
from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async
from core.constants import DEFAULT_HOST
from core.constants import DEFAULT_HOST, DATA_DIR
from core.middleware import require_admin
logger = logging.getLogger(__name__)
@@ -28,6 +29,30 @@ def setup_diagnostics_routes(
from src.service_health import collect_service_health
return await collect_service_health(rag_manager, memory_vector)
@router.get("/api/diagnostics/logs")
async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]:
require_admin(request)
limit = max(1, min(limit, 1000))
try:
log_file = os.path.join(DATA_DIR, "logs", "app.log")
if not os.path.exists(log_file):
return {"status": "success", "logs": []}
# Safe tail read of the log file (max 5MB via rotation)
with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
lines = f.readlines()
tail_lines = lines[-limit:] if len(lines) > limit else lines
tail_lines = [line.rstrip('\r\n') for line in tail_lines]
return {
"status": "success",
"logs": tail_lines
}
except Exception as e:
logger.error(f"Diagnostics logs retrieval error: {e}")
raise HTTPException(500, f"Failed to retrieve logs: {str(e)}")
@router.get("/api/db/stats")
async def get_database_stats(request: Request) -> Dict[str, Any]:
require_admin(request)
+55
View File
@@ -2232,6 +2232,61 @@
<!-- ═══ SYSTEM TAB ═══ -->
<div data-settings-panel="system" class="hidden">
<div class="admin-card" id="settings-system-logs-card">
<h2>
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-svg">
<polyline points="4 17 10 11 4 5"></polyline>
<line x1="12" y1="19" x2="20" y2="19"></line>
</svg>
Terminal Logs
</h2>
<div class="admin-toggle-sub settings-system-logs-toggle-sub">Live diagnostic logs and system output from the Odysseus process.</div>
<div class="settings-col settings-system-logs-col">
<!-- Controls row -->
<div class="settings-system-logs-controls">
<!-- Search input -->
<input type="text" id="log-search-input" placeholder="Search logs..." class="settings-system-logs-search">
<!-- Level select -->
<select id="log-level-select" class="settings-system-logs-select">
<option value="ALL">All Levels</option>
<option value="INFO">INFO</option>
<option value="WARNING">WARNING</option>
<option value="ERROR">ERROR</option>
<option value="DEBUG">DEBUG</option>
</select>
<!-- Limit select -->
<select id="log-limit-select" class="settings-system-logs-select">
<option value="100">100 lines</option>
<option value="200" selected>200 lines</option>
<option value="500">500 lines</option>
<option value="1000">1000 lines</option>
</select>
<!-- Refresh Button -->
<button type="button" class="admin-btn-sm" id="log-refresh-btn">
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-refresh-svg"><path d="M21.5 2v6h-6M21.34 15.57a10 10 0 1 1-.57-8.38l5.67-5.67"/></svg>
Refresh
</button>
<!-- Auto-refresh switch -->
<div class="settings-system-logs-autopoll-container">
<label class="admin-switch" title="Auto-polling every 3 seconds">
<input type="checkbox" id="log-auto-refresh-toggle">
<span class="admin-slider"></span>
</label>
<span>Auto-poll</span>
</div>
</div>
<!-- Console container -->
<div id="log-console-container">
<div class="settings-system-logs-placeholder">Initializing logs terminal viewer...</div>
</div>
</div>
</div>
<div class="admin-card">
<h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>Data Backup</h2>
<div class="admin-toggle-sub" style="margin-bottom:8px">Export or import your user data (memories, presets, settings, skills, preferences) as a JSON file.</div>
+197 -1
View File
@@ -2488,12 +2488,206 @@ function initDangerZone() {
});
}
/* ═══════════════════════════════════════════
TERMINAL LOGS VIEWER
═══════════════════════════════════════════ */
let logsPollInterval = null;
let isLogsPolling = false;
let cachedLogs = [];
let logsAbortController = null;
function renderLogs(isAutoPoll = false) {
const consoleContainer = el('log-console-container');
const levelSelect = el('log-level-select');
const searchInput = el('log-search-input');
if (!consoleContainer) return;
const levelFilter = levelSelect ? levelSelect.value : 'ALL';
const searchQuery = searchInput ? searchInput.value.trim().toLowerCase() : '';
let logs = cachedLogs;
// Filter by level locally
if (levelFilter !== 'ALL') {
logs = logs.filter(line => line.includes(` - ${levelFilter} - `));
}
// Filter by search query locally
if (searchQuery) {
logs = logs.filter(line => line.toLowerCase().includes(searchQuery));
}
if (logs.length === 0) {
consoleContainer.innerHTML = '<div class="settings-system-logs-placeholder">No logs found matching current filters.</div>';
return;
}
// Preserve scroll position if user is reading previous logs
const atBottom = consoleContainer.scrollHeight - consoleContainer.scrollTop - consoleContainer.clientHeight < 40;
consoleContainer.innerHTML = logs.map(line => {
let levelClass = 'log-line-default';
if (line.includes(' - INFO - ')) {
levelClass = 'log-line-info';
} else if (line.includes(' - WARNING - ')) {
levelClass = 'log-line-warning';
} else if (line.includes(' - ERROR - ') || line.includes(' - CRITICAL - ')) {
levelClass = 'log-line-error';
} else if (line.includes(' - DEBUG - ')) {
levelClass = 'log-line-debug';
}
// XSS safe escape
const escaped = line
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
return `<div class="log-line ${levelClass}">${escaped}</div>`;
}).join('');
if (!isAutoPoll || atBottom) {
consoleContainer.scrollTop = consoleContainer.scrollHeight;
}
}
async function loadLogs(isAutoPoll = false) {
const consoleContainer = el('log-console-container');
const limitSelect = el('log-limit-select');
if (!consoleContainer) return;
const limit = limitSelect ? limitSelect.value : 200;
if (logsAbortController) {
logsAbortController.abort();
}
logsAbortController = new AbortController();
const { signal } = logsAbortController;
try {
const res = await fetch(`/api/diagnostics/logs?limit=${limit}`, {
credentials: 'same-origin',
signal
});
if (!res.ok) {
if (!isAutoPoll) {
consoleContainer.innerHTML = '';
const errDiv = document.createElement('div');
errDiv.style.color = 'var(--red)';
errDiv.style.fontWeight = '600';
errDiv.textContent = `Failed to load logs: HTTP ${res.status}`;
consoleContainer.appendChild(errDiv);
}
return;
}
const data = await res.json();
if (data.status !== 'success' || !data.logs) {
if (!isAutoPoll) {
consoleContainer.innerHTML = '';
const errDiv = document.createElement('div');
errDiv.style.color = 'var(--red)';
errDiv.style.fontWeight = '600';
errDiv.textContent = 'Failed to parse logs data';
consoleContainer.appendChild(errDiv);
}
return;
}
cachedLogs = data.logs;
renderLogs(isAutoPoll);
} catch (err) {
if (err.name === 'AbortError') {
return; // Silently ignore deliberate abort
}
if (!isAutoPoll) {
consoleContainer.innerHTML = '';
const errDiv = document.createElement('div');
errDiv.style.color = 'var(--red)';
errDiv.style.fontWeight = '600';
errDiv.textContent = `Error retrieving logs: ${err.message}`;
consoleContainer.appendChild(errDiv);
}
} finally {
if (logsAbortController?.signal === signal) {
logsAbortController = null;
}
}
}
function startLogsPolling() {
if (isLogsPolling) return;
isLogsPolling = true;
const toggle = el('log-auto-refresh-toggle');
if (toggle) toggle.checked = true;
logsPollInterval = setInterval(() => {
const modal = el('settings-modal');
const systemPanel = el('settings-modal')?.querySelector('[data-settings-panel="system"]');
// Safe self-cleanup if modal or panel is hidden/closed
if (!modal || modal.classList.contains('hidden') || !systemPanel || systemPanel.classList.contains('hidden')) {
stopLogsPolling();
return;
}
loadLogs(true);
}, 3000);
}
function stopLogsPolling() {
if (!isLogsPolling) return;
isLogsPolling = false;
if (logsPollInterval) {
clearInterval(logsPollInterval);
logsPollInterval = null;
}
const toggle = el('log-auto-refresh-toggle');
if (toggle) toggle.checked = false;
}
function initLogsView() {
const refreshBtn = el('log-refresh-btn');
const levelSelect = el('log-level-select');
const limitSelect = el('log-limit-select');
const searchInput = el('log-search-input');
const autoRefreshToggle = el('log-auto-refresh-toggle');
if (refreshBtn) refreshBtn.addEventListener('click', () => loadLogs(false));
if (levelSelect) levelSelect.addEventListener('change', () => renderLogs(false));
if (limitSelect) limitSelect.addEventListener('change', () => loadLogs(false));
if (searchInput) searchInput.addEventListener('input', () => renderLogs(false));
if (autoRefreshToggle) {
autoRefreshToggle.addEventListener('change', (e) => {
if (e.target.checked) {
startLogsPolling();
} else {
stopLogsPolling();
}
});
}
// Initial fetch on view loading
loadLogs(false);
}
/* ═══════════════════════════════════════════
INIT & REFRESH
═══════════════════════════════════════════ */
function initAll() {
modalEl = el('settings-modal');
const inits = [initSignupToggle, initAddUser, initEndpointForm, initMcpForm, initCalDAV, initBackup, initDangerZone, initTokenForm, () => settingsModule.initIntegrations()];
const inits = [
initSignupToggle, initAddUser, initEndpointForm, initMcpForm,
initCalDAV, initBackup, initDangerZone, initTokenForm, initLogsView,
() => settingsModule.initIntegrations()
];
for (const fn of inits) {
try { fn(); } catch (e) { console.error('Admin init error in', fn.name || 'anonymous', e); }
}
@@ -2507,6 +2701,7 @@ function refreshAll() {
loadBuiltinTools();
loadMcpServers();
loadTokens();
loadLogs(false);
}
/* ═══════════════════════════════════════════
@@ -2523,6 +2718,7 @@ export function open(tab) {
}
export function close() {
stopLogsPolling();
settingsModule.close();
}
+99
View File
@@ -36678,3 +36678,102 @@ body.theme-frosted .modal {
border-top: 1px solid var(--border);
}
.workspace-note { margin: 0 0 8px; font-size: 11px; line-height: 1.4; }
/* Real-time Diagnostics Log Terminal UI Styles */
.settings-system-logs-svg {
vertical-align: -2px;
margin-right: 5px;
opacity: 0.6;
}
.settings-system-logs-toggle-sub {
margin-bottom: 12px;
}
.settings-system-logs-col {
gap: 10px;
}
.settings-system-logs-controls {
display: flex;
gap: 8px;
flex-wrap: wrap;
align-items: center;
}
.settings-system-logs-search {
padding: 6px 8px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 4px;
color: var(--fg);
font-family: inherit;
font-size: 11px;
flex: 1;
min-width: 140px;
}
.settings-system-logs-select {
padding: 5px 8px;
background: var(--bg);
border: 1px solid var(--border);
border-radius: 4px;
color: var(--fg);
font-family: inherit;
font-size: 11px;
min-width: 90px;
}
#log-refresh-btn {
height: 27px;
display: flex;
align-items: center;
gap: 4px;
padding: 0 8px;
}
.settings-system-logs-refresh-svg {
pointer-events: none;
}
.settings-system-logs-autopoll-container {
display: inline-flex;
align-items: center;
gap: 6px;
font-size: 11px;
user-select: none;
margin-left: auto;
}
#log-console-container {
background: #13151a;
border: 1px solid var(--border);
border-radius: 6px;
padding: 12px;
font-family: Consolas, 'Fira Code', Monaco, 'Courier New', monospace;
font-size: 11px;
height: 280px;
max-height: 280px;
overflow-y: auto;
white-space: pre-wrap;
word-break: break-all;
color: #d1d4e0;
box-shadow: inset 0 2px 8px rgba(0,0,0,0.5);
}
.settings-system-logs-placeholder {
color: var(--color-text-dim, #7f8c8d);
font-style: italic;
font-family: inherit;
}
.log-line {
margin-bottom: 3px;
line-height: 1.4;
font-size: 11px;
font-family: inherit;
}
.log-line-info {
color: var(--green, #50fa7b);
}
.log-line-warning {
color: var(--warn, #f0ad4e);
}
.log-line-error {
color: var(--red, #e06c75);
}
.log-line-debug {
color: var(--color-muted, #888);
}
.log-line-default {
color: var(--fg, #9cdef2);
}
+110
View File
@@ -0,0 +1,110 @@
"""Route-level regression tests for GET /api/diagnostics/logs."""
import pytest
fastapi = pytest.importorskip("fastapi")
pytest.importorskip("starlette.testclient")
from fastapi import FastAPI, HTTPException, Request
from starlette.testclient import TestClient
# Importing the route module pulls a few app deps; skip cleanly if unavailable.
diag = pytest.importorskip("routes.diagnostics_routes")
def _client_with_admin_gate(monkeypatch, gate, tmp_path=None):
"""Mount the diagnostics router with a mock require_admin and DATA_DIR."""
monkeypatch.setattr(diag, "require_admin", gate)
if tmp_path:
monkeypatch.setattr(diag, "DATA_DIR", str(tmp_path))
app = FastAPI()
app.include_router(diag.setup_diagnostics_routes(
rag_manager=None, rag_available=False, research_handler=None,
memory_vector=None))
return TestClient(app, raise_server_exceptions=False)
def test_logs_unauthenticated_rejected(monkeypatch):
def gate(_request: Request):
raise HTTPException(401, "Not authenticated")
client = _client_with_admin_gate(monkeypatch, gate)
r = client.get("/api/diagnostics/logs")
assert r.status_code == 401
def test_logs_non_admin_forbidden(monkeypatch):
def gate(_request: Request):
raise HTTPException(403, "Admin only")
client = _client_with_admin_gate(monkeypatch, gate)
r = client.get("/api/diagnostics/logs")
assert r.status_code == 403
def test_logs_missing_file(monkeypatch, tmp_path):
def gate(_request: Request):
return None
client = _client_with_admin_gate(monkeypatch, gate, tmp_path)
r = client.get("/api/diagnostics/logs")
assert r.status_code == 200
body = r.json()
assert body["status"] == "success"
assert body["logs"] == []
def test_logs_tailing_and_clamping(monkeypatch, tmp_path):
# Setup mock log file
log_dir = tmp_path / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
log_file = log_dir / "app.log"
# Write 1500 log lines
lines = [f"Log line {i}\n" for i in range(1, 1501)]
log_file.write_text("".join(lines), encoding="utf-8")
def gate(_request: Request):
return None
client = _client_with_admin_gate(monkeypatch, gate, tmp_path)
# 1. Default limit (200)
r = client.get("/api/diagnostics/logs")
assert r.status_code == 200
body = r.json()
assert len(body["logs"]) == 200
assert body["logs"][-1] == "Log line 1500"
assert body["logs"][0] == "Log line 1301"
# 2. Clamped upper bound (limit=2000 -> clamps to 1000)
r = client.get("/api/diagnostics/logs?limit=2000")
assert r.status_code == 200
body = r.json()
assert len(body["logs"]) == 1000
assert body["logs"][-1] == "Log line 1500"
assert body["logs"][0] == "Log line 501"
# 3. Clamped lower bound (limit=-5 -> clamps to 1)
r = client.get("/api/diagnostics/logs?limit=-5")
assert r.status_code == 200
body = r.json()
assert len(body["logs"]) == 1
assert body["logs"][0] == "Log line 1500"
# 4. Clamp limit=0 -> clamps to 1
r = client.get("/api/diagnostics/logs?limit=0")
assert r.status_code == 200
body = r.json()
assert len(body["logs"]) == 1
assert body["logs"][0] == "Log line 1500"
# 5. Exact custom limit
r = client.get("/api/diagnostics/logs?limit=5")
assert r.status_code == 200
body = r.json()
assert len(body["logs"]) == 5
assert body["logs"] == [
"Log line 1496",
"Log line 1497",
"Log line 1498",
"Log line 1499",
"Log line 1500"
]