feat(ui): add real-time diagnostic logs console (#974)

* feat(diagnostics): add admin-gated real-time diagnostics logs terminal UI

* feat(ui): resolve diagnostics logs feedback and optimize client-side caching

* feat(ui): resolve diagnostics logs feedback

app.py

@@ -69,10 +69,37 @@ from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_imag
 from starlette.responses import RedirectResponse
 
 # ========= LOGGING =========
-logging.basicConfig(
+import logging.handlers
-    level=logging.INFO,
+from core.constants import DATA_DIR
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+
+_root_logger = logging.getLogger()
+_root_logger.setLevel(logging.INFO)
+_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Clear existing handlers to avoid duplicates
+for _h in list(_root_logger.handlers):
+    _root_logger.removeHandler(_h)
+
+_console_h = logging.StreamHandler()
+_console_h.setFormatter(_formatter)
+_root_logger.addHandler(_console_h)
+
+try:
+    _log_dir = os.path.join(DATA_DIR, "logs")
+    os.makedirs(_log_dir, exist_ok=True)
+    _log_file = os.path.join(_log_dir, "app.log")
+
+    # RotatingFileHandler is not multi-process safe (e.g. if uvicorn is run with --workers N).
+    # Odysseus is single-process by convention, so this is acceptable, but be aware that
+    # concurrent log rotation issues can arise if multiple workers are configured.
+    _file_h = logging.handlers.RotatingFileHandler(
+        _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
     )
+    _file_h.setFormatter(_formatter)
+    _root_logger.addHandler(_file_h)
+except Exception as e:
+    _root_logger.warning(f"Failed to initialize file logging handler (falling back to console-only): {e}")
+
 logger = logging.getLogger(__name__)
 
 # ========= APP =========

routes/diagnostics_routes.py

@@ -1,12 +1,13 @@
 """Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research."""
 
 import logging
+import os
 from typing import Dict, Any
 
 from fastapi import APIRouter, HTTPException, Form, Request
 
 from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async
-from core.constants import DEFAULT_HOST
+from core.constants import DEFAULT_HOST, DATA_DIR
 from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,30 @@ def setup_diagnostics_routes(
         from src.service_health import collect_service_health
         return await collect_service_health(rag_manager, memory_vector)
 
+    @router.get("/api/diagnostics/logs")
+    async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]:
+        require_admin(request)
+        limit = max(1, min(limit, 1000))
+        try:
+            log_file = os.path.join(DATA_DIR, "logs", "app.log")
+            if not os.path.exists(log_file):
+                return {"status": "success", "logs": []}
+
+            # Safe tail read of the log file (max 5MB via rotation)
+            with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
+                lines = f.readlines()
+
+            tail_lines = lines[-limit:] if len(lines) > limit else lines
+            tail_lines = [line.rstrip('\r\n') for line in tail_lines]
+
+            return {
+                "status": "success",
+                "logs": tail_lines
+            }
+        except Exception as e:
+            logger.error(f"Diagnostics logs retrieval error: {e}")
+            raise HTTPException(500, f"Failed to retrieve logs: {str(e)}")
+
     @router.get("/api/db/stats")
     async def get_database_stats(request: Request) -> Dict[str, Any]:
         require_admin(request)

static/index.html

@@ -2232,6 +2232,61 @@
         <!-- ═══ SYSTEM TAB ═══ -->
         <div data-settings-panel="system" class="hidden">
 
+          <div class="admin-card" id="settings-system-logs-card">
+            <h2>
+              <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-svg">
+                <polyline points="4 17 10 11 4 5"></polyline>
+                <line x1="12" y1="19" x2="20" y2="19"></line>
+              </svg>
+              Terminal Logs
+            </h2>
+            <div class="admin-toggle-sub settings-system-logs-toggle-sub">Live diagnostic logs and system output from the Odysseus process.</div>
+
+            <div class="settings-col settings-system-logs-col">
+              <!-- Controls row -->
+              <div class="settings-system-logs-controls">
+                <!-- Search input -->
+                <input type="text" id="log-search-input" placeholder="Search logs..." class="settings-system-logs-search">
+
+                <!-- Level select -->
+                <select id="log-level-select" class="settings-system-logs-select">
+                  <option value="ALL">All Levels</option>
+                  <option value="INFO">INFO</option>
+                  <option value="WARNING">WARNING</option>
+                  <option value="ERROR">ERROR</option>
+                  <option value="DEBUG">DEBUG</option>
+                </select>
+
+                <!-- Limit select -->
+                <select id="log-limit-select" class="settings-system-logs-select">
+                  <option value="100">100 lines</option>
+                  <option value="200" selected>200 lines</option>
+                  <option value="500">500 lines</option>
+                  <option value="1000">1000 lines</option>
+                </select>
+
+                <!-- Refresh Button -->
+                <button type="button" class="admin-btn-sm" id="log-refresh-btn">
+                  <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="settings-system-logs-refresh-svg"><path d="M21.5 2v6h-6M21.34 15.57a10 10 0 1 1-.57-8.38l5.67-5.67"/></svg>
+                  Refresh
+                </button>
+
+                <!-- Auto-refresh switch -->
+                <div class="settings-system-logs-autopoll-container">
+                  <label class="admin-switch" title="Auto-polling every 3 seconds">
+                    <input type="checkbox" id="log-auto-refresh-toggle">
+                    <span class="admin-slider"></span>
+                  </label>
+                  <span>Auto-poll</span>
+                </div>
+              </div>
+
+              <!-- Console container -->
+              <div id="log-console-container">
+                <div class="settings-system-logs-placeholder">Initializing logs terminal viewer...</div>
+              </div>
+            </div>
+          </div>
           <div class="admin-card">
             <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>Data Backup</h2>
             <div class="admin-toggle-sub" style="margin-bottom:8px">Export or import your user data (memories, presets, settings, skills, preferences) as a JSON file.</div>

static/js/admin.js

@@ -2488,12 +2488,206 @@ function initDangerZone() {
   });
 }
 
+/* ═══════════════════════════════════════════
+   TERMINAL LOGS VIEWER
+   ═══════════════════════════════════════════ */
+let logsPollInterval = null;
+let isLogsPolling = false;
+let cachedLogs = [];
+let logsAbortController = null;
+
+function renderLogs(isAutoPoll = false) {
+  const consoleContainer = el('log-console-container');
+  const levelSelect = el('log-level-select');
+  const searchInput = el('log-search-input');
+
+  if (!consoleContainer) return;
+
+  const levelFilter = levelSelect ? levelSelect.value : 'ALL';
+  const searchQuery = searchInput ? searchInput.value.trim().toLowerCase() : '';
+
+  let logs = cachedLogs;
+
+  // Filter by level locally
+  if (levelFilter !== 'ALL') {
+    logs = logs.filter(line => line.includes(` - ${levelFilter} - `));
+  }
+
+  // Filter by search query locally
+  if (searchQuery) {
+    logs = logs.filter(line => line.toLowerCase().includes(searchQuery));
+  }
+
+  if (logs.length === 0) {
+    consoleContainer.innerHTML = '<div class="settings-system-logs-placeholder">No logs found matching current filters.</div>';
+    return;
+  }
+
+  // Preserve scroll position if user is reading previous logs
+  const atBottom = consoleContainer.scrollHeight - consoleContainer.scrollTop - consoleContainer.clientHeight < 40;
+
+  consoleContainer.innerHTML = logs.map(line => {
+    let levelClass = 'log-line-default';
+
+    if (line.includes(' - INFO - ')) {
+      levelClass = 'log-line-info';
+    } else if (line.includes(' - WARNING - ')) {
+      levelClass = 'log-line-warning';
+    } else if (line.includes(' - ERROR - ') || line.includes(' - CRITICAL - ')) {
+      levelClass = 'log-line-error';
+    } else if (line.includes(' - DEBUG - ')) {
+      levelClass = 'log-line-debug';
+    }
+
+    // XSS safe escape
+    const escaped = line
+      .replace(/&/g, '&amp;')
+      .replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;')
+      .replace(/"/g, '&quot;')
+      .replace(/'/g, '&#039;');
+
+    return `<div class="log-line ${levelClass}">${escaped}</div>`;
+  }).join('');
+
+  if (!isAutoPoll || atBottom) {
+    consoleContainer.scrollTop = consoleContainer.scrollHeight;
+  }
+}
+
+async function loadLogs(isAutoPoll = false) {
+  const consoleContainer = el('log-console-container');
+  const limitSelect = el('log-limit-select');
+
+  if (!consoleContainer) return;
+
+  const limit = limitSelect ? limitSelect.value : 200;
+
+  if (logsAbortController) {
+    logsAbortController.abort();
+  }
+  logsAbortController = new AbortController();
+  const { signal } = logsAbortController;
+
+  try {
+    const res = await fetch(`/api/diagnostics/logs?limit=${limit}`, {
+      credentials: 'same-origin',
+      signal
+    });
+
+    if (!res.ok) {
+      if (!isAutoPoll) {
+        consoleContainer.innerHTML = '';
+        const errDiv = document.createElement('div');
+        errDiv.style.color = 'var(--red)';
+        errDiv.style.fontWeight = '600';
+        errDiv.textContent = `Failed to load logs: HTTP ${res.status}`;
+        consoleContainer.appendChild(errDiv);
+      }
+      return;
+    }
+
+    const data = await res.json();
+    if (data.status !== 'success' || !data.logs) {
+      if (!isAutoPoll) {
+        consoleContainer.innerHTML = '';
+        const errDiv = document.createElement('div');
+        errDiv.style.color = 'var(--red)';
+        errDiv.style.fontWeight = '600';
+        errDiv.textContent = 'Failed to parse logs data';
+        consoleContainer.appendChild(errDiv);
+      }
+      return;
+    }
+
+    cachedLogs = data.logs;
+    renderLogs(isAutoPoll);
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      return; // Silently ignore deliberate abort
+    }
+    if (!isAutoPoll) {
+      consoleContainer.innerHTML = '';
+      const errDiv = document.createElement('div');
+      errDiv.style.color = 'var(--red)';
+      errDiv.style.fontWeight = '600';
+      errDiv.textContent = `Error retrieving logs: ${err.message}`;
+      consoleContainer.appendChild(errDiv);
+    }
+  } finally {
+    if (logsAbortController?.signal === signal) {
+      logsAbortController = null;
+    }
+  }
+}
+
+function startLogsPolling() {
+  if (isLogsPolling) return;
+  isLogsPolling = true;
+  const toggle = el('log-auto-refresh-toggle');
+  if (toggle) toggle.checked = true;
+
+  logsPollInterval = setInterval(() => {
+    const modal = el('settings-modal');
+    const systemPanel = el('settings-modal')?.querySelector('[data-settings-panel="system"]');
+
+    // Safe self-cleanup if modal or panel is hidden/closed
+    if (!modal || modal.classList.contains('hidden') || !systemPanel || systemPanel.classList.contains('hidden')) {
+      stopLogsPolling();
+      return;
+    }
+
+    loadLogs(true);
+  }, 3000);
+}
+
+function stopLogsPolling() {
+  if (!isLogsPolling) return;
+  isLogsPolling = false;
+  if (logsPollInterval) {
+    clearInterval(logsPollInterval);
+    logsPollInterval = null;
+  }
+  const toggle = el('log-auto-refresh-toggle');
+  if (toggle) toggle.checked = false;
+}
+
+function initLogsView() {
+  const refreshBtn = el('log-refresh-btn');
+  const levelSelect = el('log-level-select');
+  const limitSelect = el('log-limit-select');
+  const searchInput = el('log-search-input');
+  const autoRefreshToggle = el('log-auto-refresh-toggle');
+
+  if (refreshBtn) refreshBtn.addEventListener('click', () => loadLogs(false));
+  if (levelSelect) levelSelect.addEventListener('change', () => renderLogs(false));
+  if (limitSelect) limitSelect.addEventListener('change', () => loadLogs(false));
+  if (searchInput) searchInput.addEventListener('input', () => renderLogs(false));
+
+  if (autoRefreshToggle) {
+    autoRefreshToggle.addEventListener('change', (e) => {
+      if (e.target.checked) {
+        startLogsPolling();
+      } else {
+        stopLogsPolling();
+      }
+    });
+  }
+
+  // Initial fetch on view loading
+  loadLogs(false);
+}
+
 /* ═══════════════════════════════════════════
    INIT & REFRESH
    ═══════════════════════════════════════════ */
 function initAll() {
   modalEl = el('settings-modal');
-  const inits = [initSignupToggle, initAddUser, initEndpointForm, initMcpForm, initCalDAV, initBackup, initDangerZone, initTokenForm, () => settingsModule.initIntegrations()];
+  const inits = [
+    initSignupToggle, initAddUser, initEndpointForm, initMcpForm,
+    initCalDAV, initBackup, initDangerZone, initTokenForm, initLogsView,
+    () => settingsModule.initIntegrations()
+  ];
   for (const fn of inits) {
     try { fn(); } catch (e) { console.error('Admin init error in', fn.name || 'anonymous', e); }
   }
@@ -2507,6 +2701,7 @@ function refreshAll() {
   loadBuiltinTools();
   loadMcpServers();
   loadTokens();
+  loadLogs(false);
 }
 
 /* ═══════════════════════════════════════════
@@ -2523,6 +2718,7 @@ export function open(tab) {
 }
 
 export function close() {
+  stopLogsPolling();
   settingsModule.close();
 }
 

static/style.css

@@ -36678,3 +36678,102 @@ body.theme-frosted .modal {
   border-top: 1px solid var(--border);
 }
 .workspace-note { margin: 0 0 8px; font-size: 11px; line-height: 1.4; }
+
+/* Real-time Diagnostics Log Terminal UI Styles */
+.settings-system-logs-svg {
+  vertical-align: -2px;
+  margin-right: 5px;
+  opacity: 0.6;
+}
+.settings-system-logs-toggle-sub {
+  margin-bottom: 12px;
+}
+.settings-system-logs-col {
+  gap: 10px;
+}
+.settings-system-logs-controls {
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+  align-items: center;
+}
+.settings-system-logs-search {
+  padding: 6px 8px;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  color: var(--fg);
+  font-family: inherit;
+  font-size: 11px;
+  flex: 1;
+  min-width: 140px;
+}
+.settings-system-logs-select {
+  padding: 5px 8px;
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  color: var(--fg);
+  font-family: inherit;
+  font-size: 11px;
+  min-width: 90px;
+}
+#log-refresh-btn {
+  height: 27px;
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  padding: 0 8px;
+}
+.settings-system-logs-refresh-svg {
+  pointer-events: none;
+}
+.settings-system-logs-autopoll-container {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  font-size: 11px;
+  user-select: none;
+  margin-left: auto;
+}
+#log-console-container {
+  background: #13151a;
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  padding: 12px;
+  font-family: Consolas, 'Fira Code', Monaco, 'Courier New', monospace;
+  font-size: 11px;
+  height: 280px;
+  max-height: 280px;
+  overflow-y: auto;
+  white-space: pre-wrap;
+  word-break: break-all;
+  color: #d1d4e0;
+  box-shadow: inset 0 2px 8px rgba(0,0,0,0.5);
+}
+.settings-system-logs-placeholder {
+  color: var(--color-text-dim, #7f8c8d);
+  font-style: italic;
+  font-family: inherit;
+}
+.log-line {
+  margin-bottom: 3px;
+  line-height: 1.4;
+  font-size: 11px;
+  font-family: inherit;
+}
+.log-line-info {
+  color: var(--green, #50fa7b);
+}
+.log-line-warning {
+  color: var(--warn, #f0ad4e);
+}
+.log-line-error {
+  color: var(--red, #e06c75);
+}
+.log-line-debug {
+  color: var(--color-muted, #888);
+}
+.log-line-default {
+  color: var(--fg, #9cdef2);
+}

tests/test_diagnostics_logs.py

@@ -0,0 +1,110 @@
+"""Route-level regression tests for GET /api/diagnostics/logs."""
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+pytest.importorskip("starlette.testclient")
+
+from fastapi import FastAPI, HTTPException, Request
+from starlette.testclient import TestClient
+
+# Importing the route module pulls a few app deps; skip cleanly if unavailable.
+diag = pytest.importorskip("routes.diagnostics_routes")
+
+
+def _client_with_admin_gate(monkeypatch, gate, tmp_path=None):
+    """Mount the diagnostics router with a mock require_admin and DATA_DIR."""
+    monkeypatch.setattr(diag, "require_admin", gate)
+    if tmp_path:
+        monkeypatch.setattr(diag, "DATA_DIR", str(tmp_path))
+
+    app = FastAPI()
+    app.include_router(diag.setup_diagnostics_routes(
+        rag_manager=None, rag_available=False, research_handler=None,
+        memory_vector=None))
+    return TestClient(app, raise_server_exceptions=False)
+
+
+def test_logs_unauthenticated_rejected(monkeypatch):
+    def gate(_request: Request):
+        raise HTTPException(401, "Not authenticated")
+    client = _client_with_admin_gate(monkeypatch, gate)
+    r = client.get("/api/diagnostics/logs")
+    assert r.status_code == 401
+
+
+def test_logs_non_admin_forbidden(monkeypatch):
+    def gate(_request: Request):
+        raise HTTPException(403, "Admin only")
+    client = _client_with_admin_gate(monkeypatch, gate)
+    r = client.get("/api/diagnostics/logs")
+    assert r.status_code == 403
+
+
+def test_logs_missing_file(monkeypatch, tmp_path):
+    def gate(_request: Request):
+        return None
+    client = _client_with_admin_gate(monkeypatch, gate, tmp_path)
+    r = client.get("/api/diagnostics/logs")
+    assert r.status_code == 200
+    body = r.json()
+    assert body["status"] == "success"
+    assert body["logs"] == []
+
+
+def test_logs_tailing_and_clamping(monkeypatch, tmp_path):
+    # Setup mock log file
+    log_dir = tmp_path / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / "app.log"
+
+    # Write 1500 log lines
+    lines = [f"Log line {i}\n" for i in range(1, 1501)]
+    log_file.write_text("".join(lines), encoding="utf-8")
+
+    def gate(_request: Request):
+        return None
+    client = _client_with_admin_gate(monkeypatch, gate, tmp_path)
+
+    # 1. Default limit (200)
+    r = client.get("/api/diagnostics/logs")
+    assert r.status_code == 200
+    body = r.json()
+    assert len(body["logs"]) == 200
+    assert body["logs"][-1] == "Log line 1500"
+    assert body["logs"][0] == "Log line 1301"
+
+    # 2. Clamped upper bound (limit=2000 -> clamps to 1000)
+    r = client.get("/api/diagnostics/logs?limit=2000")
+    assert r.status_code == 200
+    body = r.json()
+    assert len(body["logs"]) == 1000
+    assert body["logs"][-1] == "Log line 1500"
+    assert body["logs"][0] == "Log line 501"
+
+    # 3. Clamped lower bound (limit=-5 -> clamps to 1)
+    r = client.get("/api/diagnostics/logs?limit=-5")
+    assert r.status_code == 200
+    body = r.json()
+    assert len(body["logs"]) == 1
+    assert body["logs"][0] == "Log line 1500"
+
+    # 4. Clamp limit=0 -> clamps to 1
+    r = client.get("/api/diagnostics/logs?limit=0")
+    assert r.status_code == 200
+    body = r.json()
+    assert len(body["logs"]) == 1
+    assert body["logs"][0] == "Log line 1500"
+
+    # 5. Exact custom limit
+    r = client.get("/api/diagnostics/logs?limit=5")
+    assert r.status_code == 200
+    body = r.json()
+    assert len(body["logs"]) == 5
+    assert body["logs"] == [
+        "Log line 1496",
+        "Log line 1497",
+        "Log line 1498",
+        "Log line 1499",
+        "Log line 1500"
+    ]